Compare commits

..

4 Commits

Author SHA1 Message Date
Ben McClelland
7086579590 fix: list objects trim common prefixes that match marker prefix
This checks to see if the common prefix is before the marker and
thus would have been returned in earlier list objects request.

The error case was aws cli listing multiple entries for the same
common prefix when the listing required multiple pagination
requests.

Fixes #778
2024-09-16 14:55:55 -07:00
jonaustin09
dea5e0c0b2 feat: Implemented object versioning for multipart uploads. Implemented integration tests for the versioning implementation for multipart uploads 2024-09-16 14:51:22 -07:00
jonaustin09
16995acc17 feat: Added integration tests for bucket object versioning. Made a couple of bug fixes in the versioning implementation 2024-09-16 14:51:20 -07:00
jonaustin09
1f41f91f2d feat: basic logic implementation of bucket object versioning in posix backend
New posix backend option --versioning-dir will enable storing object versions
in specified directory.
2024-09-16 14:42:24 -07:00
192 changed files with 3989 additions and 10628 deletions

View File

@@ -1,37 +0,0 @@
name: azurite functional tests
on: pull_request
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
id: go
- name: Set up Docker Compose
run: |
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
- name: Wait for Azurite to be ready
run: sleep 40
- name: Get Dependencies
run: |
go mod download
- name: Build and Run
run: |
make
./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
- name: Shut down services
run: |
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

View File

@@ -1,23 +0,0 @@
name: betteralign
on: pull_request
jobs:
build:
name: Check
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "stable"
id: go
- name: Install betteralign
run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest
- name: Run betteralign
run: betteralign -test_files ./...

28
.github/workflows/docker-bats.yaml vendored Normal file
View File

@@ -0,0 +1,28 @@
name: docker bats tests
on: pull_request
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Docker Image
run: |
mv tests/.env.docker.default tests/.env.docker
mv tests/.secrets.default tests/.secrets
docker build --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
--build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" --build-arg="MC_FOLDER=linux-amd64" \
--progress=plain -f tests/Dockerfile_test_bats -t bats_test .
- name: Set up Docker Compose
run: sudo apt-get install -y docker-compose
- name: Run Docker Container
run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from posix_backend posix_backend

View File

@@ -1,28 +0,0 @@
name: docker bats tests
on: pull_request
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build Docker Image
run: |
cp tests/.env.docker.default tests/.env.docker
cp tests/.secrets.default tests/.secrets
docker build \
--build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
--build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
--build-arg="MC_FOLDER=linux-amd64" \
--progress=plain \
-f tests/Dockerfile_test_bats \
-t bats_test .
- name: Run Docker Container
run: |
docker compose -f tests/docker-compose-bats.yml --project-directory . \
up --exit-code-from s3api_np_only s3api_np_only

View File

@@ -1,8 +1,7 @@
name: functional tests
on: pull_request
jobs:
build:
name: RunTests
runs-on: ubuntu-latest
@@ -19,7 +18,7 @@ jobs:
- name: Get Dependencies
run: |
go mod download
go get -v -t -d ./...
- name: Build and Run
run: |

View File

@@ -8,95 +8,115 @@ jobs:
fail-fast: false
matrix:
include:
- set: "mc, posix, non-file count, non-static, folder IAM"
- set: "s3cmd, posix"
LOCAL_FOLDER: /tmp/gw1
BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
IAM_TYPE: folder
RUN_SET: "mc-non-file-count"
USERS_FOLDER: /tmp/iam1
AWS_ENDPOINT_URL: https://127.0.0.1:7070
RUN_SET: "s3cmd"
RECREATE_BUCKETS: "true"
PORT: 7070
BACKEND: "posix"
- set: "mc, posix, file count, non-static, folder IAM"
- set: "s3, posix"
LOCAL_FOLDER: /tmp/gw2
BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
IAM_TYPE: folder
RUN_SET: "mc-file-count"
USERS_FOLDER: /tmp/iam2
AWS_ENDPOINT_URL: https://127.0.0.1:7071
RUN_SET: "s3"
RECREATE_BUCKETS: "true"
PORT: 7071
BACKEND: "posix"
- set: "REST, posix, non-static, all, folder IAM"
- set: "s3api, posix"
LOCAL_FOLDER: /tmp/gw3
BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam3
AWS_ENDPOINT_URL: https://127.0.0.1:7072
RUN_SET: "s3api"
RECREATE_BUCKETS: "true"
PORT: 7072
BACKEND: "posix"
- set: "mc, posix"
LOCAL_FOLDER: /tmp/gw4
BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam4
AWS_ENDPOINT_URL: https://127.0.0.1:7073
RUN_SET: "mc"
RECREATE_BUCKETS: "true"
PORT: 7073
BACKEND: "posix"
- set: "s3api-user, posix, s3 IAM"
LOCAL_FOLDER: /tmp/gw5
BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
IAM_TYPE: s3
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7074
RUN_SET: "s3api-user"
RECREATE_BUCKETS: "true"
PORT: 7074
BACKEND: "posix"
- set: "s3api non-policy, static buckets"
LOCAL_FOLDER: /tmp/gw6
BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam6
AWS_ENDPOINT_URL: https://127.0.0.1:7075
RUN_SET: "s3api-non-policy"
RECREATE_BUCKETS: "false"
PORT: 7075
BACKEND: "posix"
- set: "s3api, s3 backend"
LOCAL_FOLDER: /tmp/gw7
BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam7
AWS_ENDPOINT_URL: https://127.0.0.1:7076
RUN_SET: "s3api"
RECREATE_BUCKETS: "true"
PORT: 7076
BACKEND: "s3"
- set: "REST, posix"
LOCAL_FOLDER: /tmp/gw8
BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam8
AWS_ENDPOINT_URL: https://127.0.0.1:7077
RUN_SET: "rest"
RECREATE_BUCKETS: "true"
PORT: 7077
BACKEND: "posix"
- set: "s3, posix, non-file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3-non-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, policy, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-policy"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, user, non-static, s3 IAM"
IAM_TYPE: s3
RUN_SET: "s3api-user"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, bucket, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-bucket"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, posix, multipart, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-multipart"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, posix, object, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-object"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, posix, policy, static, folder IAM"
- set: "s3api policy, static buckets"
LOCAL_FOLDER: /tmp/gw9
BUCKET_ONE_NAME: versity-gwtest-bucket-one-8
BUCKET_TWO_NAME: versity-gwtest-bucket-two-8
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam9
AWS_ENDPOINT_URL: https://127.0.0.1:7078
RUN_SET: "s3api-policy"
RECREATE_BUCKETS: "false"
PORT: 7078
BACKEND: "posix"
- set: "s3api, posix, user, static, folder IAM"
- set: "s3api user, static buckets"
LOCAL_FOLDER: /tmp/gw10
BUCKET_ONE_NAME: versity-gwtest-bucket-one-9
BUCKET_TWO_NAME: versity-gwtest-bucket-two-9
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam10
AWS_ENDPOINT_URL: https://127.0.0.1:7079
RUN_SET: "s3api-user"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, s3, multipart|object, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
RECREATE_BUCKETS: "true"
BACKEND: "s3"
- set: "s3api, s3, policy|user, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-policy,s3api-user"
RECREATE_BUCKETS: "true"
BACKEND: "s3"
- set: "s3cmd, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3cmd, posix, non-user, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-non-user"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3cmd, posix, user, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-user"
RECREATE_BUCKETS: "true"
PORT: 7079
BACKEND: "posix"
steps:
- name: Check out code into the Go module directory
@@ -132,10 +152,17 @@ jobs:
run: |
sudo apt-get install libxml2-utils
- name: Build and run
- name: Build and run, posix backend
env:
LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
IAM_TYPE: ${{ matrix.IAM_TYPE }}
AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
RUN_SET: ${{ matrix.RUN_SET }}
PORT: ${{ matrix.PORT }}
AWS_PROFILE: versity
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
@@ -143,13 +170,6 @@ jobs:
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
CERT: ${{ github.workspace }}/cert.pem
KEY: ${{ github.workspace }}/versitygw.pem
LOCAL_FOLDER: /tmp/gw
BUCKET_ONE_NAME: versity-gwtest-bucket-one
BUCKET_TWO_NAME: versity-gwtest-bucket-two
USERS_FOLDER: /tmp/iam
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7070
PORT: 7070
S3CMD_CONFIG: tests/s3cfg.local.default
MC_ALIAS: versity
LOG_LEVEL: 4
@@ -159,10 +179,6 @@ jobs:
USERNAME_TWO: HIJKLMN
PASSWORD_TWO: 8901234
TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
REMOVE_TEST_FILE_FOLDER: true
VERSIONING_DIR: ${{ github.workspace }}/versioning
COMMAND_LOG: command.log
TIME_LOG: time.log
run: |
make testbin
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -183,9 +199,6 @@ jobs:
fi
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
- name: Time report
run: cat ${{ github.workspace }}/time.log
- name: Coverage report
run: |
go tool covdata percent -i=cover

6
.gitignore vendored
View File

@@ -62,8 +62,4 @@ tests/!s3cfg.local.default
*.patch
# grafana's local database (kept on filesystem for survival between instantiations)
metrics-exploration/grafana_data/**
# bats tools
/tests/bats-assert
/tests/bats-support
metrics-exploration/grafana_data/**

View File

@@ -18,10 +18,6 @@ GOBUILD=$(GOCMD) build
GOCLEAN=$(GOCMD) clean
GOTEST=$(GOCMD) test
# docker-compose
DCCMD=docker-compose
DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
BIN=versitygw
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -75,19 +71,19 @@ dist:
# Creates and runs S3 gateway instance in a docker container
.PHONY: up-posix
up-posix:
$(DOCKERCOMPOSE) up posix
docker compose --env-file .env.dev up posix
# Creates and runs S3 gateway proxy instance in a docker container
.PHONY: up-proxy
up-proxy:
$(DOCKERCOMPOSE) up proxy
docker compose --env-file .env.dev up proxy
# Creates and runs S3 gateway to azurite instance in a docker container
.PHONY: up-azurite
up-azurite:
$(DOCKERCOMPOSE) up azurite azuritegw
docker compose --env-file .env.dev up azurite azuritegw
# Creates and runs both S3 gateway and proxy server instances in docker containers
.PHONY: up-app
up-app:
$(DOCKERCOMPOSE) up
docker compose --env-file .env.dev up

View File

@@ -45,18 +45,18 @@ type GetBucketAclOutput struct {
type PutBucketAclInput struct {
Bucket *string
ACL types.BucketCannedACL
AccessControlPolicy *AccessControlPolicy
GrantFullControl *string
GrantRead *string
GrantReadACP *string
GrantWrite *string
GrantWriteACP *string
ACL types.BucketCannedACL
}
type AccessControlPolicy struct {
Owner *types.Owner
AccessControlList AccessControlList `xml:"AccessControlList"`
Owner *types.Owner
}
type AccessControlList struct {
@@ -352,13 +352,13 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
}
type AccessOptions struct {
Acl ACL
AclPermission types.Permission
IsRoot bool
Acc Account
Bucket string
Object string
Action Action
Acl ACL
Acc Account
IsRoot bool
Readonly bool
}

View File

@@ -63,10 +63,10 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri
}
type BucketPolicyItem struct {
Effect BucketPolicyAccessType `json:"Effect"`
Principals Principals `json:"Principal"`
Actions Actions `json:"Action"`
Resources Resources `json:"Resource"`
Effect BucketPolicyAccessType `json:"Effect"`
}
func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {

View File

@@ -28,19 +28,6 @@ const (
RoleUserPlus Role = "userplus"
)
func (r Role) IsValid() bool {
switch r {
case RoleAdmin:
return true
case RoleUser:
return true
case RoleUserPlus:
return true
default:
return false
}
}
// Account is a gateway IAM account
type Account struct {
Access string `json:"access"`
@@ -50,10 +37,6 @@ type Account struct {
GroupID int `json:"groupID"`
}
type ListUserAccountsResult struct {
Accounts []Account
}
// Mutable props, which could be changed when updating an IAM account
type MutableProps struct {
Secret *string `json:"secret"`
@@ -93,6 +76,7 @@ var (
)
type Opts struct {
RootAccount Account
Dir string
LDAPServerURL string
LDAPBindDN string
@@ -118,12 +102,11 @@ type Opts struct {
S3Region string
S3Bucket string
S3Endpoint string
RootAccount Account
CacheTTL int
CachePrune int
S3DisableSSlVerfiy bool
S3Debug bool
CacheDisable bool
CacheTTL int
CachePrune int
}
func New(o *Opts) (IAMService, error) {

View File

@@ -36,14 +36,14 @@ type IAMCache struct {
var _ IAMService = &IAMCache{}
type item struct {
exp time.Time
value Account
exp time.Time
}
type icache struct {
items map[string]item
expire time.Duration
sync.RWMutex
expire time.Duration
items map[string]item
}
func (i *icache) set(k string, v Account) {

View File

@@ -33,8 +33,6 @@ const (
// IAMServiceInternal manages the internal IAM service
type IAMServiceInternal struct {
dir string
rootAcc Account
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
@@ -42,6 +40,8 @@ type IAMServiceInternal struct {
// IAM service. All account updates should be sent to a single
// gateway instance if possible.
sync.RWMutex
dir string
rootAcc Account
}
// UpdateAcctFunc accepts the current data and returns the new data to be stored

View File

@@ -42,14 +42,6 @@ import (
// coming from iAMConfig and iamFile in iam_internal.
type IAMServiceS3 struct {
client *s3.Client
access string
secret string
region string
bucket string
endpoint string
rootAcc Account
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
@@ -58,8 +50,15 @@ type IAMServiceS3 struct {
// gateway instance if possible.
sync.RWMutex
access string
secret string
region string
bucket string
endpoint string
sslSkipVerify bool
debug bool
rootAcc Account
client *s3.Client
}
var _ IAMService = &IAMServiceS3{}

View File

@@ -25,13 +25,12 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
type BucketLockConfig struct {
Enabled bool
DefaultRetention *types.DefaultRetention
CreatedAt *time.Time
Enabled bool
}
func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
@@ -93,12 +92,12 @@ func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfigur
}
func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
var retention s3response.PutObjectRetentionInput
var retention types.ObjectLockRetention
if err := xml.Unmarshal(input, &retention); err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
if retention.RetainUntilDate.Before(time.Now()) {
if retention.RetainUntilDate == nil || retention.RetainUntilDate.Before(time.Now()) {
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
}
switch retention.Mode {
@@ -136,7 +135,7 @@ func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
}
}
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, bypass bool, be backend.Backend) error {
data, err := be.GetObjectLockConfiguration(ctx, bucket)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -172,15 +171,8 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
}
for _, obj := range objects {
var key, versionId string
if obj.Key != nil {
key = *obj.Key
}
if obj.VersionId != nil {
versionId = *obj.VersionId
}
checkRetention := true
retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
continue
}
@@ -211,7 +203,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
@@ -225,11 +217,8 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
checkLegalHold := true
status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
checkLegalHold = false
} else {
@@ -254,7 +243,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}

View File

@@ -85,10 +85,6 @@ type keyDerivator interface {
// SignerOptions is the SigV4 Signer options.
type SignerOptions struct {
// The logger to send log messages to.
Logger logging.Logger
// Disables the Signer's moving HTTP header key/value pairs from the HTTP
// request header to the request's query string. This is most commonly used
// with pre-signed requests preventing headers from being added to the
@@ -104,6 +100,9 @@ type SignerOptions struct {
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
DisableURIPathEscaping bool
// The logger to send log messages to.
Logger logging.Logger
// Enable logging of signed requests.
// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
// presigned URL.
@@ -118,8 +117,8 @@ type SignerOptions struct {
// Signer applies AWS v4 signing to given request. Use this to sign requests
// that need to be signed with AWS V4 Signatures.
type Signer struct {
keyDerivator keyDerivator
options SignerOptions
keyDerivator keyDerivator
}
// NewSigner returns a new SigV4 Signer
@@ -134,19 +133,17 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
}
type httpSigner struct {
KeyDerivator keyDerivator
Request *http.Request
Credentials aws.Credentials
Time v4Internal.SigningTime
ServiceName string
Region string
Time v4Internal.SigningTime
Credentials aws.Credentials
KeyDerivator keyDerivator
IsPreSign bool
SignedHdrs []string
PayloadHash string
SignedHdrs []string
IsPreSign bool
DisableHeaderHoisting bool
DisableURIPathEscaping bool
DisableSessionToken bool

View File

@@ -149,8 +149,8 @@ func (az *Azure) String() string {
func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
meta := map[string]*string{
string(keyAclCapital): backend.GetPtrFromString(encodeBytes(acl)),
string(keyOwnership): backend.GetPtrFromString(encodeBytes([]byte(input.ObjectOwnership))),
string(keyAclCapital): backend.GetStringPtr(encodeBytes(acl)),
string(keyOwnership): backend.GetStringPtr(encodeBytes([]byte(input.ObjectOwnership))),
}
acct, ok := ctx.Value("account").(auth.Account)
@@ -170,7 +170,7 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
return fmt.Errorf("parse default bucket lock state: %w", err)
}
meta[string(keyBucketLock)] = backend.GetPtrFromString(encodeBytes(defaultLockParsed))
meta[string(keyBucketLock)] = backend.GetStringPtr(encodeBytes(defaultLockParsed))
}
_, err := az.client.CreateContainer(ctx, *input.Bucket, &container.CreateOptions{Metadata: meta})
@@ -183,68 +183,58 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
var acl auth.ACL
if len(aclBytes) > 0 {
if err := json.Unmarshal(aclBytes, &acl); err != nil {
return fmt.Errorf("unmarshal acl: %w", err)
return fmt.Errorf("unmarshal bucket acl: %w", err)
}
}
if acl.Owner == acct.Access {
return s3err.GetAPIError(s3err.ErrBucketAlreadyOwnedByYou)
}
return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
}
return azureErrToS3Err(err)
}
func (az *Azure) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
fmt.Printf("%+v\n", input)
func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
pager := az.client.NewListContainersPager(
&service.ListContainersOptions{
Include: service.ListContainersInclude{
Metadata: true,
},
Marker: &input.ContinuationToken,
MaxResults: &input.MaxBuckets,
Prefix: &input.Prefix,
})
var buckets []s3response.ListAllMyBucketsEntry
result := s3response.ListAllMyBucketsResult{
Prefix: input.Prefix,
}
var result s3response.ListAllMyBucketsResult
resp, err := pager.NextPage(ctx)
if err != nil {
return result, azureErrToS3Err(err)
}
for _, v := range resp.ContainerItems {
if input.IsAdmin {
buckets = append(buckets, s3response.ListAllMyBucketsEntry{
Name: *v.Name,
// TODO: using modification date here instead of creation, is that ok?
CreationDate: *v.Properties.LastModified,
})
} else {
acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
if err != nil {
return result, err
}
if acl.Owner == input.Owner {
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return result, azureErrToS3Err(err)
}
for _, v := range resp.ContainerItems {
if isAdmin {
buckets = append(buckets, s3response.ListAllMyBucketsEntry{
Name: *v.Name,
// TODO: using modification date here instead of creation, is that ok?
CreationDate: *v.Properties.LastModified,
})
} else {
acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
if err != nil {
return result, err
}
if acl.Owner == owner {
buckets = append(buckets, s3response.ListAllMyBucketsEntry{
Name: *v.Name,
// TODO: using modification date here instead of creation, is that ok?
CreationDate: *v.Properties.LastModified,
})
}
}
}
}
if resp.NextMarker != nil {
result.ContinuationToken = *resp.NextMarker
}
result.Buckets.Bucket = buckets
result.Owner.ID = input.Owner
result.Owner.ID = owner
return result, nil
}
@@ -258,8 +248,8 @@ func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3
return &s3.HeadBucketOutput{}, nil
}
func (az *Azure) DeleteBucket(ctx context.Context, bucket string) error {
pager := az.client.NewListBlobsFlatPager(bucket, nil)
func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
pager := az.client.NewListBlobsFlatPager(*input.Bucket, nil)
pg, err := pager.NextPage(ctx)
if err != nil {
@@ -269,7 +259,7 @@ func (az *Azure) DeleteBucket(ctx context.Context, bucket string) error {
if len(pg.Segment.BlobItems) > 0 {
return s3err.GetAPIError(s3err.ErrBucketNotEmpty)
}
_, err = az.client.DeleteContainer(ctx, bucket, nil)
_, err = az.client.DeleteContainer(ctx, *input.Bucket, nil)
return azureErrToS3Err(err)
}
@@ -311,13 +301,13 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respon
opts.HTTPHeaders.BlobContentDisposition = po.ContentDisposition
if strings.HasSuffix(*po.Key, "/") {
// Hardcode "application/x-directory" for direcoty objects
opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DirContentType)
opts.HTTPHeaders.BlobContentType = backend.GetStringPtr(backend.DirContentType)
} else {
opts.HTTPHeaders.BlobContentType = po.ContentType
}
if opts.HTTPHeaders.BlobContentType == nil {
opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DefaultContentType)
opts.HTTPHeaders.BlobContentType = backend.GetStringPtr(backend.DefaultContentType)
}
uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts)
@@ -416,7 +406,7 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G
contentType := blobDownloadResponse.ContentType
if contentType == nil {
contentType = backend.GetPtrFromString(backend.DefaultContentType)
contentType = backend.GetStringPtr(backend.DefaultContentType)
}
return &s3.GetObjectOutput{
@@ -512,22 +502,20 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
return result, nil
}
func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
data, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: input.Bucket,
Key: input.Key,
})
if err != nil {
return s3response.GetObjectAttributesResponse{}, err
return s3response.GetObjectAttributesResult{}, err
}
return s3response.GetObjectAttributesResponse{
return s3response.GetObjectAttributesResult{
ETag: data.ETag,
LastModified: data.LastModified,
ObjectSize: data.ContentLength,
StorageClass: data.StorageClass,
LastModified: data.LastModified,
VersionId: data.VersionId,
DeleteMarker: data.DeleteMarker,
}, nil
}
@@ -581,28 +569,23 @@ Pager:
isTruncated = true
break Pager
}
marker := getString(input.Marker)
pfx := strings.TrimSuffix(*v.Name, getString(input.Delimiter))
if marker != "" && strings.HasPrefix(marker, pfx) {
continue
}
cPrefixes = append(cPrefixes, types.CommonPrefix{
Prefix: v.Name,
})
}
}
// TODO: generate common prefixes when appropriate
return s3response.ListObjectsResult{
Contents: objects,
Marker: backend.GetPtrFromString(*input.Marker),
Marker: input.Marker,
MaxKeys: input.MaxKeys,
Name: input.Bucket,
NextMarker: nextMarker,
Prefix: backend.GetPtrFromString(*input.Prefix),
Prefix: input.Prefix,
IsTruncated: &isTruncated,
Delimiter: backend.GetPtrFromString(*input.Delimiter),
Delimiter: input.Delimiter,
CommonPrefixes: cPrefixes,
}, nil
}
@@ -663,13 +646,6 @@ Pager:
isTruncated = true
break Pager
}
marker := getString(input.ContinuationToken)
pfx := strings.TrimSuffix(*v.Name, getString(input.Delimiter))
if marker != "" && strings.HasPrefix(marker, pfx) {
continue
}
cPrefixes = append(cPrefixes, types.CommonPrefix{
Prefix: v.Name,
})
@@ -678,15 +654,14 @@ Pager:
return s3response.ListObjectsV2Result{
Contents: objects,
ContinuationToken: backend.GetPtrFromString(*input.ContinuationToken),
ContinuationToken: input.ContinuationToken,
MaxKeys: input.MaxKeys,
Name: input.Bucket,
NextContinuationToken: nextMarker,
Prefix: backend.GetPtrFromString(*input.Prefix),
Prefix: input.Prefix,
IsTruncated: &isTruncated,
Delimiter: backend.GetPtrFromString(*input.Delimiter),
Delimiter: input.Delimiter,
CommonPrefixes: cPrefixes,
StartAfter: backend.GetPtrFromString(*input.StartAfter),
}, nil
}
@@ -722,8 +697,8 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
} else {
errs = append(errs, types.Error{
Key: obj.Key,
Code: backend.GetPtrFromString("InternalError"),
Message: backend.GetPtrFromString(err.Error()),
Code: backend.GetStringPtr("InternalError"),
Message: backend.GetStringPtr(err.Error()),
})
}
}
@@ -859,7 +834,7 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
// set blob legal hold status in metadata
if input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
meta[string(keyObjLegalHold)] = backend.GetPtrFromString("1")
meta[string(keyObjLegalHold)] = backend.GetStringPtr("1")
}
// set blob retention date
@@ -872,7 +847,7 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
if err != nil {
return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err)
}
meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retParsed))
meta[string(keyObjRetention)] = backend.GetStringPtr(string(retParsed))
}
uploadId := uuid.New().String()
@@ -1046,8 +1021,7 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa
prefix := string(metaTmpMultipartPrefix)
pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Include: container.ListBlobsInclude{Metadata: true},
Prefix: &prefix,
Prefix: &prefix,
})
for pager.More() {
@@ -1311,9 +1285,22 @@ func (az *Azure) GetObjectLockConfiguration(ctx context.Context, bucket string)
}
func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
err := az.isBucketObjectLockEnabled(ctx, bucket)
cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
if err != nil {
return err
return azureErrToS3Err(err)
}
if len(cfg) == 0 {
return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
}
var bucketLockConfig auth.BucketLockConfig
if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
return fmt.Errorf("parse bucket lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
}
blobClient, err := az.getBlobClient(bucket, object)
@@ -1329,12 +1316,12 @@ func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, version
meta := blobProps.Metadata
if meta == nil {
meta = map[string]*string{
string(keyObjRetention): backend.GetPtrFromString(string(retention)),
string(keyObjRetention): backend.GetStringPtr(string(retention)),
}
} else {
objLockCfg, ok := meta[string(keyObjRetention)]
if !ok {
meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retention))
meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
} else {
var lockCfg types.ObjectLockRetention
if err := json.Unmarshal([]byte(*objLockCfg), &lockCfg); err != nil {
@@ -1352,7 +1339,7 @@ func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, version
}
}
meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retention))
meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
}
}
@@ -1374,11 +1361,6 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
return nil, azureErrToS3Err(err)
}
err = az.isBucketObjectLockEnabled(ctx, bucket)
if err != nil {
return nil, err
}
retentionPtr, ok := props.Metadata[string(keyObjRetention)]
if !ok {
return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1388,9 +1370,22 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
}
func (az *Azure) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
err := az.isBucketObjectLockEnabled(ctx, bucket)
cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
if err != nil {
return err
return azureErrToS3Err(err)
}
if len(cfg) == 0 {
return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
}
var bucketLockConfig auth.BucketLockConfig
if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
return fmt.Errorf("parse bucket lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
}
blobClient, err := az.getBlobClient(bucket, object)
@@ -1437,11 +1432,6 @@ func (az *Azure) GetObjectLegalHold(ctx context.Context, bucket, object, version
return nil, azureErrToS3Err(err)
}
err = az.isBucketObjectLockEnabled(ctx, bucket)
if err != nil {
return nil, err
}
retentionPtr, ok := props.Metadata[string(keyObjLegalHold)]
if !ok {
return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1481,28 +1471,6 @@ func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response
return buckets, nil
}
func (az *Azure) isBucketObjectLockEnabled(ctx context.Context, bucket string) error {
cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
if err != nil {
return azureErrToS3Err(err)
}
if len(cfg) == 0 {
return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
}
var bucketLockConfig auth.BucketLockConfig
if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
return fmt.Errorf("parse bucket lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
}
return nil
}
func (az *Azure) getContainerURL(cntr string) string {
return fmt.Sprintf("%v/%v", strings.TrimRight(az.serviceURL, "/"), cntr)
}
@@ -1700,7 +1668,7 @@ func (az *Azure) setContainerMetaData(ctx context.Context, bucket, key string, v
}
str := encodeBytes(value)
mdmap[key] = backend.GetPtrFromString(str)
mdmap[key] = backend.GetStringPtr(str)
_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: mdmap})
if err != nil {

View File

@@ -32,14 +32,14 @@ type Backend interface {
Shutdown()
// bucket operations
ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(_ context.Context, bucket string) error
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
DeleteBucketPolicy(_ context.Context, bucket string) error
@@ -61,7 +61,7 @@ type Backend interface {
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
@@ -108,7 +108,7 @@ func (BackendUnsupported) Shutdown() {}
func (BackendUnsupported) String() string {
return "Unsupported"
}
func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -123,14 +123,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -185,8 +185,8 @@ func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.Ge
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)

View File

@@ -50,18 +50,8 @@ func (d ByObjectName) Len() int { return len(d) }
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
func GetPtrFromString(str string) *string {
if str == "" {
return nil
}
return &str
}
func GetStringFromPtr(str *string) string {
if str == nil {
return ""
}
return *str
func GetStringPtr(s string) *string {
return &s
}
func GetTimePtr(t time.Time) *time.Time {
@@ -105,7 +95,7 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
return 0, 0, errInvalidRange
}
if endOffset < startOffset {
if endOffset <= startOffset {
return 0, 0, errInvalidRange
}
@@ -119,13 +109,15 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
copySourceHeader = copySourceHeader[1:]
}
var copySource, versionId string
i := strings.LastIndex(copySourceHeader, "?versionId=")
if i == -1 {
copySource = copySourceHeader
} else {
copySource = copySourceHeader[:i]
versionId = copySourceHeader[i+11:]
cSplitted := strings.Split(copySourceHeader, "?")
copySource := cSplitted[0]
var versionId string
if len(cSplitted) > 1 {
versionIdParts := strings.Split(cSplitted[1], "=")
if len(versionIdParts) != 2 || versionIdParts[0] != "versionId" {
return "", "", "", s3err.GetAPIError(s3err.ErrInvalidRequest)
}
versionId = versionIdParts[1]
}
srcBucket, srcObject, ok := strings.Cut(copySource, "/")

View File

@@ -14,19 +14,17 @@
package meta
import "os"
// MetadataStorer defines the interface for managing metadata.
// When object == "", the operation is on the bucket.
type MetadataStorer interface {
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
// Returns the value of the attribute, or an error if the attribute does not exist.
RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
// If attribute already exists, new attribute should replace existing.
// Returns an error if the operation fails.
StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
StoreAttribute(bucket, object, attribute string, value []byte) error
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
// Returns an error if the operation fails.

View File

@@ -17,7 +17,6 @@ package meta
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
@@ -37,15 +36,7 @@ var (
type XattrMeta struct{}
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
if f != nil {
b, err := xattr.FGet(f, xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
}
return b, err
}
func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
@@ -54,11 +45,7 @@ func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute strin
}
// StoreAttribute stores the value of a specific attribute for an object in a bucket.
func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
if f != nil {
return xattr.FSet(f, xattrPrefix+attribute, value)
}
func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
}

View File

@@ -15,6 +15,11 @@ import (
"github.com/versity/versitygw/s3err"
)
var (
// TODO: make this configurable
defaultDirPerm fs.FileMode = 0755
)
// MkdirAll is similar to os.MkdirAll but it will return
// ErrObjectParentIsFile when appropriate
// MkdirAll creates a directory named path,
@@ -27,7 +32,7 @@ import (
// and returns nil.
// Any directory created will be set to provided uid/gid ownership
// if doChown is true.
func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
func MkdirAll(path string, uid, gid int, doChown bool) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
@@ -50,14 +55,14 @@ func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) erro
if j > 1 {
// Create parent.
err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
err = MkdirAll(path[:j-1], uid, gid, doChown)
if err != nil {
return err
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, dirPerm)
err = os.Mkdir(path, defaultDirPerm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.

File diff suppressed because it is too large Load Diff

View File

@@ -38,12 +38,11 @@ type tmpfile struct {
f *os.File
bucket string
objname string
isOTmp bool
size int64
needsChown bool
uid int
gid int
newDirPerm fs.FileMode
isOTmp bool
needsChown bool
}
var (
@@ -63,7 +62,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
if err != nil {
// O_TMPFILE not supported, try fallback
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
err = backend.MkdirAll(dir, uid, gid, doChown)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
@@ -109,7 +108,6 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
needsChown: doChown,
uid: uid,
gid: gid,
newDirPerm: p.newDirPerm,
}
// falloc is best effort, its fine if this fails
@@ -136,9 +134,6 @@ func (tmp *tmpfile) falloc() error {
}
func (tmp *tmpfile) link() error {
// make sure this is cleaned up in all error cases
defer tmp.f.Close()
// We use Linkat/Rename as the atomic operation for object puts. The
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
// with any other simultaneous uploads. The final operation is to move the
@@ -153,7 +148,7 @@ func (tmp *tmpfile) link() error {
dir := filepath.Dir(objPath)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
if err != nil {
return fmt.Errorf("make parent dir: %w", err)
}
@@ -175,21 +170,11 @@ func (tmp *tmpfile) link() error {
}
defer dirf.Close()
for {
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if errors.Is(err, syscall.EEXIST) {
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
continue
}
if err != nil {
return fmt.Errorf("link tmpfile (fd %q as %q): %w",
filepath.Base(tmp.f.Name()), objPath, err)
}
break
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if err != nil {
return fmt.Errorf("link tmpfile (%q in %q): %w",
filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
}
err = tmp.f.Close()

View File

@@ -41,7 +41,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
// Create a temp file for upload while in progress (see link comments below).
var err error
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
err = backend.MkdirAll(dir, uid, gid, doChown)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}

View File

@@ -75,12 +75,8 @@ func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify
return s, nil
}
func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{
ContinuationToken: &input.ContinuationToken,
MaxBuckets: &input.MaxBuckets,
Prefix: &input.Prefix,
})
func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
return s3response.ListAllMyBucketsResult{}, handleError(err)
}
@@ -100,8 +96,6 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsI
Buckets: s3response.ListAllMyBucketsList{
Bucket: buckets,
},
ContinuationToken: backend.GetStringFromPtr(output.ContinuationToken),
Prefix: backend.GetStringFromPtr(output.Prefix),
}, nil
}
@@ -118,8 +112,8 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
var tagSet []types.Tag
tagSet = append(tagSet, types.Tag{
Key: backend.GetPtrFromString(aclKey),
Value: backend.GetPtrFromString(base64Encode(acl)),
Key: backend.GetStringPtr(aclKey),
Value: backend.GetStringPtr(base64Encode(acl)),
})
_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
@@ -131,10 +125,8 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
return handleError(err)
}
func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: &bucket,
})
func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
_, err := s.client.DeleteBucket(ctx, input)
return handleError(err)
}
@@ -180,15 +172,12 @@ func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status
return handleError(err)
}
func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
Bucket: &bucket,
})
return s3response.GetBucketVersioningOutput{
Status: &out.Status,
MFADelete: &out.MFADelete,
}, handleError(err)
return out, handleError(err)
}
func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
@@ -392,7 +381,7 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
return output, nil
}
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
out, err := s.client.GetObjectAttributes(ctx, input)
parts := s3response.ObjectParts{}
@@ -419,11 +408,12 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
}
}
return s3response.GetObjectAttributesResponse{
return s3response.GetObjectAttributesResult{
ETag: out.ETag,
LastModified: out.LastModified,
ObjectSize: out.ObjectSize,
StorageClass: out.StorageClass,
VersionId: out.VersionId,
ObjectParts: &parts,
}, handleError(err)
}
@@ -530,8 +520,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
for i, tag := range tagout.TagSet {
if *tag.Key == aclKey {
tagout.TagSet[i] = types.Tag{
Key: backend.GetPtrFromString(aclKey),
Value: backend.GetPtrFromString(base64Encode(data)),
Key: backend.GetStringPtr(aclKey),
Value: backend.GetStringPtr(base64Encode(data)),
}
found = true
break
@@ -539,8 +529,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
}
if !found {
tagout.TagSet = append(tagout.TagSet, types.Tag{
Key: backend.GetPtrFromString(aclKey),
Value: backend.GetPtrFromString(base64Encode(data)),
Key: backend.GetStringPtr(aclKey),
Value: backend.GetStringPtr(base64Encode(data)),
})
}
@@ -600,7 +590,7 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: &bucket,
Policy: backend.GetPtrFromString(string(policy)),
Policy: backend.GetStringPtr(string(policy)),
})
return handleError(err)
}

View File

@@ -44,25 +44,15 @@ type ScoutfsOpts struct {
ChownGID bool
GlacierMode bool
BucketLinks bool
NewDirPerm fs.FileMode
}
type ScoutFS struct {
// bucket/object metadata storage facility
meta meta.MetadataStorer
*posix.Posix
rootfd *os.File
rootdir string
// euid/egid are the effective uid/gid of the running versitygw process
// used to determine if chowning is needed
euid int
egid int
// newDirPerm is the permissions to use when creating new directories
newDirPerm fs.FileMode
// bucket/object metadata storage facility
meta meta.MetadataStorer
// glaciermode enables the following behavior:
// GET object: if file offline, return invalid object state
@@ -79,6 +69,11 @@ type ScoutFS struct {
// when objects are uploaded
chownuid bool
chowngid bool
// euid/egid are the effective uid/gid of the running versitygw process
// used to determine if chowning is needed
euid int
egid int
}
var _ backend.Backend = &ScoutFS{}
@@ -231,7 +226,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
b, err := s.meta.RetrieveAttribute(bucket, partObjPath, etagkey)
etag := string(b)
if err != nil {
etag = ""
@@ -267,7 +262,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
// scoutfs move data is a metadata only operation that moves the data
// extent references from the source, appeding to the destination.
// this needs to be 4k aligned.
err = moveData(pf, f.File())
err = moveData(pf, f.f)
pf.Close()
if err != nil {
return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
@@ -282,76 +277,83 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
dir := filepath.Dir(objname)
if dir != "" {
uid, gid, doChown := s.getChownIDs(acct)
err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
err = backend.MkdirAll(dir, uid, gid, doChown)
if err != nil {
return nil, err
}
}
err = f.link()
if err != nil {
return nil, fmt.Errorf("link object in namespace: %w", err)
}
for k, v := range userMetaData {
err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
err = s.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
if err != nil {
// cleanup object if returning error
os.Remove(objname)
return nil, fmt.Errorf("set user attr %q: %w", k, err)
}
}
// load and set tagging
tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object tagging: %w", err)
}
tagging, err := s.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
if err == nil {
err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
if err != nil {
if err := s.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object tagging: %w", err)
}
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object tagging: %w", err)
}
// set content-type
if cType != "" {
err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
if err != nil {
if err := s.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object content type: %w", err)
}
}
// load and set legal hold
lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object legal hold: %w", err)
}
lHold, err := s.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
if err == nil {
err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
if err != nil {
if err := s.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object legal hold: %w", err)
}
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object legal hold: %w", err)
}
// load and set retention
ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object retention: %w", err)
}
ret, err := s.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
if err == nil {
err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
if err != nil {
if err := s.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object retention: %w", err)
}
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object retention: %w", err)
}
// Calculate s3 compatible md5sum for complete multipart.
s3MD5 := backend.GetMultipartMD5(parts)
err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
err = s.meta.StoreAttribute(bucket, object, etagkey, []byte(s3MD5))
if err != nil {
// cleanup object if returning error
os.Remove(objname)
return nil, fmt.Errorf("set etag attr: %w", err)
}
err = f.link()
if err != nil {
return nil, fmt.Errorf("link object in namespace: %w", err)
}
// cleanup tmp dirs
os.RemoveAll(upiddir)
// use Remove for objdir in case there are still other uploads
@@ -390,7 +392,7 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
if !isValidMeta(e) {
continue
}
b, err := s.meta.RetrieveAttribute(nil, bucket, object, e)
b, err := s.meta.RetrieveAttribute(bucket, object, e)
if err != nil {
continue
}
@@ -402,13 +404,13 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
}
var contentType, contentEncoding string
b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
b, _ := s.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
contentType = string(b)
if contentType != "" {
m[contentTypeHdr] = contentType
}
b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
b, _ = s.meta.RetrieveAttribute(bucket, object, contentEncHdr)
contentEncoding = string(b)
if contentEncoding != "" {
m[contentEncHdr] = contentEncoding
@@ -464,7 +466,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
return nil, fmt.Errorf("stat part: %w", err)
}
b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
b, err := s.meta.RetrieveAttribute(bucket, partPath, etagkey)
etag := string(b)
if err != nil {
etag = ""
@@ -512,7 +514,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
contentType = "application/x-directory"
}
b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
etag := string(b)
if err != nil {
etag = ""
@@ -552,7 +554,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
contentLength := fi.Size()
var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, "")
if err == nil {
if *status {
objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
@@ -563,7 +565,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
var objectLockMode types.ObjectLockMode
var objectLockRetainUntilDate *time.Time
retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, "")
if err == nil {
var config types.ObjectLockRetention
if err := json.Unmarshal(retention, &config); err == nil {
@@ -683,7 +685,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
etag := string(b)
if err != nil {
etag = ""
@@ -838,7 +840,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
if d.IsDir() {
// directory object only happens if directory empty
// check to see if this is a directory object by checking etag
etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
etagBytes, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
}
@@ -867,7 +869,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
}
// file object, get object info and fill out object data
b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
if errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
}

View File

@@ -40,7 +40,6 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
ChownUID: opts.ChownUID,
ChownGID: opts.ChownGID,
BucketLinks: opts.BucketLinks,
NewDirPerm: opts.NewDirPerm,
})
if err != nil {
return nil, err
@@ -59,7 +58,6 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
chownuid: opts.ChownUID,
chowngid: opts.ChownGID,
glaciermode: opts.GlacierMode,
newDirPerm: opts.NewDirPerm,
}, nil
}
@@ -70,13 +68,13 @@ type tmpfile struct {
bucket string
objname string
size int64
needsChown bool
uid int
gid int
newDirPerm fs.FileMode
needsChown bool
}
var (
// TODO: make this configurable
defaultFilePerm uint32 = 0644
)
@@ -104,7 +102,6 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
needsChown: doChown,
uid: uid,
gid: gid,
newDirPerm: s.newDirPerm,
}
if doChown {
@@ -132,7 +129,7 @@ func (tmp *tmpfile) link() error {
dir := filepath.Dir(objPath)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
if err != nil {
return fmt.Errorf("make parent dir: %w", err)
}
@@ -177,10 +174,6 @@ func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}
func moveData(from *os.File, to *os.File) error {
return scoutfs.MoveData(from, to)
}

View File

@@ -28,7 +28,9 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
return nil, fmt.Errorf("scoutfs only available on linux")
}
type tmpfile struct{}
type tmpfile struct {
f *os.File
}
var (
errNotSupported = errors.New("not supported")
@@ -54,10 +56,6 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
func (tmp *tmpfile) cleanup() {
}
func (tmp *tmpfile) File() *os.File {
return nil
}
func moveData(_, _ *os.File) error {
return errNotSupported
}

View File

@@ -19,6 +19,7 @@ import (
"errors"
"fmt"
"io/fs"
"os"
"sort"
"strings"
@@ -27,10 +28,10 @@ import (
)
type WalkResults struct {
NextMarker string
CommonPrefixes []types.CommonPrefix
Objects []s3response.Object
Truncated bool
NextMarker string
}
type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
@@ -52,15 +53,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
var newMarker string
var truncated bool
root := "."
if strings.Contains(prefix, "/") {
idx := strings.LastIndex(prefix, "/")
if idx > 0 {
root = prefix[:idx]
}
}
err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
@@ -83,9 +76,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
return fs.SkipAll
}
// After this point, return skipflag instead of nil
// so we can skip a directory without an early return
var skipflag error
if d.IsDir() {
// If prefix is defined and the directory does not match prefix,
// do not descend into the directory because nothing will
@@ -95,57 +85,51 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
// building to match. So only skip if path isn't a prefix of prefix
// and prefix isn't a prefix of path.
if prefix != "" &&
!strings.HasPrefix(path+"/", prefix) &&
!strings.HasPrefix(prefix, path+"/") {
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
return fs.SkipDir
}
// Don't recurse into subdirectories which contain the delimiter
// after reaching the prefix
if delimiter != "" &&
strings.HasPrefix(path+"/", prefix) &&
strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
skipflag = fs.SkipDir
} else {
// TODO: can we do better here rather than a second readdir
// per directory?
ents, err := fs.ReadDir(fileSystem, path)
if err != nil {
return fmt.Errorf("readdir %q: %w", path, err)
}
if len(ents) == 0 && delimiter == "" {
dirobj, err := getObj(path+"/", d)
if err == ErrSkipObj {
return skipflag
}
if err != nil {
return fmt.Errorf("directory to object %q: %w", path, err)
}
objects = append(objects, dirobj)
return skipflag
}
if len(ents) != 0 {
return skipflag
}
// TODO: can we do better here rather than a second readdir
// per directory?
ents, err := fs.ReadDir(fileSystem, path)
if err != nil {
return fmt.Errorf("readdir %q: %w", path, err)
}
path += string(os.PathSeparator)
if len(ents) == 0 && delimiter == "" {
dirobj, err := getObj(path, d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("directory to object %q: %w", path, err)
}
objects = append(objects, dirobj)
return nil
}
if len(ents) != 0 {
return nil
}
path += "/"
}
if !pastMarker {
if path == marker {
pastMarker = true
return skipflag
return nil
}
if path < marker {
return skipflag
return nil
}
}
// If object doesn't have prefix, don't include in results.
if prefix != "" && !strings.HasPrefix(path, prefix) {
return skipflag
return nil
}
if delimiter == "" {
@@ -153,7 +137,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
// prefix are included in results
obj, err := getObj(path, d)
if err == ErrSkipObj {
return skipflag
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
@@ -164,7 +148,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
pastMax = true
}
return skipflag
return nil
}
// Since delimiter is specified, we only want results that
@@ -193,7 +177,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
if !found {
obj, err := getObj(path, d)
if err == ErrSkipObj {
return skipflag
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
@@ -202,7 +186,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
if (len(objects) + len(cpmap)) == int(max) {
pastMax = true
}
return skipflag
return nil
}
// Common prefixes are a set, so should not have duplicates.
@@ -212,12 +196,12 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
cpref := prefix + before + delimiter
if cpref == marker {
pastMarker = true
return skipflag
return nil
}
if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
// skip common prefixes that are before the marker
return skipflag
return nil
}
cpmap[cpref] = struct{}{}
@@ -227,13 +211,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
return fs.SkipAll
}
return skipflag
return nil
})
if err != nil {
// suppress file not found caused by user's prefix
if errors.Is(err, fs.ErrNotExist) {
return WalkResults{}, nil
}
return WalkResults{}, err
}
@@ -268,18 +248,18 @@ func contains(a string, strs []string) bool {
}
type WalkVersioningResults struct {
NextMarker string
NextVersionIdMarker string
CommonPrefixes []types.CommonPrefix
ObjectVersions []types.ObjectVersion
DelMarkers []types.DeleteMarkerEntry
Truncated bool
NextMarker string
NextVersionIdMarker string
}
type ObjVersionFuncResult struct {
NextVersionIdMarker string
ObjectVersions []types.ObjectVersion
DelMarkers []types.DeleteMarkerEntry
NextVersionIdMarker string
Truncated bool
}
@@ -335,16 +315,8 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
// building to match. So only skip if path isn't a prefix of prefix
// and prefix isn't a prefix of path.
if prefix != "" &&
!strings.HasPrefix(path+"/", prefix) &&
!strings.HasPrefix(prefix, path+"/") {
return fs.SkipDir
}
// Don't recurse into subdirectories when listing with delimiter.
if delimiter == "/" &&
prefix != path+"/" &&
strings.HasPrefix(path+"/", prefix) {
cpmap[path+"/"] = struct{}{}
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
return fs.SkipDir
}

View File

@@ -31,18 +31,9 @@ import (
)
type walkTest struct {
fsys fs.FS
getobj backend.GetObjFunc
cases []testcase
}
type testcase struct {
name string
prefix string
delimiter string
marker string
expected backend.WalkResults
maxObjs int32
fsys fs.FS
expected backend.WalkResults
getobj backend.GetObjFunc
}
func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
@@ -97,154 +88,50 @@ func TestWalk(t *testing.T) {
"photos/2006/February/sample3.jpg": {},
"photos/2006/February/sample4.jpg": {},
},
getobj: getObj,
cases: []testcase{
{
name: "aws example",
delimiter: "/",
maxObjs: 1000,
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photos/"),
}},
Objects: []s3response.Object{{
Key: backend.GetPtrFromString("sample.jpg"),
}},
},
},
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("photos/"),
}},
Objects: []s3response.Object{{
Key: backend.GetStringPtr("sample.jpg"),
}},
},
getobj: getObj,
},
{
// test case single dir/single file
fsys: fstest.MapFS{
"test/file": {},
},
getobj: getObj,
cases: []testcase{
{
name: "single dir single file",
delimiter: "/",
maxObjs: 1000,
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("test/"),
}},
Objects: []s3response.Object{},
},
},
},
},
{
// non-standard delimiter
fsys: fstest.MapFS{
"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("test/"),
}},
Objects: []s3response.Object{},
},
getobj: getObj,
cases: []testcase{
{
name: "different delimiter 1",
delimiter: "|",
maxObjs: 1000,
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|"),
}},
},
},
{
name: "different delimiter 2",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|"),
}},
},
},
{
name: "different delimiter 3",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
}},
},
},
{
name: "different delimiter 4",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
}},
},
},
{
name: "different delimiter 5",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|6/Januar|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
}},
},
},
{
name: "different delimiter 6",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|6/Januar|y/sampl|",
expected: backend.WalkResults{
Objects: []s3response.Object{
{
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
},
{
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
},
{
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
},
},
},
},
},
},
}
for _, tt := range tests {
for _, tc := range tt.cases {
res, err := backend.Walk(context.Background(),
tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
tt.getobj, []string{})
if err != nil {
t.Errorf("tc.name: walk: %v", err)
}
compareResults(tc.name, res, tc.expected, t)
res, err := backend.Walk(context.Background(), tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
if err != nil {
t.Fatalf("walk: %v", err)
}
compareResults(res, tt.expected, t)
}
}
func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
func compareResults(got, wanted backend.WalkResults, t *testing.T) {
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
t.Errorf("%v: unexpected common prefix, got %v wanted %v",
name,
t.Errorf("unexpected common prefix, got %v wanted %v",
printCommonPrefixes(got.CommonPrefixes),
printCommonPrefixes(wanted.CommonPrefixes))
}
if !compareObjects(got.Objects, wanted.Objects) {
t.Errorf("%v: unexpected object, got %v wanted %v",
name,
t.Errorf("unexpected object, got %v wanted %v",
printObjects(got.Objects),
printObjects(wanted.Objects))
}
@@ -315,16 +202,10 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
func printObjects(list []s3response.Object) string {
res := "["
for _, cp := range list {
var key string
if cp.Key == nil {
key = "<nil>"
} else {
key = *cp.Key
}
if res == "[" {
res = res + key
res = res + *cp.Key
} else {
res = res + ", " + key
res = res + ", " + *cp.Key
}
}
return res + "]"

View File

@@ -19,7 +19,7 @@ import (
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"encoding/xml"
"encoding/json"
"fmt"
"io"
"net/http"
@@ -29,7 +29,6 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3response"
@@ -38,7 +37,6 @@ import (
var (
adminAccess string
adminSecret string
adminRegion string
adminEndpoint string
allowInsecure bool
)
@@ -173,14 +171,6 @@ func adminCommand() *cli.Command {
Required: true,
Destination: &adminSecret,
},
&cli.StringFlag{
Name: "region",
Usage: "admin s3 region string",
EnvVars: []string{"ADMIN_REGION"},
Value: "us-east-1",
Destination: &adminRegion,
Aliases: []string{"r"},
},
&cli.StringFlag{
Name: "endpoint-url",
Usage: "admin apis endpoint url",
@@ -225,24 +215,24 @@ func createUser(ctx *cli.Context) error {
GroupID: groupID,
}
accxml, err := xml.Marshal(acc)
accJson, err := json.Marshal(acc)
if err != nil {
return fmt.Errorf("failed to parse user data: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256(accxml)
hashedPayload := sha256.Sum256(accJson)
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -261,9 +251,11 @@ func createUser(ctx *cli.Context) error {
}
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
fmt.Printf("%s\n", body)
return nil
}
@@ -285,7 +277,7 @@ func deleteUser(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -304,9 +296,11 @@ func deleteUser(ctx *cli.Context) error {
}
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
fmt.Printf("%s\n", body)
return nil
}
@@ -323,24 +317,24 @@ func updateUser(ctx *cli.Context) error {
props.GroupID = &groupId
}
propsxml, err := xml.Marshal(props)
propsJSON, err := json.Marshal(props)
if err != nil {
return fmt.Errorf("failed to parse user attributes: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256(propsxml)
hashedPayload := sha256.Sum256(propsJSON)
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -359,9 +353,11 @@ func updateUser(ctx *cli.Context) error {
}
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
fmt.Printf("%s\n", body)
return nil
}
@@ -378,7 +374,7 @@ func listUsers(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -397,15 +393,15 @@ func listUsers(ctx *cli.Context) error {
}
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
var accs auth.ListUserAccountsResult
if err := xml.Unmarshal(body, &accs); err != nil {
var accs []auth.Account
if err := json.Unmarshal(body, &accs); err != nil {
return err
}
printAcctTable(accs.Accounts)
printAcctTable(accs)
return nil
}
@@ -445,7 +441,7 @@ func changeBucketOwner(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -464,9 +460,11 @@ func changeBucketOwner(ctx *cli.Context) error {
}
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
fmt.Println(string(body))
return nil
}
@@ -495,7 +493,7 @@ func listBuckets(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
@@ -514,26 +512,15 @@ func listBuckets(ctx *cli.Context) error {
}
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
var result s3response.ListBucketsResult
if err := xml.Unmarshal(body, &result); err != nil {
var buckets []s3response.Bucket
if err := json.Unmarshal(body, &buckets); err != nil {
return err
}
printBuckets(result.Buckets)
printBuckets(buckets)
return nil
}
func parseApiError(body []byte) error {
var apiErr smithy.GenericAPIError
err := xml.Unmarshal(body, &apiErr)
if err != nil {
apiErr.Code = "InternalServerError"
apiErr.Message = err.Error()
}
return &apiErr
}

View File

@@ -7,7 +7,6 @@ import (
"path/filepath"
"sync"
"testing"
"time"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
@@ -58,9 +57,7 @@ func initPosix(ctx context.Context) {
log.Fatalf("make temp directory: %v", err)
}
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
NewDirPerm: 0755,
})
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
if err != nil {
log.Fatalf("init posix: %v", err)
}
@@ -78,9 +75,6 @@ func initPosix(ctx context.Context) {
}
wg.Done()
}()
// wait for server to start
time.Sleep(1 * time.Second)
}
func TestIntegration(t *testing.T) {

View File

@@ -16,8 +16,6 @@ package main
import (
"fmt"
"io/fs"
"math"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/meta"
@@ -28,7 +26,6 @@ var (
chownuid, chowngid bool
bucketlinks bool
versioningDir string
dirPerms uint
)
func posixCommand() *cli.Command {
@@ -71,14 +68,6 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
EnvVars: []string{"VGW_VERSIONING_DIR"},
Destination: &versioningDir,
},
&cli.UintFlag{
Name: "dir-perms",
Usage: "default directory permissions for new directories",
EnvVars: []string{"VGW_DIR_PERMS"},
Destination: &dirPerms,
DefaultText: "0755",
Value: 0755,
},
},
}
}
@@ -94,16 +83,11 @@ func runPosix(ctx *cli.Context) error {
return fmt.Errorf("posix xattr check: %v", err)
}
if dirPerms > math.MaxUint32 {
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
}
be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
BucketLinks: bucketlinks,
VersioningDir: versioningDir,
NewDirPerm: fs.FileMode(dirPerms),
})
if err != nil {
return fmt.Errorf("init posix: %v", err)

View File

@@ -16,8 +16,6 @@ package main
import (
"fmt"
"io/fs"
"math"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/scoutfs"
@@ -71,14 +69,6 @@ move interfaces as well as support for tiered filesystems.`,
EnvVars: []string{"VGW_BUCKET_LINKS"},
Destination: &bucketlinks,
},
&cli.UintFlag{
Name: "dir-perms",
Usage: "default directory permissions for new directories",
EnvVars: []string{"VGW_DIR_PERMS"},
Destination: &dirPerms,
DefaultText: "0755",
Value: 0755,
},
},
}
}
@@ -88,16 +78,11 @@ func runScoutfs(ctx *cli.Context) error {
return fmt.Errorf("no directory provided for operation")
}
if dirPerms > math.MaxUint32 {
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
}
var opts scoutfs.ScoutfsOpts
opts.GlacierMode = glacier
opts.ChownUID = chownuid
opts.ChownGID = chowngid
opts.BucketLinks = bucketlinks
opts.NewDirPerm = fs.FileMode(dirPerms)
be, err := scoutfs.New(ctx.Args().Get(0), opts)
if err != nil {

View File

@@ -37,7 +37,6 @@ var (
pathStyle bool
checksumDisable bool
versioningEnabled bool
azureTests bool
)
func testCommand() *cli.Command {
@@ -96,26 +95,12 @@ func initTestCommands() []*cli.Command {
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
&cli.BoolFlag{
Name: "azure-test-mode",
Usage: "Skips tests that are not supported by Azure",
Destination: &azureTests,
Aliases: []string{"azure"},
},
},
},
{
Name: "posix",
Usage: "Tests posix specific features",
Action: getAction(integration.TestPosix),
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "versioning-enabled",
Usage: "Test posix when versioning is enabled",
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
},
},
{
Name: "iam",
@@ -303,9 +288,6 @@ func getAction(tf testFunc) func(*cli.Context) error {
if versioningEnabled {
opts = append(opts, integration.WithVersioningEnabled())
}
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
s := integration.NewS3Conf(opts...)
tf(s)
@@ -337,22 +319,11 @@ func extractIntTests() (commands []*cli.Command) {
if debug {
opts = append(opts, integration.WithDebug())
}
if versioningEnabled {
opts = append(opts, integration.WithVersioningEnabled())
}
s := integration.NewS3Conf(opts...)
err := testFunc(s)
return err
},
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "versioning-enabled",
Usage: "Test the bucket object versioning, if the versioning is enabled",
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
},
})
}
return

View File

@@ -311,12 +311,6 @@ ROOT_SECRET_ACCESS_KEY=
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false
# The default permissions mode when creating new directories is 0755. Use
# VGW_DIR_PERMS option to set a different mode for any new directory that the
# gateway creates. This applies to buckets created through the gateway as well
# as any parent directories automatically created with object uploads.
#VGW_DIR_PERMS=0755
###########
# scoutfs #
###########
@@ -352,12 +346,6 @@ ROOT_SECRET_ACCESS_KEY=
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false
# The default permissions mode when creating new directories is 0755. Use
# VGW_DIR_PERMS option to set a different mode for any new directory that the
# gateway creates. This applies to buckets created through the gateway as well
# as any parent directories automatically created with object uploads.
#VGW_DIR_PERMS=0755
######
# s3 #
######

67
go.mod
View File

@@ -3,13 +3,13 @@ module github.com/versity/versitygw
go 1.21.0
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
github.com/DataDog/datadog-go/v5 v5.5.0
github.com/aws/aws-sdk-go-v2 v1.32.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
github.com/aws/smithy-go v1.22.0
github.com/aws/aws-sdk-go-v2 v1.30.5
github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2
github.com/aws/smithy-go v1.20.4
github.com/go-ldap/ldap/v3 v3.4.8
github.com/gofiber/fiber/v2 v2.52.5
github.com/google/go-cmp v0.6.0
@@ -19,24 +19,23 @@ require (
github.com/oklog/ulid/v2 v2.1.0
github.com/pkg/xattr v0.4.10
github.com/segmentio/kafka-go v0.4.47
github.com/smira/go-statsd v1.3.4
github.com/urfave/cli/v2 v2.27.5
github.com/valyala/fasthttp v1.57.0
github.com/smira/go-statsd v1.3.3
github.com/urfave/cli/v2 v2.27.4
github.com/valyala/fasthttp v1.55.0
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
golang.org/x/sync v0.8.0
golang.org/x/sys v0.26.0
golang.org/x/sys v0.25.0
)
require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -50,27 +49,27 @@ require (
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.6.0 // indirect
)
require (
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
github.com/aws/aws-sdk-go-v2/config v1.28.1
github.com/aws/aws-sdk-go-v2/credentials v1.17.42
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.33
github.com/aws/aws-sdk-go-v2/credentials v1.17.32
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect

148
go.sum
View File

@@ -1,21 +1,17 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -23,55 +19,51 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw=
github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI=
github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM=
github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35 h1:ihPPdcCVSN0IvBByXwqVp28/l4VosBZ6sDulcvU2J7w=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35/go.mod h1:JkgEhs3SVF51Dj3m1Bj+yL8IznpxzkwlA3jLg3x7Kls=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU=
github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks=
github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I=
github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18 h1:9DIp7vhmOPmueCDwpXa45bEbLHHTt1kcxChdTJWWxvI=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18/go.mod h1:aJv/Fwz8r56ozwYFRC4bzoeL1L17GYQYemfblOBux1M=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U=
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE=
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg=
github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o=
github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
@@ -117,11 +109,9 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -152,8 +142,6 @@ github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -164,8 +152,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
@@ -177,12 +165,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE=
github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -195,8 +183,6 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -206,8 +192,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -223,14 +209,12 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -250,8 +234,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -267,10 +251,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

View File

@@ -72,14 +72,6 @@ var (
ActionPutBucketOwnershipControls = "s3_PutBucketOwnershipControls"
ActionGetBucketOwnershipControls = "s3_GetBucketOwnershipControls"
ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
// Admin actions
ActionAdminCreateUser = "admin_CreateUser"
ActionAdminUpdateUser = "admin_UpdateUser"
ActionAdminDeleteUser = "admin_DeleteUser"
ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
ActionAdminListUsers = "admin_ListUsers"
ActionAdminListBuckets = "admin_ListBuckets"
)
func init() {

View File

@@ -43,14 +43,13 @@ type Tag struct {
// Manager is a manager of metrics plugins
type Manager struct {
wg sync.WaitGroup
ctx context.Context
addDataChan chan datapoint
config Config
publishers []publisher
wg sync.WaitGroup
publishers []publisher
addDataChan chan datapoint
}
type Config struct {
@@ -221,6 +220,6 @@ func (m *Manager) addForwarder(addChan <-chan datapoint) {
type datapoint struct {
key string
tags []Tag
value int64
tags []Tag
}

View File

@@ -5,19 +5,17 @@ rm -rf /tmp/gw
mkdir /tmp/gw
rm -rf /tmp/covdata
mkdir /tmp/covdata
rm -rf /tmp/versioing.covdata
mkdir /tmp/versioning.covdata
rm -rf /tmp/versioningdir
mkdir /tmp/versioningdir
# run server in background not versioning-enabled
# port: 7070(default)
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
# run server in background
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
GW_PID=$!
# wait a second for server to start up
sleep 1
# check if versioning-enabled gateway process is still running
# check if server is still running
if ! kill -0 $GW_PID; then
echo "server no longer running"
exit 1
@@ -25,7 +23,7 @@ fi
# run tests
# full flow tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow -vs; then
echo "full flow tests failed"
kill $GW_PID
exit 1
@@ -43,39 +41,8 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
exit 1
fi
kill $GW_PID
# run server in background versioning-enabled
# port: 7071
GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7071 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
GW_VS_PID=$!
# wait a second for server to start up
sleep 1
# check if versioning-enabled gateway process is still running
if ! kill -0 $GW_VS_PID; then
echo "versioning-enabled server no longer running"
exit 1
fi
# run tests
# full flow tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 full-flow -vs; then
echo "versioning-enabled full-flow tests failed"
kill $GW_VS_PID
exit 1
fi
# posix tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 posix -vs; then
echo "versiongin-enabled posix tests failed"
kill $GW_VS_PID
exit 1
fi
# kill off server
kill $GW_VS_PID
kill $GW_PID
exit 0
# if the above binary was built with -cover enabled (make testbin),

View File

@@ -26,11 +26,11 @@ import (
)
type S3AdminServer struct {
backend backend.Backend
app *fiber.App
backend backend.Backend
router *S3AdminRouter
cert *tls.Certificate
port string
cert *tls.Certificate
}
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
@@ -53,9 +53,6 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
app.Use(middlewares.VerifyMD5Body(l))
// Admin role checker
app.Use(middlewares.IsAdmin(l))
server.router.Init(app, be, iam, l)
return server

View File

@@ -16,19 +16,15 @@ package controllers
import (
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
"github.com/versity/versitygw/s3response"
)
type AdminController struct {
@@ -42,124 +38,187 @@ func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLo
}
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:CreateUser",
})
}
var usr auth.Account
err := xml.Unmarshal(ctx.Body(), &usr)
err := json.Unmarshal(ctx.Body(), &usr)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
return sendResponse(ctx, fmt.Errorf("failed to parse request body: %w", err), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusBadRequest,
action: "admin:CreateUser",
})
}
if !usr.Role.IsValid() {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
return sendResponse(ctx, errors.New("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusBadRequest,
action: "admin:CreateUser",
})
}
err = c.iam.CreateAccount(usr)
if err != nil {
status := fiber.StatusInternalServerError
err = fmt.Errorf("failed to create user: %w", err)
if strings.Contains(err.Error(), "user already exists") {
err = s3err.GetAPIError(s3err.ErrAdminUserExists)
status = fiber.StatusConflict
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
return sendResponse(ctx, err, nil,
&metaOptions{
status: status,
logger: c.l,
action: "admin:CreateUser",
})
}
return SendResponse(ctx, nil,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
Status: http.StatusCreated,
})
return sendResponse(ctx, nil, "The user has been created successfully", &metaOptions{
status: fiber.StatusCreated,
logger: c.l,
action: "admin:CreateUser",
})
}
func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:UpdateUser",
})
}
access := ctx.Query("access")
if access == "" {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
return sendResponse(ctx, errors.New("missing user access parameter"), nil,
&metaOptions{
status: fiber.StatusBadRequest,
logger: c.l,
action: "admin:UpdateUser",
})
}
var props auth.MutableProps
if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
if err := json.Unmarshal(ctx.Body(), &props); err != nil {
return sendResponse(ctx, fmt.Errorf("invalid request body %w", err), nil,
&metaOptions{
status: fiber.StatusBadRequest,
logger: c.l,
action: "admin:UpdateUser",
})
}
err := c.iam.UpdateUserAccount(access, props)
if err != nil {
status := fiber.StatusInternalServerError
err = fmt.Errorf("failed to update user account: %w", err)
if strings.Contains(err.Error(), "user not found") {
err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
status = fiber.StatusNotFound
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
return sendResponse(ctx, err, nil,
&metaOptions{
status: status,
logger: c.l,
action: "admin:UpdateUser",
})
}
return SendResponse(ctx, nil,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
return sendResponse(ctx, nil, "the user has been updated successfully",
&metaOptions{
logger: c.l,
action: "admin:UpdateUser",
})
}
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
access := ctx.Query("access")
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:DeleteUser",
})
}
err := c.iam.DeleteUserAccount(access)
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminDeleteUser,
if err != nil {
return sendResponse(ctx, err, nil,
&metaOptions{
logger: c.l,
action: "admin:DeleteUser",
})
}
return sendResponse(ctx, nil, "The user has been deleted successfully",
&metaOptions{
logger: c.l,
action: "admin:DeleteUser",
})
}
func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:ListUsers",
})
}
accs, err := c.iam.ListUserAccounts()
return SendXMLResponse(ctx,
auth.ListUserAccountsResult{
Accounts: accs,
}, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminListUsers,
return sendResponse(ctx, err, accs,
&metaOptions{
logger: c.l,
action: "admin:ListUsers",
})
}
func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:ChangeBucketOwner",
})
}
owner := ctx.Query("owner")
bucket := ctx.Query("bucket")
accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
return sendResponse(ctx, err, nil,
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
})
}
if len(accs) > 0 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
return sendResponse(ctx, errors.New("user specified as the new bucket owner does not exist"), nil,
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
status: fiber.StatusNotFound,
})
}
@@ -176,28 +235,91 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
aclParsed, err := json.Marshal(acl)
if err != nil {
return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
return sendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err), nil,
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
})
}
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
return sendResponse(ctx, err, "Bucket owner has been updated successfully",
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
})
}
func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:ListBuckets",
})
}
buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
return SendXMLResponse(ctx,
s3response.ListBucketsResult{
Buckets: buckets,
}, err, &MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminListBuckets,
return sendResponse(ctx, err, buckets,
&metaOptions{
logger: c.l,
action: "admin:ListBuckets",
})
}
type metaOptions struct {
action string
status int
logger s3log.AuditLogger
}
func sendResponse(ctx *fiber.Ctx, err error, data any, m *metaOptions) error {
status := m.status
if err != nil {
if status == 0 {
status = fiber.StatusInternalServerError
}
if m.logger != nil {
m.logger.Log(ctx, err, []byte(err.Error()), s3log.LogMeta{
Action: m.action,
HttpStatus: status,
})
}
return ctx.Status(status).SendString(err.Error())
}
if status == 0 {
status = fiber.StatusOK
}
msg, ok := data.(string)
if ok {
if m.logger != nil {
m.logger.Log(ctx, nil, []byte(msg), s3log.LogMeta{
Action: m.action,
HttpStatus: status,
})
}
return ctx.Status(status).SendString(msg)
}
dataJSON, err := json.Marshal(data)
if err != nil {
return err
}
if m.logger != nil {
m.logger.Log(ctx, nil, dataJSON, s3log.LogMeta{
HttpStatus: status,
Action: m.action,
})
}
ctx.Set(fiber.HeaderContentType, fiber.MIMEApplicationJSON)
return ctx.Status(status).Send(dataJSON)
}

View File

@@ -15,11 +15,12 @@
package controllers
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gofiber/fiber/v2"
@@ -42,60 +43,67 @@ func TestAdminController_CreateUser(t *testing.T) {
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/create-user", adminController.CreateUser)
succUser := `
<Account>
<Access>access</Access>
<Secret>secret</Secret>
<Role>admin</Role>
<UserID>0</UserID>
<GroupID>0</GroupID>
</Account>
`
invuser := `
<Account>
<Access>access</Access>
<Secret>secret</Secret>
<Role>invalid_role</Role>
<UserID>0</UserID>
<GroupID>0</GroupID>
</Account>
`
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
usr := auth.Account{
Access: "access",
Secret: "secret",
Role: "invalid role",
}
user, _ := json.Marshal(&usr)
usr.Role = "admin"
succUsr, _ := json.Marshal(&usr)
appErr.Patch("/create-user", adminController.CreateUser)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Admin-create-user-malformed-body",
name: "Admin-create-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
},
wantErr: false,
statusCode: 201,
},
{
name: "Admin-create-user-invalid-user-role",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
},
wantErr: false,
statusCode: 400,
},
{
name: "Admin-create-user-invalid-requester-role",
app: app,
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Admin-create-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
},
wantErr: false,
statusCode: 201,
statusCode: 403,
},
}
for _, tt := range tests {
@@ -126,8 +134,24 @@ func TestAdminController_UpdateUser(t *testing.T) {
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/update-user", adminController.UpdateUser)
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appErr.Patch("/update-user", adminController.UpdateUser)
successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
adminControllerErr := AdminController{
iam: &IAMServiceMock{
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
@@ -138,28 +162,25 @@ func TestAdminController_UpdateUser(t *testing.T) {
appNotFound := fiber.New()
appNotFound.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
succUser := `
<Account>
<Secret>secret</Secret>
<UserID>0</UserID>
<GroupID>0</GroupID>
</Account>
`
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Admin-update-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
},
wantErr: false,
statusCode: 200,
@@ -168,10 +189,10 @@ func TestAdminController_UpdateUser(t *testing.T) {
name: "Admin-update-user-missing-access",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
},
wantErr: false,
statusCode: 404,
statusCode: 400,
},
{
name: "Admin-update-user-invalid-request-body",
@@ -182,11 +203,20 @@ func TestAdminController_UpdateUser(t *testing.T) {
wantErr: false,
statusCode: 400,
},
{
name: "Admin-update-user-invalid-requester-role",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
},
wantErr: false,
statusCode: 403,
},
{
name: "Admin-update-user-not-found",
app: appNotFound,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
},
wantErr: false,
statusCode: 404,
@@ -220,14 +250,28 @@ func TestAdminController_DeleteUser(t *testing.T) {
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/delete-user", adminController.DeleteUser)
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appErr.Patch("/delete-user", adminController.DeleteUser)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Admin-delete-user-success",
@@ -238,6 +282,15 @@ func TestAdminController_DeleteUser(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Admin-delete-user-invalid-requester-role",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
},
wantErr: false,
statusCode: 403,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
@@ -274,18 +327,48 @@ func TestAdminController_ListUsers(t *testing.T) {
}
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appErr.Patch("/list-users", adminControllerErr.ListUsers)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appRoleErr.Patch("/list-users", adminController.ListUsers)
appSucc := fiber.New()
appSucc.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appSucc.Patch("/list-users", adminController.ListUsers)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Admin-list-users-access-denied",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
},
wantErr: false,
statusCode: 403,
},
{
name: "Admin-list-users-iam-error",
app: appErr,
@@ -352,21 +435,57 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
appIamErr := fiber.New()
appIamErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
appIamNoSuchUser := fiber.New()
appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Change-bucket-owner-access-denied",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 403,
},
{
name: "Change-bucket-owner-check-account-server-error",
app: appIamErr,
@@ -421,15 +540,39 @@ func TestAdminController_ListBuckets(t *testing.T) {
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/list-buckets", adminController.ListBuckets)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "List-buckets-incorrect-role",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
},
wantErr: false,
statusCode: 403,
},
{
name: "List-buckets-success",
app: app,

View File

@@ -41,7 +41,7 @@ var _ backend.Backend = &BackendMock{}
// CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
// panic("mock out the CreateMultipartUpload method")
// },
// DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error {
// DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
// panic("mock out the DeleteBucket method")
// },
// DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
@@ -74,7 +74,7 @@ var _ backend.Backend = &BackendMock{}
// GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
// panic("mock out the GetBucketTagging method")
// },
// GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
// GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
// panic("mock out the GetBucketVersioning method")
// },
// GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
@@ -83,7 +83,7 @@ var _ backend.Backend = &BackendMock{}
// GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
// panic("mock out the GetObjectAcl method")
// },
// GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
// GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
// panic("mock out the GetObjectAttributes method")
// },
// GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error) {
@@ -104,7 +104,7 @@ var _ backend.Backend = &BackendMock{}
// HeadObjectFunc: func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
// panic("mock out the HeadObject method")
// },
// ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
// ListBucketsFunc: func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
// panic("mock out the ListBuckets method")
// },
// ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
@@ -202,7 +202,7 @@ type BackendMock struct {
CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
// DeleteBucketFunc mocks the DeleteBucket method.
DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error
DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
@@ -235,7 +235,7 @@ type BackendMock struct {
GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
// GetBucketVersioningFunc mocks the GetBucketVersioning method.
GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
// GetObjectFunc mocks the GetObject method.
GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
@@ -244,7 +244,7 @@ type BackendMock struct {
GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
// GetObjectAttributesFunc mocks the GetObjectAttributes method.
GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
// GetObjectLegalHoldFunc mocks the GetObjectLegalHold method.
GetObjectLegalHoldFunc func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error)
@@ -265,7 +265,7 @@ type BackendMock struct {
HeadObjectFunc func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
// ListBucketsFunc mocks the ListBuckets method.
ListBucketsFunc func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
ListBucketsFunc func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
// ListBucketsAndOwnersFunc mocks the ListBucketsAndOwners method.
ListBucketsAndOwnersFunc func(contextMoqParam context.Context) ([]s3response.Bucket, error)
@@ -388,8 +388,8 @@ type BackendMock struct {
DeleteBucket []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// DeleteBucketInput is the deleteBucketInput argument value.
DeleteBucketInput *s3.DeleteBucketInput
}
// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
DeleteBucketOwnershipControls []struct {
@@ -547,8 +547,10 @@ type BackendMock struct {
ListBuckets []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// ListBucketsInput is the listBucketsInput argument value.
ListBucketsInput s3response.ListBucketsInput
// Owner is the owner argument value.
Owner string
// IsAdmin is the isAdmin argument value.
IsAdmin bool
}
// ListBucketsAndOwners holds details about calls to the ListBucketsAndOwners method.
ListBucketsAndOwners []struct {
@@ -1010,21 +1012,21 @@ func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
}
// DeleteBucket calls DeleteBucketFunc.
func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, bucket string) error {
func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
if mock.DeleteBucketFunc == nil {
panic("BackendMock.DeleteBucketFunc: method is nil but Backend.DeleteBucket was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
ContextMoqParam context.Context
DeleteBucketInput *s3.DeleteBucketInput
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
ContextMoqParam: contextMoqParam,
DeleteBucketInput: deleteBucketInput,
}
mock.lockDeleteBucket.Lock()
mock.calls.DeleteBucket = append(mock.calls.DeleteBucket, callInfo)
mock.lockDeleteBucket.Unlock()
return mock.DeleteBucketFunc(contextMoqParam, bucket)
return mock.DeleteBucketFunc(contextMoqParam, deleteBucketInput)
}
// DeleteBucketCalls gets all the calls that were made to DeleteBucket.
@@ -1032,12 +1034,12 @@ func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, bucket st
//
// len(mockedBackend.DeleteBucketCalls())
func (mock *BackendMock) DeleteBucketCalls() []struct {
ContextMoqParam context.Context
Bucket string
ContextMoqParam context.Context
DeleteBucketInput *s3.DeleteBucketInput
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
ContextMoqParam context.Context
DeleteBucketInput *s3.DeleteBucketInput
}
mock.lockDeleteBucket.RLock()
calls = mock.calls.DeleteBucket
@@ -1410,7 +1412,7 @@ func (mock *BackendMock) GetBucketTaggingCalls() []struct {
}
// GetBucketVersioning calls GetBucketVersioningFunc.
func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
if mock.GetBucketVersioningFunc == nil {
panic("BackendMock.GetBucketVersioningFunc: method is nil but Backend.GetBucketVersioning was just called")
}
@@ -1518,7 +1520,7 @@ func (mock *BackendMock) GetObjectAclCalls() []struct {
}
// GetObjectAttributes calls GetObjectAttributesFunc.
func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
if mock.GetObjectAttributesFunc == nil {
panic("BackendMock.GetObjectAttributesFunc: method is nil but Backend.GetObjectAttributes was just called")
}
@@ -1790,21 +1792,23 @@ func (mock *BackendMock) HeadObjectCalls() []struct {
}
// ListBuckets calls ListBucketsFunc.
func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
if mock.ListBucketsFunc == nil {
panic("BackendMock.ListBucketsFunc: method is nil but Backend.ListBuckets was just called")
}
callInfo := struct {
ContextMoqParam context.Context
ListBucketsInput s3response.ListBucketsInput
ContextMoqParam context.Context
Owner string
IsAdmin bool
}{
ContextMoqParam: contextMoqParam,
ListBucketsInput: listBucketsInput,
ContextMoqParam: contextMoqParam,
Owner: owner,
IsAdmin: isAdmin,
}
mock.lockListBuckets.Lock()
mock.calls.ListBuckets = append(mock.calls.ListBuckets, callInfo)
mock.lockListBuckets.Unlock()
return mock.ListBucketsFunc(contextMoqParam, listBucketsInput)
return mock.ListBucketsFunc(contextMoqParam, owner, isAdmin)
}
// ListBucketsCalls gets all the calls that were made to ListBuckets.
@@ -1812,12 +1816,14 @@ func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, listBucket
//
// len(mockedBackend.ListBucketsCalls())
func (mock *BackendMock) ListBucketsCalls() []struct {
ContextMoqParam context.Context
ListBucketsInput s3response.ListBucketsInput
ContextMoqParam context.Context
Owner string
IsAdmin bool
} {
var calls []struct {
ContextMoqParam context.Context
ListBucketsInput s3response.ListBucketsInput
ContextMoqParam context.Context
Owner string
IsAdmin bool
}
mock.lockListBuckets.RLock()
calls = mock.calls.ListBuckets

View File

@@ -51,9 +51,8 @@ type S3ApiController struct {
}
const (
iso8601Format = "20060102T150405Z"
iso8601TimeFormatExtended = "Mon Jan _2 15:04:05 2006"
defaultContentType = "binary/octet-stream"
iso8601Format = "20060102T150405Z"
defaultContentType = "binary/octet-stream"
)
func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) S3ApiController {
@@ -69,32 +68,8 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs
}
func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
cToken := ctx.Query("continuation-token")
prefix := ctx.Query("prefix")
maxBucketsStr := ctx.Query("max-buckets")
acct := ctx.Locals("account").(auth.Account)
maxBuckets, err := utils.ParseUint(maxBucketsStr)
if err != nil || maxBuckets > 10000 {
if c.debug {
log.Printf("error parsing max-buckets %q: %v\n", maxBucketsStr, err)
}
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListAllMyBuckets,
})
}
res, err := c.be.ListBuckets(ctx.Context(),
s3response.ListBucketsInput{
Owner: acct.Access,
IsAdmin: acct.Role == auth.RoleAdmin,
MaxBuckets: maxBuckets,
ContinuationToken: cToken,
Prefix: prefix,
})
res, err := c.be.ListBuckets(ctx.Context(), acct.Access, acct.Role == "admin")
return SendXMLResponse(ctx, res, err,
&MetaOpts{
Logger: c.logger,
@@ -108,6 +83,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
key := ctx.Params("key")
keyEnd := ctx.Params("*1")
uploadId := ctx.Query("uploadId")
maxParts := int32(ctx.QueryInt("max-parts", -1))
partNumberMarker := ctx.Query("part-number-marker")
acceptRange := ctx.Get("Range")
acct := ctx.Locals("account").(auth.Account)
@@ -245,6 +221,16 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
}
if uploadId != "" {
if maxParts < 0 && ctx.Request().URI().QueryArgs().Has("max-parts") {
return SendResponse(ctx,
s3err.GetAPIError(s3err.ErrInvalidMaxParts),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
})
}
if partNumberMarker != "" {
n, err := strconv.Atoi(partNumberMarker)
if err != nil || n < 0 {
@@ -262,24 +248,8 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
})
}
}
mxParts := ctx.Query("max-parts")
maxParts, err := utils.ParseUint(mxParts)
if err != nil {
if c.debug {
log.Printf("error parsing max parts %q: %v",
mxParts, err)
}
return SendResponse(ctx,
s3err.GetAPIError(s3err.ErrInvalidMaxParts),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
})
}
err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: types.PermissionRead,
@@ -298,13 +268,17 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
BucketOwner: parsedAcl.Owner,
})
}
var mxParts *int32
if ctx.Request().URI().QueryArgs().Has("max-parts") {
mxParts = &maxParts
}
res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
PartNumberMarker: &partNumberMarker,
MaxParts: &maxParts,
MaxParts: mxParts,
})
return SendXMLResponse(ctx, res, err,
&MetaOpts{
@@ -372,19 +346,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker")
maxPartsParsed, err := utils.ParseUint(maxParts)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxParts),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
})
}
attrs, err := utils.ParseObjectAttributes(ctx)
if err != nil {
if c.debug {
log.Printf("error parsing object attributes: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
@@ -393,6 +354,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
BucketOwner: parsedAcl.Owner,
})
}
attrs := utils.ParseObjectAttributes(ctx)
res, err := c.be.GetObjectAttributes(ctx.Context(),
&s3.GetObjectAttributesInput{
@@ -403,22 +365,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
VersionId: &versionId,
})
if err != nil {
hdrs := []utils.CustomHeader{}
if res.DeleteMarker != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-delete-marker",
Value: "true",
})
}
if getstring(res.VersionId) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-version-id",
Value: getstring(res.VersionId),
})
}
utils.SetResponseHeaders(ctx, hdrs)
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
@@ -427,31 +373,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
BucketOwner: parsedAcl.Owner,
})
}
hdrs := []utils.CustomHeader{}
if getstring(res.VersionId) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-version-id",
Value: getstring(res.VersionId),
})
}
if res.DeleteMarker != nil && *res.DeleteMarker {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-delete-marker",
Value: "true",
})
}
if res.LastModified != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Last-Modified",
Value: res.LastModified.UTC().Format(iso8601TimeFormatExtended),
})
}
utils.SetResponseHeaders(ctx, hdrs)
return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), nil,
return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
@@ -814,7 +736,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
log.Printf("error parsing max keys %q: %v",
maxkeysStr, err)
}
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
@@ -947,13 +869,12 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
log.Printf("error parsing max uploads %q: %v",
maxUploadsStr, err)
}
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxUploads),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListMultipartUploads,
BucketOwner: parsedAcl.Owner,
})
return SendXMLResponse(ctx, nil, err, &MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListMultipartUploads,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.ListMultipartUploads(ctx.Context(),
&s3.ListMultipartUploadsInput{
@@ -998,7 +919,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
log.Printf("error parsing max keys %q: %v",
maxkeysStr, err)
}
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
@@ -1049,7 +970,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
log.Printf("error parsing max keys %q: %v",
maxkeysStr, err)
}
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
@@ -1150,7 +1071,6 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
MetricsMng: c.mm,
Action: metrics.ActionPutBucketTagging,
BucketOwner: parsedAcl.Owner,
Status: http.StatusNoContent,
})
}
@@ -2323,7 +2243,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
})
}
err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &keyStart}}, true, c.be)
err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{keyStart}, true, c.be)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
@@ -2548,7 +2468,10 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
})
}
err = c.be.DeleteBucket(ctx.Context(), bucket)
err = c.be.DeleteBucket(ctx.Context(),
&s3.DeleteBucketInput{
Bucket: &bucket,
})
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
@@ -2604,7 +2527,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
// The AWS CLI sends 'True', while Go SDK sends 'true'
bypass := strings.EqualFold(bypassHdr, "true")
err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, c.be)
err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, utils.ParseDeleteObjects(dObj.Objects), bypass, c.be)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
@@ -2757,7 +2680,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
// The AWS CLI sends 'True', while Go SDK sends 'true'
bypass := strings.EqualFold(bypassHdr, "true")
err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key, VersionId: &versionId}}, bypass, c.be)
err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{key}, bypass, c.be)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
@@ -2954,6 +2877,15 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
BucketOwner: parsedAcl.Owner,
})
}
if res == nil {
return SendResponse(ctx, fmt.Errorf("head object nil response"),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionHeadObject,
BucketOwner: parsedAcl.Owner,
})
}
utils.SetMetaHeaders(ctx, res.Metadata)
headers := []utils.CustomHeader{
@@ -3308,14 +3240,14 @@ type MetaOpts struct {
Logger s3log.AuditLogger
EvSender s3event.S3EventSender
MetricsMng *metrics.Manager
ObjectETag *string
VersionId *string
ContentLength int64
Action string
BucketOwner string
EventName s3event.EventType
ContentLength int64
ObjectSize int64
ObjectCount int64
EventName s3event.EventType
ObjectETag *string
VersionId *string
Status int
}

View File

@@ -92,7 +92,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, nil
},
},
@@ -109,7 +109,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{
be: &BackendMock{
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
},
},
@@ -123,11 +123,11 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
appErr.Get("/", s3ApiControllerErr.ListBuckets)
tests := []struct {
name string
args args
app *fiber.App
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "List-bucket-method-not-allowed",
@@ -187,8 +187,8 @@ func TestS3ApiController_GetActions(t *testing.T) {
GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return &s3.GetObjectAclOutput{}, nil
},
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
return s3response.GetObjectAttributesResponse{}, nil
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
return s3response.GetObjectAttributesResult{}, nil
},
GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
return &s3.GetObjectOutput{
@@ -233,11 +233,11 @@ func TestS3ApiController_GetActions(t *testing.T) {
getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Get-actions-get-tags-success",
@@ -382,8 +382,8 @@ func TestS3ApiController_ListActions(t *testing.T) {
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
return map[string]string{}, nil
},
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, nil
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
return &s3.GetBucketVersioningOutput{}, nil
},
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
return s3response.ListVersionsResult{}, nil
@@ -435,11 +435,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
appError.Get("/:bucket", s3ApiControllerError.ListActions)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Get-bucket-tagging-non-existing-bucket",
@@ -728,11 +728,11 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Put-bucket-tagging-invalid-body",
@@ -750,7 +750,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
},
wantErr: false,
statusCode: 204,
statusCode: 200,
},
{
name: "Put-bucket-ownership-controls-invalid-ownership",
@@ -1034,11 +1034,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Put-object-part-error-case",
@@ -1217,7 +1217,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
DeleteBucketFunc: func(_ context.Context, bucket string) error {
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
return nil
},
DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
@@ -1243,11 +1243,11 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
app.Delete("/:bucket", s3ApiController.DeleteBucket)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Delete-bucket-success",
@@ -1334,11 +1334,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
request.Header.Set("Content-Type", "application/xml")
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Delete-Objects-success",
@@ -1432,11 +1432,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Abort-multipart-upload-success",
@@ -1541,11 +1541,11 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Head-bucket-success",
@@ -1628,7 +1628,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
return acldata, nil
},
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
return nil, s3err.GetAPIError(42)
},
},
}
@@ -1643,11 +1643,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Head-object-success",
@@ -1723,11 +1723,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
tests := []struct {
name string
app *fiber.App
args args
name string
statusCode int
wantErr bool
statusCode int
}{
{
name: "Restore-object-success",
@@ -1808,10 +1808,10 @@ func Test_XMLresponse(t *testing.T) {
ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
tests := []struct {
args args
name string
statusCode int
args args
wantErr bool
statusCode int
}{
{
name: "Internal-server-error",
@@ -1883,10 +1883,10 @@ func Test_response(t *testing.T) {
ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
tests := []struct {
args args
name string
statusCode int
args args
wantErr bool
statusCode int
}{
{
name: "Internal-server-error",

View File

@@ -1,59 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"strings"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != auth.RoleAdmin {
path := ctx.Path()
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
&controllers.MetaOpts{
Logger: logger,
Action: detectAction(path),
})
}
return ctx.Next()
}
}
func detectAction(path string) (action string) {
if strings.Contains(path, "create-user") {
action = metrics.ActionAdminCreateUser
} else if strings.Contains(path, "update-user") {
action = metrics.ActionAdminUpdateUser
} else if strings.Contains(path, "delete-user") {
action = metrics.ActionAdminDeleteUser
} else if strings.Contains(path, "list-user") {
action = metrics.ActionAdminListUsers
} else if strings.Contains(path, "list-buckets") {
action = metrics.ActionAdminListBuckets
} else if strings.Contains(path, "change-bucket-owner") {
action = metrics.ActionAdminChangeBucketOwner
}
return action
}

View File

@@ -149,8 +149,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
}
type accounts struct {
iam auth.IAMService
root RootUserConfig
iam auth.IAMService
}
func (a accounts) getAccount(access string) (auth.Account, error) {

View File

@@ -26,11 +26,12 @@ import (
func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
return func(ctx *fiber.Ctx) error {
unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
reqURL := ctx.Request().URI().String()
decoded, err := url.Parse(reqURL)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
}
ctx.Path(unescp)
ctx.Path(decoded.Path)
return ctx.Next()
}
}

View File

@@ -20,7 +20,6 @@ import (
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
@@ -36,22 +35,22 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
adminController := controllers.NewAdminController(iam, be, aLogger)
// CreateUser admin api
app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
app.Patch("/create-user", adminController.CreateUser)
// DeleteUsers admin api
app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
app.Patch("/delete-user", adminController.DeleteUser)
// UpdateUser admin api
app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
app.Patch("update-user", adminController.UpdateUser)
// ListUsers admin api
app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
app.Patch("/list-users", adminController.ListUsers)
// ChangeBucketOwner admin api
app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
// ListBucketsAndOwners admin api
app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
app.Patch("/list-buckets", adminController.ListBuckets)
}
// ListBuckets action

View File

@@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) {
iam auth.IAMService
}
tests := []struct {
args args
sa *S3ApiRouter
name string
sa *S3ApiRouter
args args
}{
{
name: "Initialize S3 api router",

View File

@@ -29,15 +29,15 @@ import (
)
type S3ApiServer struct {
backend backend.Backend
app *fiber.App
backend backend.Backend
router *S3ApiRouter
cert *tls.Certificate
port string
health string
cert *tls.Certificate
quiet bool
debug bool
readonly bool
health string
}
func New(

View File

@@ -39,9 +39,9 @@ func TestNew(t *testing.T) {
port := ":7070"
tests := []struct {
wantS3ApiServer *S3ApiServer
args args
name string
args args
wantS3ApiServer *S3ApiServer
wantErr bool
}{
{
@@ -78,8 +78,8 @@ func TestNew(t *testing.T) {
func TestS3ApiServer_Serve(t *testing.T) {
tests := []struct {
sa *S3ApiServer
name string
sa *S3ApiServer
wantErr bool
}{
{

View File

@@ -41,10 +41,10 @@ const (
// the data is completely read.
type AuthReader struct {
ctx *fiber.Ctx
r *HashReader
auth AuthData
secret string
size int
r *HashReader
debug bool
}

View File

@@ -48,15 +48,15 @@ const (
// object data stream
type ChunkReader struct {
r io.Reader
chunkHash hash.Hash
signingKey []byte
prevSig string
parsedSig string
strToSignPrefix string
signingKey []byte
stash []byte
currentChunkSize int64
chunkDataLeft int64
trailerExpected int
stash []byte
chunkHash hash.Hash
strToSignPrefix string
skipcheck bool
}

View File

@@ -41,10 +41,10 @@ const (
// data requests where the data size is not known until
// the data is completely read.
type PresignedAuthReader struct {
r io.Reader
ctx *fiber.Ctx
auth AuthData
secret string
r io.Reader
debug bool
}

View File

@@ -23,13 +23,13 @@ import (
func Test_validateExpiration(t *testing.T) {
type args struct {
date time.Time
str string
date time.Time
}
tests := []struct {
name string
args args
err error
name string
}{
{
name: "empty-expiration",

View File

@@ -39,10 +39,6 @@ var (
bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
)
const (
upperhex = "0123456789ABCDEF"
)
func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
metadata = make(map[string]string)
headers.DisableNormalizing()
@@ -68,9 +64,7 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
body = bytes.NewReader(req.Body())
}
escapedURI := escapeOriginalURI(ctx)
httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
if err != nil {
return nil, errors.New("error in creating an http request")
}
@@ -180,7 +174,7 @@ func ParseUint(str string) (int32, error) {
}
num, err := strconv.ParseUint(str, 10, 16)
if err != nil {
return 1000, fmt.Errorf("invalid uint: %w", err)
return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
}
return int32(num), nil
}
@@ -254,51 +248,35 @@ func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
return
}
func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResponse) s3response.GetObjectAttributesResponse {
// These properties shouldn't appear in the final response body
output.LastModified = nil
output.VersionId = nil
output.DeleteMarker = nil
if _, ok := attrs[s3response.ObjectAttributesEtag]; !ok {
func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResult) s3response.GetObjectAttributesResult {
if _, ok := attrs[types.ObjectAttributesEtag]; !ok {
output.ETag = nil
}
if _, ok := attrs[s3response.ObjectAttributesObjectParts]; !ok {
if _, ok := attrs[types.ObjectAttributesObjectParts]; !ok {
output.ObjectParts = nil
}
if _, ok := attrs[s3response.ObjectAttributesObjectSize]; !ok {
if _, ok := attrs[types.ObjectAttributesObjectSize]; !ok {
output.ObjectSize = nil
}
if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
output.StorageClass = ""
}
fmt.Printf("%+v\n", output)
return output
}
func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
attrs := map[s3response.ObjectAttributes]struct{}{}
var err error
func ParseObjectAttributes(ctx *fiber.Ctx) map[types.ObjectAttributes]struct{} {
attrs := map[types.ObjectAttributes]struct{}{}
ctx.Request().Header.VisitAll(func(key, value []byte) {
if string(key) == "X-Amz-Object-Attributes" {
oattrs := strings.Split(string(value), ",")
for _, a := range oattrs {
attr := s3response.ObjectAttributes(a)
if !attr.IsValid() {
err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
break
}
attrs[attr] = struct{}{}
attrs[types.ObjectAttributes(a)] = struct{}{}
}
}
})
if len(attrs) == 0 {
return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
}
return attrs, err
return attrs
}
type objLockCfg struct {
@@ -361,74 +339,3 @@ func IsValidOwnership(val types.ObjectOwnership) bool {
return false
}
}
func escapeOriginalURI(ctx *fiber.Ctx) string {
path := ctx.Path()
// Escape the URI original path
escapedURI := escapePath(path)
// Add the URI query params
query := string(ctx.Request().URI().QueryArgs().QueryString())
if query != "" {
escapedURI = escapedURI + "?" + query
}
return escapedURI
}
// Escapes the path string
// Most of the parts copied from std url
func escapePath(s string) string {
hexCount := 0
for i := 0; i < len(s); i++ {
c := s[i]
if shouldEscape(c) {
hexCount++
}
}
if hexCount == 0 {
return s
}
var buf [64]byte
var t []byte
required := len(s) + 2*hexCount
if required <= len(buf) {
t = buf[:required]
} else {
t = make([]byte, required)
}
j := 0
for i := 0; i < len(s); i++ {
switch c := s[i]; {
case shouldEscape(c):
t[j] = '%'
t[j+1] = upperhex[c>>4]
t[j+2] = upperhex[c&15]
j += 3
default:
t[j] = s[i]
j++
}
}
return string(t)
}
// Checks if the character needs to be escaped
func shouldEscape(c byte) bool {
if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
return false
}
switch c {
case '-', '_', '.', '~', '/':
return false
}
return true
}

View File

@@ -19,12 +19,10 @@ import (
"net/http"
"reflect"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3response"
)
@@ -49,11 +47,11 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
tests := []struct {
name string
args args
want *http.Request
name string
hdrs []string
wantErr bool
hdrs []string
}{
{
name: "Success-response",
@@ -101,9 +99,9 @@ func TestGetUserMetaData(t *testing.T) {
req := ctx.Request()
tests := []struct {
name string
args args
wantMetadata map[string]string
name string
}{
{
name: "Success-empty-response",
@@ -285,68 +283,47 @@ func TestParseUint(t *testing.T) {
func TestFilterObjectAttributes(t *testing.T) {
type args struct {
attrs map[s3response.ObjectAttributes]struct{}
output s3response.GetObjectAttributesResponse
attrs map[types.ObjectAttributes]struct{}
output s3response.GetObjectAttributesResult
}
etag, objSize := "etag", int64(3222)
delMarker := true
tests := []struct {
name string
args args
want s3response.GetObjectAttributesResponse
want s3response.GetObjectAttributesResult
}{
{
name: "keep only ETag",
args: args{
attrs: map[s3response.ObjectAttributes]struct{}{
s3response.ObjectAttributesEtag: {},
attrs: map[types.ObjectAttributes]struct{}{
types.ObjectAttributesEtag: {},
},
output: s3response.GetObjectAttributesResponse{
output: s3response.GetObjectAttributesResult{
ObjectSize: &objSize,
ETag: &etag,
},
},
want: s3response.GetObjectAttributesResponse{ETag: &etag},
want: s3response.GetObjectAttributesResult{ETag: &etag},
},
{
name: "keep multiple props",
args: args{
attrs: map[s3response.ObjectAttributes]struct{}{
s3response.ObjectAttributesEtag: {},
s3response.ObjectAttributesObjectSize: {},
s3response.ObjectAttributesStorageClass: {},
attrs: map[types.ObjectAttributes]struct{}{
types.ObjectAttributesEtag: {},
types.ObjectAttributesObjectSize: {},
types.ObjectAttributesStorageClass: {},
},
output: s3response.GetObjectAttributesResponse{
output: s3response.GetObjectAttributesResult{
ObjectSize: &objSize,
ETag: &etag,
ObjectParts: &s3response.ObjectParts{},
VersionId: &etag,
},
},
want: s3response.GetObjectAttributesResponse{
want: s3response.GetObjectAttributesResult{
ETag: &etag,
ObjectSize: &objSize,
},
},
{
name: "make sure LastModified, DeleteMarker and VersionId are removed",
args: args{
attrs: map[s3response.ObjectAttributes]struct{}{
s3response.ObjectAttributesEtag: {},
},
output: s3response.GetObjectAttributesResponse{
ObjectSize: &objSize,
ETag: &etag,
ObjectParts: &s3response.ObjectParts{},
VersionId: &etag,
LastModified: backend.GetTimePtr(time.Now()),
DeleteMarker: &delMarker,
},
},
want: s3response.GetObjectAttributesResponse{
ETag: &etag,
VersionId: &etag,
},
},
}
@@ -405,125 +382,3 @@ func TestIsValidOwnership(t *testing.T) {
})
}
}
func Test_shouldEscape(t *testing.T) {
type args struct {
c byte
}
tests := []struct {
name string
args args
want bool
}{
{
name: "shouldn't-escape-alphanum",
args: args{
c: 'h',
},
want: false,
},
{
name: "shouldn't-escape-unreserved-char",
args: args{
c: '_',
},
want: false,
},
{
name: "shouldn't-escape-unreserved-number",
args: args{
c: '0',
},
want: false,
},
{
name: "shouldn't-escape-path-separator",
args: args{
c: '/',
},
want: false,
},
{
name: "should-escape-special-char-1",
args: args{
c: '&',
},
want: true,
},
{
name: "should-escape-special-char-2",
args: args{
c: '*',
},
want: true,
},
{
name: "should-escape-special-char-3",
args: args{
c: '(',
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := shouldEscape(tt.args.c); got != tt.want {
t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
}
})
}
}
func Test_escapePath(t *testing.T) {
type args struct {
s string
}
tests := []struct {
name string
args args
want string
}{
{
name: "empty-string",
args: args{
s: "",
},
want: "",
},
{
name: "alphanum-path",
args: args{
s: "/test-bucket/test-key",
},
want: "/test-bucket/test-key",
},
{
name: "path-with-unescapable-chars",
args: args{
s: "/test~bucket/test.key",
},
want: "/test~bucket/test.key",
},
{
name: "path-with-escapable-chars",
args: args{
s: "/bucket-*(/test=key&",
},
want: "/bucket-%2A%28/test%3Dkey%26",
},
{
name: "path-with-space",
args: args{
s: "/test-bucket/my key",
},
want: "/test-bucket/my%20key",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := escapePath(tt.args.s); got != tt.want {
t.Errorf("escapePath() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -57,7 +57,6 @@ const (
ErrAccessDenied
ErrMethodNotAllowed
ErrBucketNotEmpty
ErrVersionedBucketNotEmpty
ErrBucketAlreadyExists
ErrBucketAlreadyOwnedByYou
ErrNoSuchBucket
@@ -66,11 +65,9 @@ const (
ErrInvalidBucketName
ErrInvalidDigest
ErrInvalidMaxKeys
ErrInvalidMaxBuckets
ErrInvalidMaxUploads
ErrInvalidMaxParts
ErrInvalidPartNumberMarker
ErrInvalidObjectAttributes
ErrInvalidPart
ErrInvalidPartNumber
ErrInternalError
@@ -125,7 +122,6 @@ const (
ErrNoSuchBucketPolicy
ErrBucketTaggingNotFound
ErrObjectLockInvalidHeaders
ErrObjectAttributesInvalidHeader
ErrRequestTimeTooSkewed
ErrInvalidBucketAclWithObjectOwnership
ErrBothCannedAndHeaderGrants
@@ -138,21 +134,12 @@ const (
ErrKeyTooLong
ErrInvalidVersionId
ErrNoSuchVersion
ErrSuspendedVersioningNotAllowed
// Non-AWS errors
ErrExistingObjectIsDirectory
ErrObjectParentIsFile
ErrDirectoryObjectContainsData
ErrQuotaExceeded
ErrVersioningNotConfigured
// Admin api errors
ErrAdminAccessDenied
ErrAdminUserNotFound
ErrAdminUserExists
ErrAdminInvalidUserRole
ErrAdminMissingUserAcess
)
var errorCodeResponse = map[ErrorCode]APIError{
@@ -171,11 +158,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The bucket you tried to delete is not empty.",
HTTPStatusCode: http.StatusConflict,
},
ErrVersionedBucketNotEmpty: {
Code: "BucketNotEmpty",
Description: "The bucket you tried to delete is not empty. You must delete all versions in the bucket.",
HTTPStatusCode: http.StatusConflict,
},
ErrBucketAlreadyExists: {
Code: "BucketAlreadyExists",
Description: "The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again.",
@@ -196,11 +178,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The Content-Md5 you specified is not valid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxBuckets: {
Code: "InvalidArgument",
Description: "Argument max-buckets must be an integer between 1 and 10000.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxUploads: {
Code: "InvalidArgument",
Description: "Argument max-uploads must be an integer between 0 and 2147483647.",
@@ -221,11 +198,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Argument partNumberMarker must be an integer.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidObjectAttributes: {
Code: "InvalidArgument",
Description: "Invalid attribute name specified.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNoSuchBucket: {
Code: "NoSuchBucket",
Description: "The specified bucket does not exist.",
@@ -383,7 +355,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrInvalidAccessKeyID: {
Code: "InvalidAccessKeyId",
Description: "The AWS Access Key Id you provided does not exist in our records.",
Description: "The access key ID you provided does not exist in our records.",
HTTPStatusCode: http.StatusForbidden,
},
ErrRequestNotReadyYet: {
@@ -463,12 +435,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrNoSuchObjectLockConfiguration: {
Code: "NoSuchObjectLockConfiguration",
Description: "The specified object does not have a ObjectLock configuration.",
Description: "The specified object does not have an ObjectLock configuration.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidBucketObjectLockConfiguration: {
Code: "InvalidRequest",
Description: "Bucket is missing Object Lock Configuration.",
Description: "Bucket is missing ObjectLockConfiguration.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrObjectLockConfigurationNotAllowed: {
@@ -506,11 +478,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrObjectAttributesInvalidHeader: {
Code: "InvalidRequest",
Description: "The x-amz-object-attributes header specifying the attributes to be retrieved is either missing or empty",
HTTPStatusCode: http.StatusBadRequest,
},
ErrRequestTimeTooSkewed: {
Code: "RequestTimeTooSkewed",
Description: "The difference between the request time and the server's time is too large.",
@@ -552,9 +519,8 @@ var errorCodeResponse = map[ErrorCode]APIError{
HTTPStatusCode: http.StatusNotFound,
},
ErrInvalidMetadataDirective: {
Code: "InvalidArgument",
Description: "Unknown metadata directive.",
HTTPStatusCode: http.StatusBadRequest,
Code: "InvalidArgument",
Description: "Unknown metadata directive.",
},
ErrInvalidVersionId: {
Code: "InvalidArgument",
@@ -571,11 +537,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The specified version does not exist.",
HTTPStatusCode: http.StatusNotFound,
},
ErrSuspendedVersioningNotAllowed: {
Code: "InvalidBucketState",
Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
HTTPStatusCode: http.StatusBadRequest,
},
// non aws errors
ErrExistingObjectIsDirectory: {
@@ -598,38 +559,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Your request was denied due to quota exceeded.",
HTTPStatusCode: http.StatusForbidden,
},
ErrVersioningNotConfigured: {
Code: "VersioningNotConfigured",
Description: "Versioning has not been configured for the gateway.",
HTTPStatusCode: http.StatusNotImplemented,
},
// Admin api errors
ErrAdminAccessDenied: {
Code: "XAdminAccessDenied",
Description: "Only admin users have access to this resource.",
HTTPStatusCode: http.StatusForbidden,
},
ErrAdminUserNotFound: {
Code: "XAdminUserNotFound",
Description: "No user exists with the provided access key ID.",
HTTPStatusCode: http.StatusNotFound,
},
ErrAdminUserExists: {
Code: "XAdminUserExists",
Description: "A user with the provided access key ID already exists.",
HTTPStatusCode: http.StatusConflict,
},
ErrAdminInvalidUserRole: {
Code: "XAdminInvalidArgument",
Description: "User role has to be one of the following: 'user', 'admin', 'userplus'.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminMissingUserAcess: {
Code: "XAdminInvalidArgument",
Description: "User access key ID is missing.",
HTTPStatusCode: http.StatusNotFound,
},
}
// GetAPIError provides API Error for input API error code.

View File

@@ -30,11 +30,11 @@ type S3EventSender interface {
}
type EventMeta struct {
ObjectETag *string
VersionId *string
BucketOwner string
EventName EventType
ObjectSize int64
ObjectETag *string
VersionId *string
}
type EventSchema struct {
@@ -42,8 +42,6 @@ type EventSchema struct {
}
type EventRecord struct {
ResponseElements EventResponseElements `json:"responseElements"`
GlacierEventData EventGlacierData `json:"glacierEventData"`
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"`
AwsRegion string `json:"awsRegion"`
@@ -51,7 +49,9 @@ type EventRecord struct {
EventName EventType `json:"eventName"`
UserIdentity EventUserIdentity `json:"userIdentity"`
RequestParameters EventRequestParams `json:"requestParameters"`
ResponseElements EventResponseElements `json:"responseElements"`
S3 EventS3Data `json:"s3"`
GlacierEventData EventGlacierData `json:"glacierEventData"`
}
type EventUserIdentity struct {
@@ -99,11 +99,11 @@ type EventS3BucketData struct {
}
type EventObjectData struct {
Key string `json:"key"`
Size int64 `json:"size"`
ETag *string `json:"eTag"`
VersionId *string `json:"versionId"`
Key string `json:"key"`
Sequencer string `json:"sequencer"`
Size int64 `json:"size"`
}
type EventConfig struct {

View File

@@ -31,9 +31,9 @@ import (
var sequencer = 0
type Kafka struct {
key string
writer *kafka.Writer
filter EventFilter
key string
mu sync.Mutex
}

View File

@@ -27,10 +27,10 @@ import (
)
type NatsEventSender struct {
client *nats.Conn
filter EventFilter
topic string
client *nats.Conn
mu sync.Mutex
filter EventFilter
}
func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {

View File

@@ -30,9 +30,9 @@ import (
)
type Webhook struct {
url string
client *http.Client
filter EventFilter
url string
mu sync.Mutex
}

View File

@@ -33,8 +33,8 @@ type AuditLogger interface {
type LogMeta struct {
BucketOwner string
Action string
ObjectSize int64
Action string
HttpStatus int
}
@@ -45,16 +45,21 @@ type LogConfig struct {
}
type LogFields struct {
Time time.Time
BucketOwner string
Bucket string
Time time.Time
RemoteIP string
Requester string
RequestID string
Operation string
Key string
RequestURI string
HttpStatus int
ErrorCode string
BytesSent int
ObjectSize int64
TotalTime int64
TurnAroundTime int64
Referer string
UserAgent string
VersionID string
@@ -66,11 +71,6 @@ type LogFields struct {
TLSVersion string
AccessPointARN string
AclRequired string
HttpStatus int
BytesSent int
ObjectSize int64
TotalTime int64
TurnAroundTime int64
}
type AdminLogFields struct {
@@ -80,17 +80,17 @@ type AdminLogFields struct {
RequestID string
Operation string
RequestURI string
HttpStatus int
ErrorCode string
BytesSent int
TotalTime int64
TurnAroundTime int64
Referer string
UserAgent string
SignatureVersion string
CipherSuite string
AuthenticationType string
TLSVersion string
HttpStatus int
BytesSent int
TotalTime int64
TurnAroundTime int64
}
type Loggers struct {

View File

@@ -34,10 +34,10 @@ const (
// FileLogger is a local file audit log
type FileLogger struct {
f *os.File
logfile string
mu sync.Mutex
f *os.File
gotErr bool
mu sync.Mutex
}
var _ AuditLogger = &FileLogger{}

View File

@@ -33,8 +33,8 @@ import (
// WebhookLogger is a webhook URL audit log
type WebhookLogger struct {
url string
mu sync.Mutex
url string
}
var _ AuditLogger = &WebhookLogger{}

View File

@@ -19,14 +19,9 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
)
const (
iso8601TimeFormat = "2006-01-02T15:04:05.000Z"
iso8601TimeFormatExtended = "2006-01-02T15:04:05.000000Z"
iso8601TimeFormatWithTZ = "2006-01-02T15:04:05-0700"
)
const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
type PutObjectOutput struct {
ETag string
@@ -35,22 +30,22 @@ type PutObjectOutput struct {
// Part describes part metadata.
type Part struct {
PartNumber int
LastModified time.Time
ETag string
PartNumber int
Size int64
}
func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type Alias Part
aux := &struct {
*Alias
LastModified string `xml:"LastModified"`
*Alias
}{
Alias: (*Alias)(&p),
}
aux.LastModified = p.LastModified.UTC().Format(iso8601TimeFormat)
aux.LastModified = p.LastModified.UTC().Format(RFC3339TimeFormat)
return e.EncodeElement(aux, start)
}
@@ -59,61 +54,57 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
Initiator Initiator
Owner Owner
Bucket string
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
// The class of storage used to store the object.
StorageClass types.StorageClass
// List of parts.
Parts []Part `xml:"Part"`
PartNumberMarker int
NextPartNumberMarker int
MaxParts int
IsTruncated bool
// List of parts.
Parts []Part `xml:"Part"`
}
type ObjectAttributes string
const (
ObjectAttributesEtag ObjectAttributes = "ETag"
ObjectAttributesChecksum ObjectAttributes = "Checksum"
ObjectAttributesObjectParts ObjectAttributes = "ObjectParts"
ObjectAttributesStorageClass ObjectAttributes = "StorageClass"
ObjectAttributesObjectSize ObjectAttributes = "ObjectSize"
)
func (o ObjectAttributes) IsValid() bool {
return o == ObjectAttributesChecksum ||
o == ObjectAttributesEtag ||
o == ObjectAttributesObjectParts ||
o == ObjectAttributesObjectSize ||
o == ObjectAttributesStorageClass
}
type GetObjectAttributesResponse struct {
ETag *string
ObjectSize *int64
ObjectParts *ObjectParts
// Not included in the response body
VersionId *string
type GetObjectAttributesResult struct {
ETag *string
LastModified *time.Time
DeleteMarker *bool
StorageClass types.StorageClass `xml:",omitempty"`
ObjectSize *int64
StorageClass types.StorageClass
VersionId *string
ObjectParts *ObjectParts
}
func (r GetObjectAttributesResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type Alias GetObjectAttributesResult
aux := &struct {
LastModified *string `xml:"LastModified"`
*Alias
}{
Alias: (*Alias)(&r),
}
if r.LastModified != nil {
formattedTime := r.LastModified.UTC().Format(RFC3339TimeFormat)
aux.LastModified = &formattedTime
}
return e.EncodeElement(aux, start)
}
type ObjectParts struct {
Parts []types.ObjectPart `xml:"Part"`
PartNumberMarker int
NextPartNumberMarker int
MaxParts int
IsTruncated bool
Parts []types.ObjectPart `xml:"Part"`
}
// ListMultipartUploadsResponse - s3 api list multipart uploads response.
@@ -128,17 +119,18 @@ type ListMultipartUploadsResult struct {
Delimiter string
Prefix string
EncodingType string `xml:"EncodingType,omitempty"`
MaxUploads int
IsTruncated bool
// List of pending uploads.
Uploads []Upload `xml:"Upload"`
// Delimed common prefixes.
CommonPrefixes []CommonPrefix
MaxUploads int
IsTruncated bool
}
type ListObjectsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
Name *string
Prefix *string
Marker *string
@@ -146,13 +138,13 @@ type ListObjectsResult struct {
MaxKeys *int32
Delimiter *string
IsTruncated *bool
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
EncodingType types.EncodingType
Contents []Object
CommonPrefixes []types.CommonPrefix
EncodingType types.EncodingType
}
type ListObjectsV2Result struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
Name *string
Prefix *string
StartAfter *string
@@ -162,10 +154,9 @@ type ListObjectsV2Result struct {
MaxKeys *int32
Delimiter *string
IsTruncated *bool
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
EncodingType types.EncodingType
Contents []Object
CommonPrefixes []types.CommonPrefix
EncodingType types.EncodingType
}
type Object struct {
@@ -188,7 +179,7 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
}
if o.LastModified != nil {
formattedTime := o.LastModified.UTC().Format(iso8601TimeFormat)
formattedTime := o.LastModified.UTC().Format(RFC3339TimeFormat)
aux.LastModified = &formattedTime
}
@@ -197,24 +188,24 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
// Upload describes in progress multipart upload
type Upload struct {
Initiated time.Time
Initiator Initiator
Owner Owner
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
StorageClass types.StorageClass
Initiated time.Time
}
func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type Alias Upload
aux := &struct {
*Alias
Initiated string `xml:"Initiated"`
*Alias
}{
Alias: (*Alias)(&u),
}
aux.Initiated = u.Initiated.UTC().Format(iso8601TimeFormat)
aux.Initiated = u.Initiated.UTC().Format(RFC3339TimeFormat)
return e.EncodeElement(aux, start)
}
@@ -261,11 +252,11 @@ type DeleteResult struct {
}
type SelectObjectContentPayload struct {
Expression *string
ExpressionType types.ExpressionType
RequestProgress *types.RequestProgress
InputSerialization *types.InputSerialization
OutputSerialization *types.OutputSerialization
ScanRange *types.ScanRange
ExpressionType types.ExpressionType
}
type SelectObjectContentResult struct {
@@ -281,37 +272,27 @@ type Bucket struct {
Owner string `json:"owner"`
}
type ListBucketsInput struct {
Owner string
ContinuationToken string
Prefix string
MaxBuckets int32
IsAdmin bool
}
type ListAllMyBucketsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
Owner CanonicalUser
ContinuationToken string `xml:"ContinuationToken,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
Buckets ListAllMyBucketsList
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
Owner CanonicalUser
Buckets ListAllMyBucketsList
}
type ListAllMyBucketsEntry struct {
CreationDate time.Time
Name string
CreationDate time.Time
}
func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type Alias ListAllMyBucketsEntry
aux := &struct {
*Alias
CreationDate string `xml:"CreationDate"`
*Alias
}{
Alias: (*Alias)(&r),
}
aux.CreationDate = r.CreationDate.UTC().Format(iso8601TimeFormat)
aux.CreationDate = r.CreationDate.UTC().Format(RFC3339TimeFormat)
return e.EncodeElement(aux, start)
}
@@ -335,13 +316,13 @@ type CopyObjectResult struct {
func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type Alias CopyObjectResult
aux := &struct {
*Alias
LastModified string `xml:"LastModified"`
*Alias
}{
Alias: (*Alias)(&r),
}
aux.LastModified = r.LastModified.UTC().Format(iso8601TimeFormat)
aux.LastModified = r.LastModified.UTC().Format(RFC3339TimeFormat)
return e.EncodeElement(aux, start)
}
@@ -399,71 +380,7 @@ type ListVersionsResult struct {
NextKeyMarker *string
NextVersionIdMarker *string
Prefix *string
RequestCharged types.RequestCharged
VersionIdMarker *string
Versions []types.ObjectVersion `xml:"Version"`
}
type GetBucketVersioningOutput struct {
MFADelete *types.MFADeleteStatus
Status *types.BucketVersioningStatus
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
}
type PutObjectRetentionInput struct {
RetainUntilDate AmzDate
XMLName xml.Name `xml:"Retention"`
Mode types.ObjectLockRetentionMode
}
type AmzDate struct {
time.Time
}
// Parses the date from xml string and validates for predefined date formats
func (d *AmzDate) UnmarshalXML(e *xml.Decoder, startElement xml.StartElement) error {
var dateStr string
err := e.DecodeElement(&dateStr, &startElement)
if err != nil {
return err
}
retDate, err := d.ISO8601Parse(dateStr)
if err != nil {
return s3err.GetAPIError(s3err.ErrInvalidRequest)
}
*d = AmzDate{retDate}
return nil
}
// Encodes expiration date if it is non-zero
// Encodes empty string if it's zero
func (d AmzDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if d.IsZero() {
return nil
}
return e.EncodeElement(d.UTC().Format(iso8601TimeFormat), startElement)
}
// Parses ISO8601 date string to time.Time by
// validating different time layouts
func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
for _, layout := range []string{
iso8601TimeFormat,
iso8601TimeFormatExtended,
iso8601TimeFormatWithTZ,
time.RFC3339,
} {
t, err = time.Parse(layout, date)
if err == nil {
return t, nil
}
}
return t, err
}
// Admin api response types
type ListBucketsResult struct {
Buckets []Bucket
}

View File

@@ -204,6 +204,7 @@ func genErrorMessage(errorCode, errorMessage string) []byte {
type GetProgress func() (bytesScanned int64, bytesProcessed int64)
type MessageHandler struct {
sync.Mutex
ctx context.Context
cancel context.CancelFunc
writer *bufio.Writer
@@ -212,7 +213,6 @@ type MessageHandler struct {
stopCh chan bool
resetCh chan bool
bytesReturned int64
sync.Mutex
}
// NewMessageHandler creates a new MessageHandler instance and starts the event streaming

View File

@@ -27,6 +27,3 @@ USERNAME_TWO=HIJKLMN
PASSWORD_TWO=OPQRSTU
TEST_FILE_FOLDER=$PWD/versity-gwtest-files
REMOVE_TEST_FILE_FOLDER=true
VERSIONING_DIR=/tmp/versioning
COMMAND_LOG=command.log
TIME_LOG=time.log

View File

@@ -47,7 +47,7 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
USER tester
RUN mkdir -p /home/tester/tests
COPY --chown=tester:tester . /home/tester
COPY --chown=tester:tester . /home/tester/tests
# add bats support libraries
RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests

View File

@@ -86,5 +86,4 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
ENV WORKSPACE=.
ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
ENTRYPOINT ["tests/run.sh"]
CMD ["s3api,s3,s3cmd,mc,rest"]
CMD ["tests/run_all.sh"]

View File

@@ -81,56 +81,4 @@ For the s3 backend, see the **S3 Backend** instructions above.
If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries.
A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`
## Environment Parameters
**AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**: identical to the same parameters in **s3**.
**VERSITY_EXE**: location of the versity executable relative to test folder.
**RUN_VERSITYGW**: whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
**BACKEND**: the storage backend type for the gateway, e.g. **posix** or **s3**.
**LOCAL_FOLDER**: if running with a **posix** backend, the backend storage folder.
**BUCKET_ONE_NAME**, **BUCKET_TWO_NAME**: test bucket names.
**RECREATE_BUCKETS**: whether to delete buckets between tests. If set to false, the bucket will be restored to an original state for the purpose of ensuring consistent tests, but not deleted.
**CERT**, **KEY**: certificate and key locations if using SSL.
**S3CMD_CONFIG**: location of **s3cmd** config file if running **s3cmd** tests.
**SECRETS_FILE**: file where sensitive values, such as **AWS_SECRET_ACCESS_KEY**, should be stored.
**MC_ALIAS**: Minio MC alias if running MC tests.
**LOG_LEVEL**: level for test logger (1 - only critical, 2 - errors, 3 - warnings, 4 - info, 5 - debug info, 6 - tracing)
**GOCOVERDIR**: folder to put golang coverage info in, if checking coverage info.
**USERS_FOLDER**: folder to use if storing IAM data in a folder.
**IAM_TYPE**: how to store IAM data (**s3** or **folder**).
**TEST_LOG_FILE**: log file location for these bats tests.
**VERSITY_LOG_FILE**: log file for versity application as it is tested by bats tests.
**DIRECT**: if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes).
**DIRECT_DISPLAY_NAME**: username if **DIRECT** is set to **true**.
**COVERAGE_DB**: database to store client command coverage info and usage counts, if using.
**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**: credentials for users created and tested for non-root user **versitygw** operations.
**TEST_FILE_FOLDER**: where to put temporary test files.
**REMOVE_TEST_FILE_FOLDER**: whether to delete the test file folder between tests, should be set to **true** unless checking the files after a single test, or not yet sure that the test folder is in a safe location to avoid deleting other files.
**VERSIONING_DIR**: where to put gateway file versioning info.
**COMMAND_LOG**: where to store list of client commands, which if using will be reported during test failures.
A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`

View File

@@ -20,7 +20,7 @@ abort_multipart_upload() {
log 2 "'abort multipart upload' command requires bucket, key, upload ID"
return 1
fi
if ! error=$(send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
log 2 "Error aborting upload: $error"
return 1
fi
@@ -33,10 +33,10 @@ abort_multipart_upload_with_user() {
return 1
fi
record_command "abort-multipart-upload" "client:s3api"
if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
log 2 "Error aborting upload: $abort_multipart_upload_error"
export abort_multipart_upload_error
return 1
fi
return 0
}
}

View File

@@ -1,34 +0,0 @@
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/logger.sh
send_command() {
if [ $# -eq 0 ]; then
return 1
fi
if [ -n "$COMMAND_LOG" ]; then
args=(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" "$@")
if ! mask_arg_array "${args[@]}"; then
return 1
fi
# shellcheck disable=SC2154
echo "${masked_args[*]}" >> "$COMMAND_LOG"
"$@"
return $?
fi
"$@"
}

View File

@@ -21,7 +21,7 @@ complete_multipart_upload() {
fi
log 5 "complete multipart upload id: $3, parts: $4"
record_command "complete-multipart-upload" "client:s3api"
error=$(send_command aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
if [[ $completed -ne 0 ]]; then
log 2 "error completing multipart upload: $error"
return 1

View File

@@ -23,14 +23,14 @@ copy_object() {
local error
record_command "copy-object" "client:$1"
if [[ $1 == 's3' ]]; then
error=$(send_command aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
error=$(send_command aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
error=$(aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
elif [[ $1 == 's3cmd' ]]; then
log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4"
error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
elif [[ $1 == 'mc' ]]; then
error=$(send_command mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
else
echo "'copy-object' not implemented for '$1'"
return 1
@@ -45,7 +45,7 @@ copy_object() {
copy_object_empty() {
record-command "copy-object" "client:s3api"
error=$(send_command aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
if [[ $result -eq 0 ]]; then
log 2 "copy object with empty parameters returned no error"
return 1

View File

@@ -30,14 +30,14 @@ create_bucket() {
local error
log 6 "create bucket"
if [[ $1 == 's3' ]]; then
error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
elif [[ $1 == "s3cmd" ]]; then
log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == "mc" ]]; then
error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
else
log 2 "invalid command type $1"
return 1
@@ -56,11 +56,11 @@ create_bucket_with_user() {
fi
local exit_code=0
if [[ $1 == "aws" ]] || [[ $1 == "s3api" ]]; then
error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == "s3cmd" ]]; then
error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == "mc" ]]; then
error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
else
log 2 "invalid command type $1"
return 1
@@ -80,7 +80,7 @@ create_bucket_object_lock_enabled() {
fi
local exit_code=0
error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
error=$(aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
if [ $exit_code -ne 0 ]; then
log 2 "error creating bucket: $error"
return 1

View File

@@ -24,7 +24,7 @@ create_multipart_upload() {
return 1
fi
if ! multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
log 2 "Error creating multipart upload: $multipart_data"
return 1
fi
@@ -44,7 +44,7 @@ create_multipart_upload_with_user() {
return 1
fi
if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
log 2 "Error creating multipart upload: $multipart_data"
return 1
fi
@@ -65,7 +65,7 @@ create_multipart_upload_params() {
return 1
fi
local multipart_data
multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload \
multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload \
--bucket "$1" \
--key "$2" \
--content-type "$3" \
@@ -96,7 +96,7 @@ create_multipart_upload_custom() {
done
log 5 "${*:3}"
log 5 "aws --no-verify-ssl s3api create-multipart-upload --bucket $1 --key $2 ${*:3}"
multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
if [[ $result -ne 0 ]]; then
log 2 "error creating custom multipart data command: $multipart_data"
return 1
@@ -107,19 +107,3 @@ create_multipart_upload_custom() {
log 5 "upload id: $upload_id"
return 0
}
create_multipart_upload_rest() {
if [ $# -ne 2 ]; then
log 2 "'create_multipart_upload_rest' requires bucket name, key"
return 1
fi
if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then
log 2 "error creating multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")"
return 1
fi
return 0
}

View File

@@ -1,27 +0,0 @@
#!/usr/bin/env bash
create_presigned_url() {
if [[ $# -ne 3 ]]; then
log 2 "create presigned url function requires command type, bucket, and filename"
return 1
fi
local presign_result=0
if [[ $1 == 'aws' ]]; then
presigned_url=$(send_command aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
elif [[ $1 == 's3cmd' ]]; then
presigned_url=$(send_command s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
elif [[ $1 == 'mc' ]]; then
presigned_url_data=$(send_command mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
presigned_url="${presigned_url_data#*Share: }"
else
log 2 "unrecognized command type $1"
return 1
fi
if [[ $presign_result -ne 0 ]]; then
log 2 "error generating presigned url: $presigned_url"
return 1
fi
export presigned_url
return 0
}

View File

@@ -31,13 +31,13 @@ delete_bucket() {
exit_code=0
if [[ $1 == 's3' ]]; then
error=$(send_command aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
error=$(aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
error=$(send_command aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
elif [[ $1 == 's3cmd' ]]; then
error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == 'mc' ]]; then
error=$(send_command mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
else
log 2 "Invalid command type $1"
return 1

View File

@@ -22,11 +22,11 @@ delete_bucket_policy() {
fi
local delete_result=0
if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
error=$(send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
elif [[ $1 == 's3cmd' ]]; then
error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
elif [[ $1 == 'mc' ]]; then
error=$(send_command mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
else
log 2 "command 'delete bucket policy' not implemented for '$1'"
return 1
@@ -44,7 +44,7 @@ delete_bucket_policy_with_user() {
log 2 "'delete bucket policy with user' command requires bucket, username, password"
return 1
fi
if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
log 2 "error deleting bucket policy: $delete_bucket_policy_error"
export delete_bucket_policy_error
return 1

View File

@@ -22,9 +22,9 @@ delete_bucket_tagging() {
fi
local result
if [[ $1 == 'aws' ]]; then
tags=$(send_command aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
elif [[ $1 == 'mc' ]]; then
tags=$(send_command mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
else
log 2 "invalid command type $1"
return 1
@@ -43,7 +43,7 @@ delete_bucket_tagging_with_user() {
log 2 "delete bucket tagging command missing username, password, bucket name"
return 1
fi
if ! error=$(send_command AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
if ! error=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
log 2 "error deleting bucket tagging with user: $error"
return 1
fi

View File

@@ -24,15 +24,13 @@ delete_object() {
fi
local exit_code=0
if [[ $1 == 's3' ]]; then
delete_object_error=$(send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
delete_object_error=$(aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
elif [[ $1 == 's3cmd' ]]; then
delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
elif [[ $1 == 'mc' ]]; then
delete_object_error=$(send_command mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
elif [[ $1 == 'rest' ]]; then
delete_object_rest "$2" "$3" || exit_code=$?
delete_object_error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
else
log 2 "invalid command type $1"
return 1
@@ -51,37 +49,13 @@ delete_object_bypass_retention() {
log 2 "'delete-object with bypass retention' requires bucket, key, user, password"
return 1
fi
if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
log 2 "error deleting object with bypass retention: $delete_object_error"
return 1
fi
return 0
}
delete_object_version() {
if [[ $# -ne 3 ]]; then
log 2 "'delete_object_version' requires bucket, key, version ID"
return 1
fi
if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then
log 2 "error deleting object version: $delete_object_error"
return 1
fi
return 0
}
delete_object_version_bypass_retention() {
if [[ $# -ne 3 ]]; then
log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID"
return 1
fi
if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then
log 2 "error deleting object version with bypass retention: $delete_object_error"
return 1
fi
return 0
}
delete_object_with_user() {
record_command "delete-object" "client:$1"
if [ $# -ne 5 ]; then
@@ -90,11 +64,11 @@ delete_object_with_user() {
fi
local exit_code=0
if [[ $1 == 's3' ]]; then
delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
elif [[ $1 == 's3cmd' ]]; then
delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
else
log 2 "command 'delete object with user' not implemented for '$1'"
return 1
@@ -105,44 +79,4 @@ delete_object_with_user() {
return 1
fi
return 0
}
delete_object_rest() {
if [ $# -ne 2 ]; then
log 2 "'delete_object_rest' requires bucket name, object name"
return 1
fi
generate_hash_for_payload ""
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
# shellcheck disable=SC2154
canonical_request="DELETE
/$1/$2
host:$aws_endpoint_url_address
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD"
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
log 2 "error generating sts string"
return 1
fi
get_signature
# shellcheck disable=SC2154
reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-H "x-amz-date: $current_date_time" \
-o "$TEST_FILE_FOLDER"/delete_object_error.txt 2>&1)
if [[ "$reply" != "204" ]]; then
log 2 "delete object command returned error: $(cat "$TEST_FILE_FOLDER"/delete_object_error.txt)"
return 1
fi
return 0
}

View File

@@ -20,13 +20,10 @@ delete_object_tagging() {
echo "delete object tagging command missing command type, bucket, key"
return 1
fi
delete_result=0
if [[ $1 == 'aws' ]]; then
error=$(send_command aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
error=$(aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
elif [[ $1 == 'mc' ]]; then
error=$(send_command mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
elif [ "$1" == 'rest' ]; then
delete_object_tagging_rest "$2" "$3" || delete_result=$?
error=$(mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
else
echo "delete-object-tagging command not implemented for '$1'"
return 1
@@ -36,46 +33,4 @@ delete_object_tagging() {
return 1
fi
return 0
}
delete_object_tagging_rest() {
if [ $# -ne 2 ]; then
log 2 "'delete_object_tagging' requires bucket, key"
return 1
fi
generate_hash_for_payload ""
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
# shellcheck disable=SC2154
canonical_request="DELETE
/$1/$2
tagging=
host:$aws_endpoint_url_address
x-amz-content-sha256:$payload_hash
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
$payload_hash"
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
log 2 "error generating sts string"
return 1
fi
get_signature
# shellcheck disable=SC2154
reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-H "x-amz-content-sha256: $payload_hash" \
-H "x-amz-date: $current_date_time" \
-d "$tagging" -o "$TEST_FILE_FOLDER"/delete_tagging_error.txt 2>&1)
log 5 "reply status code: $reply"
if [[ "$reply" != "204" ]]; then
log 2 "reply error: $reply"
log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/delete_tagging_error.txt)"
return 1
fi
return 0
}
}

View File

@@ -20,7 +20,7 @@ delete_objects() {
log 2 "'delete-objects' command requires bucket name, two object keys"
return 1
fi
if ! error=$(send_command aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
if ! error=$(aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
\"Objects\": [
{\"Key\": \"$2\"},
{\"Key\": \"$3\"}

View File

@@ -22,9 +22,9 @@ get_bucket_acl() {
fi
local exit_code=0
if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
acl=$(send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
acl=$(aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
elif [[ $1 == 's3cmd' ]]; then
acl=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
acl=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
else
log 2 "command 'get bucket acl' not implemented for $1"
return 1
@@ -42,7 +42,7 @@ get_bucket_acl_with_user() {
log 2 "'get bucket ACL with user' command requires bucket name, username, password"
return 1
fi
if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
log 2 "error getting bucket ACLs: $bucket_acl"
return 1
fi

View File

@@ -42,7 +42,7 @@ get_bucket_location_aws() {
echo "get bucket location (aws) requires bucket name"
return 1
fi
location_json=$(send_command aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
location_json=$(aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
if [[ $location_result -ne 0 ]]; then
echo "error getting bucket location: $location"
return 1
@@ -57,7 +57,7 @@ get_bucket_location_s3cmd() {
echo "get bucket location (s3cmd) requires bucket name"
return 1
fi
info=$(send_command s3cmd --no-check-certificate info "s3://$1") || results=$?
info=$(s3cmd --no-check-certificate info "s3://$1") || results=$?
if [[ $results -ne 0 ]]; then
echo "error getting s3cmd info: $info"
return 1
@@ -72,7 +72,7 @@ get_bucket_location_mc() {
echo "get bucket location (mc) requires bucket name"
return 1
fi
info=$(send_command mc --insecure stat "$MC_ALIAS/$1") || results=$?
info=$(mc --insecure stat "$MC_ALIAS/$1") || results=$?
if [[ $results -ne 0 ]]; then
echo "error getting s3cmd info: $info"
return 1

View File

@@ -26,7 +26,7 @@ get_bucket_ownership_controls() {
return 1
fi
if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
return 1
fi

View File

@@ -44,7 +44,7 @@ get_bucket_policy_aws() {
log 2 "aws 'get bucket policy' command requires bucket"
return 1
fi
policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
log 5 "$policy_json"
if [[ $get_result -ne 0 ]]; then
@@ -66,7 +66,7 @@ get_bucket_policy_with_user() {
log 2 "'get bucket policy with user' command requires bucket, username, password"
return 1
fi
if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
else
@@ -87,7 +87,7 @@ get_bucket_policy_s3cmd() {
return 1
fi
if ! info=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
if ! info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
log 2 "error getting bucket policy: $info"
return 1
fi
@@ -129,7 +129,7 @@ get_bucket_policy_mc() {
echo "aws 'get bucket policy' command requires bucket"
return 1
fi
bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
bucket_policy=$(mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
if [[ $get_result -ne 0 ]]; then
echo "error getting policy: $bucket_policy"
return 1

View File

@@ -22,9 +22,9 @@ get_bucket_tagging() {
record_command "get-bucket-tagging" "client:$1"
local result
if [[ $1 == 'aws' ]]; then
tags=$(send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
elif [[ $1 == 'mc' ]]; then
tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
else
fail "invalid command type $1"
fi
@@ -49,7 +49,7 @@ get_bucket_tagging_with_user() {
fi
record_command "get-bucket-tagging" "client:s3api"
local result
if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then
if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then
log 5 "tags error: $tags"
if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
export tags=

View File

@@ -17,32 +17,15 @@
get_bucket_versioning() {
record_command "get-bucket-versioning" "client:s3api"
if [[ $# -ne 2 ]]; then
log 2 "get bucket versioning command requires command type, bucket name"
log 2 "put bucket versioning command requires command type, bucket name"
return 1
fi
local get_result=0
if [[ $1 == 's3api' ]]; then
versioning=$(send_command aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
error=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
fi
if [[ $get_result -ne 0 ]]; then
log 2 "error getting bucket versioning: $versioning"
return 1
fi
return 0
}
get_bucket_versioning_rest() {
log 6 "get_object_rest"
if [ $# -ne 1 ]; then
log 2 "'get_bucket_versioning_rest' requires bucket name"
return 1
fi
if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OUTPUT_FILE="$TEST_FILE_FOLDER/versioning.txt" ./tests/rest_scripts/get_bucket_versioning.sh); then
log 2 "error getting bucket versioning: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "get-bucket-versioning returned code $result: $(cat "$TEST_FILE_FOLDER/versioning.txt")"
log 2 "error getting bucket versioning: $error"
return 1
fi
return 0

View File

@@ -23,15 +23,13 @@ get_object() {
fi
local exit_code=0
if [[ $1 == 's3' ]]; then
get_object_error=$(send_command aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
get_object_error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
elif [[ $1 == 's3cmd' ]]; then
get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
elif [[ $1 == 'mc' ]]; then
get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
elif [[ $1 == 'rest' ]]; then
get_object_rest "$2" "$3" "$4" || exit_code=$?
get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
else
log 2 "'get object' command not implemented for '$1'"
return 1
@@ -50,7 +48,8 @@ get_object_with_range() {
log 2 "'get object with range' requires bucket, key, range, outfile"
return 1
fi
if ! get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1); then
get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1) || local exit_code=$?
if [[ $exit_code -ne 0 ]]; then
log 2 "error getting object with range: $get_object_error"
return 1
fi
@@ -66,13 +65,13 @@ get_object_with_user() {
fi
local exit_code=0
if [[ $1 == 's3' ]] || [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
elif [[ $1 == "s3cmd" ]]; then
log 5 "s3cmd filename: $3"
get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
elif [[ $1 == "mc" ]]; then
log 5 "save location: $4"
get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
else
log 2 "'get_object_with_user' not implemented for client '$1'"
return 1
@@ -84,45 +83,3 @@ get_object_with_user() {
fi
return 0
}
get_object_rest() {
log 6 "get_object_rest"
if [ $# -ne 3 ]; then
log 2 "'get_object_rest' requires bucket name, object name, output file"
return 1
fi
generate_hash_for_payload ""
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
# shellcheck disable=SC2154
canonical_request="GET
/$1/$2
host:$aws_endpoint_url_address
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD"
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
log 2 "error generating sts string"
return 1
fi
get_signature
# shellcheck disable=SC2154
reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1/$2" \
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-H "x-amz-date: $current_date_time" \
-o "$3" 2>&1)
log 5 "reply: $reply"
if [[ "$reply" != "200" ]]; then
log 2 "get object command returned error: $(cat "$3")"
return 1
fi
return 0
}

View File

@@ -20,7 +20,7 @@ get_object_attributes() {
log 2 "'get object attributes' command requires bucket, key"
return 1
fi
attributes=$(send_command aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$?
attributes=$(aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$?
if [[ $get_result -ne 0 ]]; then
log 2 "error getting object attributes: $attributes"
return 1

View File

@@ -20,26 +20,10 @@ get_object_legal_hold() {
return 1
fi
record_command "get-object-legal-hold" "client:s3api"
legal_hold=$(send_command aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
legal_hold=$(aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
if [[ $get_result -ne 0 ]]; then
log 2 "error getting object legal hold: $legal_hold"
return 1
fi
return 0
}
get_object_legal_hold_rest() {
if [ $# -ne 2 ]; then
log 2 "'get_object_legal_hold_rest' requires bucket, key"
return 1
fi
if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/legal_hold.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then
log 2 "error getting object legal hold: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")"
return 1
fi
return 0
}

View File

@@ -20,7 +20,7 @@ get_object_lock_configuration() {
log 2 "'get object lock configuration' command missing bucket name"
return 1
fi
if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
if ! lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
log 2 "error obtaining lock config: $lock_config"
# shellcheck disable=SC2034
get_object_lock_config_err=$lock_config
@@ -28,44 +28,4 @@ get_object_lock_configuration() {
fi
lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning")
return 0
}
get_object_lock_configuration_rest() {
log 6 "get_object_lock_configuration_rest"
if [ $# -ne 1 ]; then
log 2 "'get_object_lock_configuration_rest' requires bucket name"
return 1
fi
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
# shellcheck disable=SC2154
canonical_request="GET
/$1
object-lock=
host:$aws_endpoint_url_address
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD"
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
log 2 "error generating sts string"
return 1
fi
get_signature
# shellcheck disable=SC2154
reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-H "x-amz-date: $current_date_time" \
-o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
log 5 "reply: $reply"
if [[ "$reply" != "200" ]]; then
log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
return 1
fi
return 0
}

View File

@@ -20,57 +20,11 @@ get_object_retention() {
log 2 "'get object retention' command requires bucket, key"
return 1
fi
if ! retention=$(send_command aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then
if ! retention=$(aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then
log 2 "error getting object retention: $retention"
get_object_retention_error=$retention
export get_object_retention_error
return 1
fi
return 0
}
get_object_retention_rest() {
if [ $# -ne 2 ]; then
log 2 "'get_object_tagging_rest' requires bucket, key"
return 1
fi
generate_hash_for_payload ""
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
# shellcheck disable=SC2154
canonical_request="GET
/$1/$2
retention=
host:$aws_endpoint_url_address
x-amz-content-sha256:$payload_hash
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
$payload_hash"
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
log 2 "error generating sts string"
return 1
fi
get_signature
# shellcheck disable=SC2154
reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?retention" \
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-H "x-amz-content-sha256: $payload_hash" \
-H "x-amz-date: $current_date_time" \
-o "$TEST_FILE_FOLDER"/object_retention.txt 2>&1)
log 5 "reply status code: $reply"
if [[ "$reply" != "200" ]]; then
if [ "$reply" == "404" ]; then
return 1
fi
log 2 "reply error: $reply"
log 2 "get object retention command returned error: $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
return 2
fi
log 5 "object tags: $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
return 0
}
}

Some files were not shown because too many files have changed in this diff Show More