fix: list objects trim common prefixes that match marker prefix

This checks to see if the common prefix is before the marker and
thus would have been returned in earlier list objects request.

The error case was aws cli listing multiple entries for the same
common prefix when the listing required multiple pagination
requests.

Fixes #778

.github/workflows/azurite.yml

@@ -1,37 +0,0 @@
-name: azurite functional tests
-
-on: pull_request
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-              
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 'stable'
-        id: go
-
-      - name: Set up Docker Compose
-        run: |
-          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
-
-      - name: Wait for Azurite to be ready
-        run: sleep 40
-
-      - name: Get Dependencies
-        run: |
-          go mod download
-  
-      - name: Build and Run
-        run: |
-          make
-          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
-  
-      - name: Shut down services
-        run: |
-          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/docker-bats.yaml

@@ -0,0 +1,28 @@
+name: docker bats tests
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker Image
+        run: |
+          mv tests/.env.docker.default tests/.env.docker
+          mv tests/.secrets.default tests/.secrets
+          docker build --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain -f tests/Dockerfile_test_bats -t bats_test .
+
+      - name: Set up Docker Compose
+        run: sudo apt-get install -y docker-compose
+
+      - name: Run Docker Container
+        run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from posix_backend posix_backend

.github/workflows/docker-bats.yml

@@ -1,28 +0,0 @@
-name: docker bats tests
-
-on: pull_request
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Build Docker Image
-        run: |
-          cp tests/.env.docker.default tests/.env.docker
-          cp tests/.secrets.default tests/.secrets
-          docker build \
-            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
-            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
-            --build-arg="MC_FOLDER=linux-amd64" \
-            --progress=plain \
-            -f tests/Dockerfile_test_bats \
-            -t bats_test .
-
-      - name: Run Docker Container
-        run: |
-          docker compose -f tests/docker-compose-bats.yml --project-directory . \
-          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/functional.yml

@@ -1,8 +1,7 @@
 name: functional tests
-
 on: pull_request
-
 jobs:
+
   build:
     name: RunTests
     runs-on: ubuntu-latest
@@ -19,7 +18,7 @@ jobs:
 
     - name: Get Dependencies
       run: |
-        go mod download
+        go get -v -t -d ./...
 
     - name: Build and Run
       run: |

.github/workflows/system.yml

@@ -8,95 +8,115 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - set: "mc, posix, non-file count, non-static, folder IAM"
+          - set: "s3cmd, posix"
+            LOCAL_FOLDER: /tmp/gw1
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
             IAM_TYPE: folder
-            RUN_SET: "mc-non-file-count"
+            USERS_FOLDER: /tmp/iam1
+            AWS_ENDPOINT_URL: https://127.0.0.1:7070
+            RUN_SET: "s3cmd"
             RECREATE_BUCKETS: "true"
+            PORT: 7070
             BACKEND: "posix"
-          - set: "mc, posix, file count, non-static, folder IAM"
+          - set: "s3, posix"
+            LOCAL_FOLDER: /tmp/gw2
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
             IAM_TYPE: folder
-            RUN_SET: "mc-file-count"
+            USERS_FOLDER: /tmp/iam2
+            AWS_ENDPOINT_URL: https://127.0.0.1:7071
+            RUN_SET: "s3"
             RECREATE_BUCKETS: "true"
+            PORT: 7071
             BACKEND: "posix"
-          - set: "REST, posix, non-static, all, folder IAM"
+          - set: "s3api, posix"
+            LOCAL_FOLDER: /tmp/gw3
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
             IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam3
+            AWS_ENDPOINT_URL: https://127.0.0.1:7072
+            RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
+            PORT: 7072
+            BACKEND: "posix"
+          - set: "mc, posix"
+            LOCAL_FOLDER: /tmp/gw4
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam4
+            AWS_ENDPOINT_URL: https://127.0.0.1:7073
+            RUN_SET: "mc"
+            RECREATE_BUCKETS: "true"
+            PORT: 7073
+            BACKEND: "posix"
+          - set: "s3api-user, posix, s3 IAM"
+            LOCAL_FOLDER: /tmp/gw5
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
+            IAM_TYPE: s3
+            USERS_BUCKET: versity-gwtest-iam
+            AWS_ENDPOINT_URL: https://127.0.0.1:7074
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "true"
+            PORT: 7074
+            BACKEND: "posix"
+          - set: "s3api non-policy, static buckets"
+            LOCAL_FOLDER: /tmp/gw6
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam6
+            AWS_ENDPOINT_URL: https://127.0.0.1:7075
+            RUN_SET: "s3api-non-policy"
+            RECREATE_BUCKETS: "false"
+            PORT: 7075
+            BACKEND: "posix"
+          - set: "s3api, s3 backend"
+            LOCAL_FOLDER: /tmp/gw7
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam7
+            AWS_ENDPOINT_URL: https://127.0.0.1:7076
+            RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
+            PORT: 7076
+            BACKEND: "s3"
+          - set: "REST, posix"
+            LOCAL_FOLDER: /tmp/gw8
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam8
+            AWS_ENDPOINT_URL: https://127.0.0.1:7077
             RUN_SET: "rest"
             RECREATE_BUCKETS: "true"
+            PORT: 7077
             BACKEND: "posix"
-          - set: "s3, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-non-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, user, non-static, s3 IAM"
-            IAM_TYPE: s3
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, multipart, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-multipart"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, object, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-object"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, static, folder IAM"
+          - set: "s3api policy, static buckets"
+            LOCAL_FOLDER: /tmp/gw9
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-8
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-8
             IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam9
+            AWS_ENDPOINT_URL: https://127.0.0.1:7078
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "false"
+            PORT: 7078
             BACKEND: "posix"
-          - set: "s3api, posix, user, static, folder IAM"
+          - set: "s3api user, static buckets"
+            LOCAL_FOLDER: /tmp/gw10
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-9
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-9
             IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam10
+            AWS_ENDPOINT_URL: https://127.0.0.1:7079
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, s3, multipart|object, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
-          - set: "s3api, s3, policy|user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy,s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
-          - set: "s3cmd, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, non-user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-non-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-user"
-            RECREATE_BUCKETS: "true"
+            PORT: 7079
             BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
@@ -132,10 +152,17 @@ jobs:
         run: |
           sudo apt-get install libxml2-utils
 
-      - name: Build and run
+      - name: Build and run, posix backend
         env:
+          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
+          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
+          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
+          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
+          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
           IAM_TYPE: ${{ matrix.IAM_TYPE }}
+          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
           RUN_SET: ${{ matrix.RUN_SET }}
+          PORT: ${{ matrix.PORT }}
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
@@ -143,13 +170,6 @@ jobs:
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
-          LOCAL_FOLDER: /tmp/gw
-          BUCKET_ONE_NAME: versity-gwtest-bucket-one
-          BUCKET_TWO_NAME: versity-gwtest-bucket-two
-          USERS_FOLDER: /tmp/iam
-          USERS_BUCKET: versity-gwtest-iam
-          AWS_ENDPOINT_URL: https://127.0.0.1:7070
-          PORT: 7070
           S3CMD_CONFIG: tests/s3cfg.local.default
           MC_ALIAS: versity
           LOG_LEVEL: 4
@@ -159,10 +179,6 @@ jobs:
           USERNAME_TWO: HIJKLMN
           PASSWORD_TWO: 8901234
           TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
-          REMOVE_TEST_FILE_FOLDER: true
-          VERSIONING_DIR: ${{ github.workspace }}/versioning
-          COMMAND_LOG: command.log
-          TIME_LOG: time.log
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -183,9 +199,6 @@ jobs:
           fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
-      - name: Time report
-        run: cat ${{ github.workspace }}/time.log
-
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

.gitignore

@@ -62,8 +62,4 @@ tests/!s3cfg.local.default
 *.patch
 
 # grafana's local database (kept on filesystem for survival between instantiations)
-metrics-exploration/grafana_data/**
-
-# bats tools
-/tests/bats-assert
-/tests/bats-support
+metrics-exploration/grafana_data/**

Makefile

@@ -18,10 +18,6 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 
-# docker-compose
-DCCMD=docker-compose
-DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
-
 BIN=versitygw
 
 VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -75,19 +71,19 @@ dist:
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
-	$(DOCKERCOMPOSE) up posix
+	docker compose --env-file .env.dev up posix
 
 # Creates and runs S3 gateway proxy instance in a docker container
 .PHONY: up-proxy
 up-proxy:
-	$(DOCKERCOMPOSE) up proxy
+	docker compose --env-file .env.dev up proxy
 
 # Creates and runs S3 gateway to azurite instance in a docker container
 .PHONY: up-azurite
 up-azurite:
-	$(DOCKERCOMPOSE) up azurite azuritegw
+	docker compose --env-file .env.dev up azurite azuritegw
 
 # Creates and runs both S3 gateway and proxy server instances in docker containers
 .PHONY: up-app
 up-app:
-	$(DOCKERCOMPOSE) up
+	docker compose --env-file .env.dev up

auth/iam.go

@@ -28,19 +28,6 @@ const (
 	RoleUserPlus Role = "userplus"
 )
 
-func (r Role) IsValid() bool {
-	switch r {
-	case RoleAdmin:
-		return true
-	case RoleUser:
-		return true
-	case RoleUserPlus:
-		return true
-	default:
-		return false
-	}
-}
-
 // Account is a gateway IAM account
 type Account struct {
 	Access  string `json:"access"`
@@ -50,10 +37,6 @@ type Account struct {
 	GroupID int    `json:"groupID"`
 }
 
-type ListUserAccountsResult struct {
-	Accounts []Account
-}
-
 // Mutable props, which could be changed when updating an IAM account
 type MutableProps struct {
 	Secret  *string `json:"secret"`

auth/object_lock.go

@@ -25,7 +25,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3response"
 )
 
 type BucketLockConfig struct {
@@ -93,12 +92,12 @@ func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfigur
 }
 
 func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
-	var retention s3response.PutObjectRetentionInput
+	var retention types.ObjectLockRetention
 	if err := xml.Unmarshal(input, &retention); err != nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
-	if retention.RetainUntilDate.Before(time.Now()) {
+	if retention.RetainUntilDate == nil || retention.RetainUntilDate.Before(time.Now()) {
 		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
 	}
 	switch retention.Mode {
@@ -136,7 +135,7 @@ func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
 	}
 }
 
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, bypass bool, be backend.Backend) error {
 	data, err := be.GetObjectLockConfiguration(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -172,15 +171,8 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 	}
 
 	for _, obj := range objects {
-		var key, versionId string
-		if obj.Key != nil {
-			key = *obj.Key
-		}
-		if obj.VersionId != nil {
-			versionId = *obj.VersionId
-		}
 		checkRetention := true
-		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
+		retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
 			continue
 		}
@@ -211,7 +203,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 							if err != nil {
 								return err
 							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+							err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
 							if err != nil {
 								return s3err.GetAPIError(s3err.ErrObjectLocked)
 							}
@@ -225,11 +217,8 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 
 		checkLegalHold := true
 
-		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
+		status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
 		if err != nil {
-			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
-				continue
-			}
 			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
 				checkLegalHold = false
 			} else {
@@ -254,7 +243,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 					if err != nil {
 						return err
 					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+					err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
 					if err != nil {
 						return s3err.GetAPIError(s3err.ErrObjectLocked)
 					}

backend/azure/azure.go

@@ -149,8 +149,8 @@ func (az *Azure) String() string {
 
 func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
 	meta := map[string]*string{
-		string(keyAclCapital): backend.GetPtrFromString(encodeBytes(acl)),
-		string(keyOwnership):  backend.GetPtrFromString(encodeBytes([]byte(input.ObjectOwnership))),
+		string(keyAclCapital): backend.GetStringPtr(encodeBytes(acl)),
+		string(keyOwnership):  backend.GetStringPtr(encodeBytes([]byte(input.ObjectOwnership))),
 	}
 
 	acct, ok := ctx.Value("account").(auth.Account)
@@ -170,7 +170,7 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 			return fmt.Errorf("parse default bucket lock state: %w", err)
 		}
 
-		meta[string(keyBucketLock)] = backend.GetPtrFromString(encodeBytes(defaultLockParsed))
+		meta[string(keyBucketLock)] = backend.GetStringPtr(encodeBytes(defaultLockParsed))
 	}
 
 	_, err := az.client.CreateContainer(ctx, *input.Bucket, &container.CreateOptions{Metadata: meta})
@@ -183,67 +183,58 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 		var acl auth.ACL
 		if len(aclBytes) > 0 {
 			if err := json.Unmarshal(aclBytes, &acl); err != nil {
-				return fmt.Errorf("unmarshal acl: %w", err)
+				return fmt.Errorf("unmarshal bucket acl: %w", err)
 			}
 		}
-
 		if acl.Owner == acct.Access {
 			return s3err.GetAPIError(s3err.ErrBucketAlreadyOwnedByYou)
 		}
-		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
 	}
 	return azureErrToS3Err(err)
 }
 
-func (az *Azure) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
 	pager := az.client.NewListContainersPager(
 		&service.ListContainersOptions{
 			Include: service.ListContainersInclude{
 				Metadata: true,
 			},
-			Marker:     &input.ContinuationToken,
-			MaxResults: &input.MaxBuckets,
-			Prefix:     &input.Prefix,
 		})
 
 	var buckets []s3response.ListAllMyBucketsEntry
-	result := s3response.ListAllMyBucketsResult{
-		Prefix: input.Prefix,
-	}
+	var result s3response.ListAllMyBucketsResult
 
-	resp, err := pager.NextPage(ctx)
-	if err != nil {
-		return result, azureErrToS3Err(err)
-	}
-	for _, v := range resp.ContainerItems {
-		if input.IsAdmin {
-			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-				Name: *v.Name,
-				// TODO: using modification date here instead of creation, is that ok?
-				CreationDate: *v.Properties.LastModified,
-			})
-		} else {
-			acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
-			if err != nil {
-				return result, err
-			}
-
-			if acl.Owner == input.Owner {
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return result, azureErrToS3Err(err)
+		}
+		for _, v := range resp.ContainerItems {
+			if isAdmin {
 				buckets = append(buckets, s3response.ListAllMyBucketsEntry{
 					Name: *v.Name,
 					// TODO: using modification date here instead of creation, is that ok?
 					CreationDate: *v.Properties.LastModified,
 				})
+			} else {
+				acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
+				if err != nil {
+					return result, err
+				}
+
+				if acl.Owner == owner {
+					buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+						Name: *v.Name,
+						// TODO: using modification date here instead of creation, is that ok?
+						CreationDate: *v.Properties.LastModified,
+					})
+				}
 			}
 		}
 	}
 
-	if resp.NextMarker != nil {
-		result.ContinuationToken = *resp.NextMarker
-	}
-
 	result.Buckets.Bucket = buckets
-	result.Owner.ID = input.Owner
+	result.Owner.ID = owner
 
 	return result, nil
 }
@@ -257,8 +248,8 @@ func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3
 	return &s3.HeadBucketOutput{}, nil
 }
 
-func (az *Azure) DeleteBucket(ctx context.Context, bucket string) error {
-	pager := az.client.NewListBlobsFlatPager(bucket, nil)
+func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	pager := az.client.NewListBlobsFlatPager(*input.Bucket, nil)
 
 	pg, err := pager.NextPage(ctx)
 	if err != nil {
@@ -268,7 +259,7 @@ func (az *Azure) DeleteBucket(ctx context.Context, bucket string) error {
 	if len(pg.Segment.BlobItems) > 0 {
 		return s3err.GetAPIError(s3err.ErrBucketNotEmpty)
 	}
-	_, err = az.client.DeleteContainer(ctx, bucket, nil)
+	_, err = az.client.DeleteContainer(ctx, *input.Bucket, nil)
 	return azureErrToS3Err(err)
 }
 
@@ -310,13 +301,13 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respon
 	opts.HTTPHeaders.BlobContentDisposition = po.ContentDisposition
 	if strings.HasSuffix(*po.Key, "/") {
 		// Hardcode "application/x-directory" for direcoty objects
-		opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DirContentType)
+		opts.HTTPHeaders.BlobContentType = backend.GetStringPtr(backend.DirContentType)
 	} else {
 		opts.HTTPHeaders.BlobContentType = po.ContentType
 	}
 
 	if opts.HTTPHeaders.BlobContentType == nil {
-		opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DefaultContentType)
+		opts.HTTPHeaders.BlobContentType = backend.GetStringPtr(backend.DefaultContentType)
 	}
 
 	uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts)
@@ -415,7 +406,7 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G
 
 	contentType := blobDownloadResponse.ContentType
 	if contentType == nil {
-		contentType = backend.GetPtrFromString(backend.DefaultContentType)
+		contentType = backend.GetStringPtr(backend.DefaultContentType)
 	}
 
 	return &s3.GetObjectOutput{
@@ -511,22 +502,20 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 	return result, nil
 }
 
-func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 	data, err := az.HeadObject(ctx, &s3.HeadObjectInput{
 		Bucket: input.Bucket,
 		Key:    input.Key,
 	})
 	if err != nil {
-		return s3response.GetObjectAttributesResponse{}, err
+		return s3response.GetObjectAttributesResult{}, err
 	}
 
-	return s3response.GetObjectAttributesResponse{
+	return s3response.GetObjectAttributesResult{
 		ETag:         data.ETag,
+		LastModified: data.LastModified,
 		ObjectSize:   data.ContentLength,
 		StorageClass: data.StorageClass,
-		LastModified: data.LastModified,
-		VersionId:    data.VersionId,
-		DeleteMarker: data.DeleteMarker,
 	}, nil
 }
 
@@ -580,28 +569,23 @@ Pager:
 				isTruncated = true
 				break Pager
 			}
-
-			marker := getString(input.Marker)
-			pfx := strings.TrimSuffix(*v.Name, getString(input.Delimiter))
-			if marker != "" && strings.HasPrefix(marker, pfx) {
-				continue
-			}
-
 			cPrefixes = append(cPrefixes, types.CommonPrefix{
 				Prefix: v.Name,
 			})
 		}
 	}
 
+	// TODO: generate common prefixes when appropriate
+
 	return s3response.ListObjectsResult{
 		Contents:       objects,
-		Marker:         backend.GetPtrFromString(*input.Marker),
+		Marker:         input.Marker,
 		MaxKeys:        input.MaxKeys,
 		Name:           input.Bucket,
 		NextMarker:     nextMarker,
-		Prefix:         backend.GetPtrFromString(*input.Prefix),
+		Prefix:         input.Prefix,
 		IsTruncated:    &isTruncated,
-		Delimiter:      backend.GetPtrFromString(*input.Delimiter),
+		Delimiter:      input.Delimiter,
 		CommonPrefixes: cPrefixes,
 	}, nil
 }
@@ -662,13 +646,6 @@ Pager:
 				isTruncated = true
 				break Pager
 			}
-
-			marker := getString(input.ContinuationToken)
-			pfx := strings.TrimSuffix(*v.Name, getString(input.Delimiter))
-			if marker != "" && strings.HasPrefix(marker, pfx) {
-				continue
-			}
-
 			cPrefixes = append(cPrefixes, types.CommonPrefix{
 				Prefix: v.Name,
 			})
@@ -677,15 +654,14 @@ Pager:
 
 	return s3response.ListObjectsV2Result{
 		Contents:              objects,
-		ContinuationToken:     backend.GetPtrFromString(*input.ContinuationToken),
+		ContinuationToken:     input.ContinuationToken,
 		MaxKeys:               input.MaxKeys,
 		Name:                  input.Bucket,
 		NextContinuationToken: nextMarker,
-		Prefix:                backend.GetPtrFromString(*input.Prefix),
+		Prefix:                input.Prefix,
 		IsTruncated:           &isTruncated,
-		Delimiter:             backend.GetPtrFromString(*input.Delimiter),
+		Delimiter:             input.Delimiter,
 		CommonPrefixes:        cPrefixes,
-		StartAfter:            backend.GetPtrFromString(*input.StartAfter),
 	}, nil
 }
 
@@ -721,8 +697,8 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
 			} else {
 				errs = append(errs, types.Error{
 					Key:     obj.Key,
-					Code:    backend.GetPtrFromString("InternalError"),
-					Message: backend.GetPtrFromString(err.Error()),
+					Code:    backend.GetStringPtr("InternalError"),
+					Message: backend.GetStringPtr(err.Error()),
 				})
 			}
 		}
@@ -858,7 +834,7 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 
 	// set blob legal hold status in metadata
 	if input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
-		meta[string(keyObjLegalHold)] = backend.GetPtrFromString("1")
+		meta[string(keyObjLegalHold)] = backend.GetStringPtr("1")
 	}
 
 	// set blob retention date
@@ -871,7 +847,7 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 		if err != nil {
 			return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err)
 		}
-		meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retParsed))
+		meta[string(keyObjRetention)] = backend.GetStringPtr(string(retParsed))
 	}
 
 	uploadId := uuid.New().String()
@@ -1045,8 +1021,7 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa
 	prefix := string(metaTmpMultipartPrefix)
 
 	pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-		Include: container.ListBlobsInclude{Metadata: true},
-		Prefix:  &prefix,
+		Prefix: &prefix,
 	})
 
 	for pager.More() {
@@ -1310,9 +1285,22 @@ func (az *Azure) GetObjectLockConfiguration(ctx context.Context, bucket string)
 }
 
 func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	err := az.isBucketObjectLockEnabled(ctx, bucket)
+	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
 	if err != nil {
-		return err
+		return azureErrToS3Err(err)
+	}
+
+	if len(cfg) == 0 {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	var bucketLockConfig auth.BucketLockConfig
+	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse bucket lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
 	}
 
 	blobClient, err := az.getBlobClient(bucket, object)
@@ -1328,12 +1316,12 @@ func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, version
 	meta := blobProps.Metadata
 	if meta == nil {
 		meta = map[string]*string{
-			string(keyObjRetention): backend.GetPtrFromString(string(retention)),
+			string(keyObjRetention): backend.GetStringPtr(string(retention)),
 		}
 	} else {
 		objLockCfg, ok := meta[string(keyObjRetention)]
 		if !ok {
-			meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retention))
+			meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
 		} else {
 			var lockCfg types.ObjectLockRetention
 			if err := json.Unmarshal([]byte(*objLockCfg), &lockCfg); err != nil {
@@ -1351,7 +1339,7 @@ func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, version
 				}
 			}
 
-			meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retention))
+			meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
 		}
 	}
 
@@ -1373,11 +1361,6 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
 		return nil, azureErrToS3Err(err)
 	}
 
-	err = az.isBucketObjectLockEnabled(ctx, bucket)
-	if err != nil {
-		return nil, err
-	}
-
 	retentionPtr, ok := props.Metadata[string(keyObjRetention)]
 	if !ok {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1387,9 +1370,22 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
 }
 
 func (az *Azure) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
-	err := az.isBucketObjectLockEnabled(ctx, bucket)
+	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
 	if err != nil {
-		return err
+		return azureErrToS3Err(err)
+	}
+
+	if len(cfg) == 0 {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	var bucketLockConfig auth.BucketLockConfig
+	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse bucket lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
 	}
 
 	blobClient, err := az.getBlobClient(bucket, object)
@@ -1436,11 +1432,6 @@ func (az *Azure) GetObjectLegalHold(ctx context.Context, bucket, object, version
 		return nil, azureErrToS3Err(err)
 	}
 
-	err = az.isBucketObjectLockEnabled(ctx, bucket)
-	if err != nil {
-		return nil, err
-	}
-
 	retentionPtr, ok := props.Metadata[string(keyObjLegalHold)]
 	if !ok {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1458,10 +1449,7 @@ func (az *Azure) ChangeBucketOwner(ctx context.Context, bucket string, acl []byt
 // The action actually returns the containers owned by the user, who initialized the gateway
 // TODO: Not sure if there's a way to list all the containers and owners?
 func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
-	opts := &service.ListContainersOptions{
-		Include: service.ListContainersInclude{Metadata: true},
-	}
-	pager := az.client.NewListContainersPager(opts)
+	pager := az.client.NewListContainersPager(nil)
 
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
@@ -1483,28 +1471,6 @@ func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response
 	return buckets, nil
 }
 
-func (az *Azure) isBucketObjectLockEnabled(ctx context.Context, bucket string) error {
-	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
-	if err != nil {
-		return azureErrToS3Err(err)
-	}
-
-	if len(cfg) == 0 {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
-	}
-
-	var bucketLockConfig auth.BucketLockConfig
-	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse bucket lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
-	}
-
-	return nil
-}
-
 func (az *Azure) getContainerURL(cntr string) string {
 	return fmt.Sprintf("%v/%v", strings.TrimRight(az.serviceURL, "/"), cntr)
 }
@@ -1702,7 +1668,7 @@ func (az *Azure) setContainerMetaData(ctx context.Context, bucket, key string, v
 	}
 
 	str := encodeBytes(value)
-	mdmap[key] = backend.GetPtrFromString(str)
+	mdmap[key] = backend.GetStringPtr(str)
 
 	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: mdmap})
 	if err != nil {
@@ -1737,11 +1703,9 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string
 }
 
 func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
-	var acl auth.ACL
-
 	data, ok := meta[string(key)]
 	if !ok {
-		return &acl, nil
+		return nil, s3err.GetAPIError(s3err.ErrInternalError)
 	}
 
 	value, err := decodeString(*data)
@@ -1749,6 +1713,7 @@ func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 		return nil, err
 	}
 
+	var acl auth.ACL
 	if len(value) == 0 {
 		return &acl, nil
 	}

backend/backend.go

@@ -32,14 +32,14 @@ type Backend interface {
 	Shutdown()
 
 	// bucket operations
-	ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
 	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
 	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
-	DeleteBucket(_ context.Context, bucket string) error
+	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
 	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
-	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
+	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
@@ -61,7 +61,7 @@ type Backend interface {
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
 	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
 	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
@@ -108,7 +108,7 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -123,14 +123,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
 func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
-	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -185,8 +185,8 @@ func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.Ge
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
-	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+	return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)

backend/common.go

@@ -50,18 +50,8 @@ func (d ByObjectName) Len() int           { return len(d) }
 func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
 
-func GetPtrFromString(str string) *string {
-	if str == "" {
-		return nil
-	}
-	return &str
-}
-
-func GetStringFromPtr(str *string) string {
-	if str == nil {
-		return ""
-	}
-	return *str
+func GetStringPtr(s string) *string {
+	return &s
 }
 
 func GetTimePtr(t time.Time) *time.Time {
@@ -105,7 +95,7 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 		return 0, 0, errInvalidRange
 	}
 
-	if endOffset < startOffset {
+	if endOffset <= startOffset {
 		return 0, 0, errInvalidRange
 	}
 
@@ -119,13 +109,15 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
 		copySourceHeader = copySourceHeader[1:]
 	}
 
-	var copySource, versionId string
-	i := strings.LastIndex(copySourceHeader, "?versionId=")
-	if i == -1 {
-		copySource = copySourceHeader
-	} else {
-		copySource = copySourceHeader[:i]
-		versionId = copySourceHeader[i+11:]
+	cSplitted := strings.Split(copySourceHeader, "?")
+	copySource := cSplitted[0]
+	var versionId string
+	if len(cSplitted) > 1 {
+		versionIdParts := strings.Split(cSplitted[1], "=")
+		if len(versionIdParts) != 2 || versionIdParts[0] != "versionId" {
+			return "", "", "", s3err.GetAPIError(s3err.ErrInvalidRequest)
+		}
+		versionId = versionIdParts[1]
 	}
 
 	srcBucket, srcObject, ok := strings.Cut(copySource, "/")

backend/meta/meta.go

@@ -14,19 +14,17 @@
 
 package meta
 
-import "os"
-
 // MetadataStorer defines the interface for managing metadata.
 // When object == "", the operation is on the bucket.
 type MetadataStorer interface {
 	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
 	// Returns the value of the attribute, or an error if the attribute does not exist.
-	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
+	RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
 
 	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
 	// If attribute already exists, new attribute should replace existing.
 	// Returns an error if the operation fails.
-	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
+	StoreAttribute(bucket, object, attribute string, value []byte) error
 
 	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
 	// Returns an error if the operation fails.

backend/meta/xattr.go

@@ -17,7 +17,6 @@ package meta
 import (
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
@@ -37,15 +36,7 @@ var (
 type XattrMeta struct{}
 
 // RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
-	if f != nil {
-		b, err := xattr.FGet(f, xattrPrefix+attribute)
-		if errors.Is(err, xattr.ENOATTR) {
-			return nil, ErrNoSuchKey
-		}
-		return b, err
-	}
-
+func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
 	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
 	if errors.Is(err, xattr.ENOATTR) {
 		return nil, ErrNoSuchKey
@@ -54,11 +45,7 @@ func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute strin
 }
 
 // StoreAttribute stores the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
-	if f != nil {
-		return xattr.FSet(f, xattrPrefix+attribute, value)
-	}
-
+func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
 	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
 }
 

backend/mkdir.go

@@ -15,6 +15,11 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
+var (
+	// TODO: make this configurable
+	defaultDirPerm fs.FileMode = 0755
+)
+
 // MkdirAll is similar to os.MkdirAll but it will return
 // ErrObjectParentIsFile when appropriate
 // MkdirAll creates a directory named path,
@@ -27,7 +32,7 @@ import (
 // and returns nil.
 // Any directory created will be set to provided uid/gid ownership
 // if doChown is true.
-func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
+func MkdirAll(path string, uid, gid int, doChown bool) error {
 	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
 	dir, err := os.Stat(path)
 	if err == nil {
@@ -50,14 +55,14 @@ func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) erro
 
 	if j > 1 {
 		// Create parent.
-		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
+		err = MkdirAll(path[:j-1], uid, gid, doChown)
 		if err != nil {
 			return err
 		}
 	}
 
 	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, dirPerm)
+	err = os.Mkdir(path, defaultDirPerm)
 	if err != nil {
 		// Handle arguments like "foo/." by
 		// double-checking that directory doesn't exist.

backend/posix/with_otmpfile.go

@@ -43,7 +43,6 @@ type tmpfile struct {
 	needsChown bool
 	uid        int
 	gid        int
-	newDirPerm fs.FileMode
 }
 
 var (
@@ -63,7 +62,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -109,7 +108,6 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
-		newDirPerm: p.newDirPerm,
 	}
 
 	// falloc is best effort, its fine if this fails
@@ -136,9 +134,6 @@ func (tmp *tmpfile) falloc() error {
 }
 
 func (tmp *tmpfile) link() error {
-	// make sure this is cleaned up in all error cases
-	defer tmp.f.Close()
-
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
 	// with any other simultaneous uploads. The final operation is to move the
@@ -153,7 +148,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -175,21 +170,11 @@ func (tmp *tmpfile) link() error {
 	}
 	defer dirf.Close()
 
-	for {
-		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-		if errors.Is(err, syscall.EEXIST) {
-			err := os.Remove(objPath)
-			if err != nil && !errors.Is(err, fs.ErrNotExist) {
-				return fmt.Errorf("remove stale path: %w", err)
-			}
-			continue
-		}
-		if err != nil {
-			return fmt.Errorf("link tmpfile (fd %q as %q): %w",
-				filepath.Base(tmp.f.Name()), objPath, err)
-		}
-		break
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if err != nil {
+		return fmt.Errorf("link tmpfile (%q in %q): %w",
+			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
 	}
 
 	err = tmp.f.Close()

backend/posix/without_otmpfile.go

@@ -41,7 +41,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 
 	// Create a temp file for upload while in progress (see link comments below).
 	var err error
-	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	err = backend.MkdirAll(dir, uid, gid, doChown)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}

backend/s3proxy/s3.go

@@ -75,12 +75,8 @@ func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify
 	return s, nil
 }
 
-func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
-	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{
-		ContinuationToken: &input.ContinuationToken,
-		MaxBuckets:        &input.MaxBuckets,
-		Prefix:            &input.Prefix,
-	})
+func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
 	if err != nil {
 		return s3response.ListAllMyBucketsResult{}, handleError(err)
 	}
@@ -100,8 +96,6 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsI
 		Buckets: s3response.ListAllMyBucketsList{
 			Bucket: buckets,
 		},
-		ContinuationToken: backend.GetStringFromPtr(output.ContinuationToken),
-		Prefix:            backend.GetStringFromPtr(output.Prefix),
 	}, nil
 }
 
@@ -118,8 +112,8 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 
 	var tagSet []types.Tag
 	tagSet = append(tagSet, types.Tag{
-		Key:   backend.GetPtrFromString(aclKey),
-		Value: backend.GetPtrFromString(base64Encode(acl)),
+		Key:   backend.GetStringPtr(aclKey),
+		Value: backend.GetStringPtr(base64Encode(acl)),
 	})
 
 	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
@@ -131,10 +125,8 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 	return handleError(err)
 }
 
-func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
-	_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
-		Bucket: &bucket,
-	})
+func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	_, err := s.client.DeleteBucket(ctx, input)
 	return handleError(err)
 }
 
@@ -180,15 +172,12 @@ func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status
 	return handleError(err)
 }
 
-func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
 	out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
 		Bucket: &bucket,
 	})
 
-	return s3response.GetBucketVersioningOutput{
-		Status:    &out.Status,
-		MFADelete: &out.MFADelete,
-	}, handleError(err)
+	return out, handleError(err)
 }
 
 func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
@@ -392,7 +381,7 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
 	return output, nil
 }
 
-func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 	out, err := s.client.GetObjectAttributes(ctx, input)
 
 	parts := s3response.ObjectParts{}
@@ -419,11 +408,12 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
 		}
 	}
 
-	return s3response.GetObjectAttributesResponse{
+	return s3response.GetObjectAttributesResult{
 		ETag:         out.ETag,
 		LastModified: out.LastModified,
 		ObjectSize:   out.ObjectSize,
 		StorageClass: out.StorageClass,
+		VersionId:    out.VersionId,
 		ObjectParts:  &parts,
 	}, handleError(err)
 }
@@ -530,8 +520,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
 	for i, tag := range tagout.TagSet {
 		if *tag.Key == aclKey {
 			tagout.TagSet[i] = types.Tag{
-				Key:   backend.GetPtrFromString(aclKey),
-				Value: backend.GetPtrFromString(base64Encode(data)),
+				Key:   backend.GetStringPtr(aclKey),
+				Value: backend.GetStringPtr(base64Encode(data)),
 			}
 			found = true
 			break
@@ -539,8 +529,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
 	}
 	if !found {
 		tagout.TagSet = append(tagout.TagSet, types.Tag{
-			Key:   backend.GetPtrFromString(aclKey),
-			Value: backend.GetPtrFromString(base64Encode(data)),
+			Key:   backend.GetStringPtr(aclKey),
+			Value: backend.GetStringPtr(base64Encode(data)),
 		})
 	}
 
@@ -600,7 +590,7 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
 func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
 	_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
 		Bucket: &bucket,
-		Policy: backend.GetPtrFromString(string(policy)),
+		Policy: backend.GetStringPtr(string(policy)),
 	})
 	return handleError(err)
 }

backend/scoutfs/scoutfs.go

@@ -44,7 +44,6 @@ type ScoutfsOpts struct {
 	ChownGID    bool
 	GlacierMode bool
 	BucketLinks bool
-	NewDirPerm  fs.FileMode
 }
 
 type ScoutFS struct {
@@ -75,9 +74,6 @@ type ScoutFS struct {
 	// used to determine if chowning is needed
 	euid int
 	egid int
-
-	// newDirPerm is the permissions to use when creating new directories
-	newDirPerm fs.FileMode
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -230,7 +226,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
-		b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(bucket, partObjPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
@@ -266,7 +262,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = moveData(pf, f.File())
+		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
 			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
@@ -281,76 +277,83 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	dir := filepath.Dir(objname)
 	if dir != "" {
 		uid, gid, doChown := s.getChownIDs(acct)
-		err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
 			return nil, err
 		}
 	}
+	err = f.link()
+	if err != nil {
+		return nil, fmt.Errorf("link object in namespace: %w", err)
+	}
 
 	for k, v := range userMetaData {
-		err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
+		err = s.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
+			// cleanup object if returning error
+			os.Remove(objname)
 			return nil, fmt.Errorf("set user attr %q: %w", k, err)
 		}
 	}
 
 	// load and set tagging
-	tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
-	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object tagging: %w", err)
-	}
+	tagging, err := s.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
 	if err == nil {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
-		if err != nil {
+		if err := s.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
+			// cleanup object
+			os.Remove(objname)
 			return nil, fmt.Errorf("set object tagging: %w", err)
 		}
 	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object tagging: %w", err)
+	}
 
 	// set content-type
 	if cType != "" {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
-		if err != nil {
+		if err := s.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
+			// cleanup object
+			os.Remove(objname)
 			return nil, fmt.Errorf("set object content type: %w", err)
 		}
 	}
 
 	// load and set legal hold
-	lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
-	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object legal hold: %w", err)
-	}
+	lHold, err := s.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
 	if err == nil {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
-		if err != nil {
+		if err := s.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
+			// cleanup object
+			os.Remove(objname)
 			return nil, fmt.Errorf("set object legal hold: %w", err)
 		}
 	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object legal hold: %w", err)
+	}
 
 	// load and set retention
-	ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
-	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object retention: %w", err)
-	}
+	ret, err := s.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
 	if err == nil {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
-		if err != nil {
+		if err := s.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
+			// cleanup object
+			os.Remove(objname)
 			return nil, fmt.Errorf("set object retention: %w", err)
 		}
 	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object retention: %w", err)
+	}
 
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
-	err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
+	err = s.meta.StoreAttribute(bucket, object, etagkey, []byte(s3MD5))
 	if err != nil {
+		// cleanup object if returning error
+		os.Remove(objname)
 		return nil, fmt.Errorf("set etag attr: %w", err)
 	}
 
-	err = f.link()
-	if err != nil {
-		return nil, fmt.Errorf("link object in namespace: %w", err)
-	}
-
 	// cleanup tmp dirs
 	os.RemoveAll(upiddir)
 	// use Remove for objdir in case there are still other uploads
@@ -389,7 +392,7 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
 		if !isValidMeta(e) {
 			continue
 		}
-		b, err := s.meta.RetrieveAttribute(nil, bucket, object, e)
+		b, err := s.meta.RetrieveAttribute(bucket, object, e)
 		if err != nil {
 			continue
 		}
@@ -401,13 +404,13 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
 	}
 
 	var contentType, contentEncoding string
-	b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
+	b, _ := s.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
 	contentType = string(b)
 	if contentType != "" {
 		m[contentTypeHdr] = contentType
 	}
 
-	b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
+	b, _ = s.meta.RetrieveAttribute(bucket, object, contentEncHdr)
 	contentEncoding = string(b)
 	if contentEncoding != "" {
 		m[contentEncHdr] = contentEncoding
@@ -463,7 +466,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 			return nil, fmt.Errorf("stat part: %w", err)
 		}
 
-		b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(bucket, partPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
@@ -490,7 +493,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	objPath := filepath.Join(bucket, object)
 
 	fi, err := os.Stat(objPath)
-	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
+	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 	if errors.Is(err, syscall.ENAMETOOLONG) {
@@ -511,7 +514,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 		contentType = "application/x-directory"
 	}
 
-	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
+	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -551,7 +554,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	contentLength := fi.Size()
 
 	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
-	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
+	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, "")
 	if err == nil {
 		if *status {
 			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
@@ -562,7 +565,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 
 	var objectLockMode types.ObjectLockMode
 	var objectLockRetainUntilDate *time.Time
-	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
+	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, "")
 	if err == nil {
 		var config types.ObjectLockRetention
 		if err := json.Unmarshal(retention, &config); err == nil {
@@ -614,7 +617,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 	objPath := filepath.Join(bucket, object)
 
 	fi, err := os.Stat(objPath)
-	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
+	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 	if errors.Is(err, syscall.ENAMETOOLONG) {
@@ -682,7 +685,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 
 	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
+	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -837,7 +840,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
+			etagBytes, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
 			if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
 				return s3response.Object{}, backend.ErrSkipObj
 			}
@@ -866,7 +869,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		}
 
 		// file object, get object info and fill out object data
-		b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
+		b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
 			return s3response.Object{}, backend.ErrSkipObj
 		}

backend/scoutfs/scoutfs_compat.go

@@ -40,7 +40,6 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		ChownUID:    opts.ChownUID,
 		ChownGID:    opts.ChownGID,
 		BucketLinks: opts.BucketLinks,
-		NewDirPerm:  opts.NewDirPerm,
 	})
 	if err != nil {
 		return nil, err
@@ -59,7 +58,6 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		chownuid:    opts.ChownUID,
 		chowngid:    opts.ChownGID,
 		glaciermode: opts.GlacierMode,
-		newDirPerm:  opts.NewDirPerm,
 	}, nil
 }
 
@@ -73,10 +71,10 @@ type tmpfile struct {
 	needsChown bool
 	uid        int
 	gid        int
-	newDirPerm fs.FileMode
 }
 
 var (
+	// TODO: make this configurable
 	defaultFilePerm uint32 = 0644
 )
 
@@ -104,7 +102,6 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
-		newDirPerm: s.newDirPerm,
 	}
 
 	if doChown {
@@ -132,7 +129,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -177,10 +174,6 @@ func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
 
-func (tmp *tmpfile) File() *os.File {
-	return tmp.f
-}
-
 func moveData(from *os.File, to *os.File) error {
 	return scoutfs.MoveData(from, to)
 }

backend/scoutfs/scoutfs_incompat.go

@@ -28,7 +28,9 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
-type tmpfile struct{}
+type tmpfile struct {
+	f *os.File
+}
 
 var (
 	errNotSupported = errors.New("not supported")
@@ -54,10 +56,6 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
-func (tmp *tmpfile) File() *os.File {
-	return nil
-}
-
 func moveData(_, _ *os.File) error {
 	return errNotSupported
 }

backend/walk.go

@@ -19,9 +19,9 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"os"
 	"sort"
 	"strings"
-	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3response"
@@ -53,15 +53,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 	var newMarker string
 	var truncated bool
 
-	root := "."
-	if strings.Contains(prefix, "/") {
-		idx := strings.LastIndex(prefix, "/")
-		if idx > 0 {
-			root = prefix[:idx]
-		}
-	}
-
-	err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -84,9 +76,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipAll
 		}
 
-		// After this point, return skipflag instead of nil
-		// so we can skip a directory without an early return
-		var skipflag error
 		if d.IsDir() {
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
@@ -96,57 +85,51 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// building to match. So only skip if path isn't a prefix of prefix
 			// and prefix isn't a prefix of path.
 			if prefix != "" &&
-				!strings.HasPrefix(path+"/", prefix) &&
-				!strings.HasPrefix(prefix, path+"/") {
+				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
+				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
 				return fs.SkipDir
 			}
 
-			// Don't recurse into subdirectories which contain the delimiter
-			// after reaching the prefix
-			if delimiter != "" &&
-				strings.HasPrefix(path+"/", prefix) &&
-				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
-				skipflag = fs.SkipDir
-			} else {
-				// TODO: can we do better here rather than a second readdir
-				// per directory?
-				ents, err := fs.ReadDir(fileSystem, path)
-				if err != nil {
-					return fmt.Errorf("readdir %q: %w", path, err)
-				}
-				if len(ents) == 0 && delimiter == "" {
-					dirobj, err := getObj(path+"/", d)
-					if err == ErrSkipObj {
-						return skipflag
-					}
-					if err != nil {
-						return fmt.Errorf("directory to object %q: %w", path, err)
-					}
-					objects = append(objects, dirobj)
-
-					return skipflag
-				}
-
-				if len(ents) != 0 {
-					return skipflag
-				}
+			// TODO: can we do better here rather than a second readdir
+			// per directory?
+			ents, err := fs.ReadDir(fileSystem, path)
+			if err != nil {
+				return fmt.Errorf("readdir %q: %w", path, err)
+			}
+
+			path += string(os.PathSeparator)
+
+			if len(ents) == 0 && delimiter == "" {
+				dirobj, err := getObj(path, d)
+				if err == ErrSkipObj {
+					return nil
+				}
+				if err != nil {
+					return fmt.Errorf("directory to object %q: %w", path, err)
+				}
+				objects = append(objects, dirobj)
+
+				return nil
+			}
+
+			if len(ents) != 0 {
+				return nil
 			}
-			path += "/"
 		}
 
 		if !pastMarker {
 			if path == marker {
 				pastMarker = true
-				return skipflag
+				return nil
 			}
 			if path < marker {
-				return skipflag
+				return nil
 			}
 		}
 
 		// If object doesn't have prefix, don't include in results.
 		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return skipflag
+			return nil
 		}
 
 		if delimiter == "" {
@@ -154,7 +137,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// prefix are included in results
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return skipflag
+				return nil
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -165,7 +148,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 				pastMax = true
 			}
 
-			return skipflag
+			return nil
 		}
 
 		// Since delimiter is specified, we only want results that
@@ -194,7 +177,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		if !found {
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return skipflag
+				return nil
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -203,7 +186,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			if (len(objects) + len(cpmap)) == int(max) {
 				pastMax = true
 			}
-			return skipflag
+			return nil
 		}
 
 		// Common prefixes are a set, so should not have duplicates.
@@ -213,12 +196,12 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		cpref := prefix + before + delimiter
 		if cpref == marker {
 			pastMarker = true
-			return skipflag
+			return nil
 		}
 
 		if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
 			// skip common prefixes that are before the marker
-			return skipflag
+			return nil
 		}
 
 		cpmap[cpref] = struct{}{}
@@ -228,13 +211,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipAll
 		}
 
-		return skipflag
+		return nil
 	})
 	if err != nil {
-		// suppress file not found caused by user's prefix
-		if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
-			return WalkResults{}, nil
-		}
 		return WalkResults{}, err
 	}
 
@@ -336,16 +315,8 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 			// building to match. So only skip if path isn't a prefix of prefix
 			// and prefix isn't a prefix of path.
 			if prefix != "" &&
-				!strings.HasPrefix(path+"/", prefix) &&
-				!strings.HasPrefix(prefix, path+"/") {
-				return fs.SkipDir
-			}
-
-			// Don't recurse into subdirectories when listing with delimiter.
-			if delimiter == "/" &&
-				prefix != path+"/" &&
-				strings.HasPrefix(path+"/", prefix) {
-				cpmap[path+"/"] = struct{}{}
+				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
+				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
 				return fs.SkipDir
 			}
 

backend/walk_test.go

@@ -31,18 +31,9 @@ import (
 )
 
 type walkTest struct {
-	fsys   fs.FS
-	getobj backend.GetObjFunc
-	cases  []testcase
-}
-
-type testcase struct {
-	name      string
-	prefix    string
-	delimiter string
-	marker    string
-	maxObjs   int32
-	expected  backend.WalkResults
+	fsys     fs.FS
+	expected backend.WalkResults
+	getobj   backend.GetObjFunc
 }
 
 func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
@@ -97,154 +88,50 @@ func TestWalk(t *testing.T) {
 				"photos/2006/February/sample3.jpg": {},
 				"photos/2006/February/sample4.jpg": {},
 			},
-			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "aws example",
-					delimiter: "/",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photos/"),
-						}},
-						Objects: []s3response.Object{{
-							Key: backend.GetPtrFromString("sample.jpg"),
-						}},
-					},
-				},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("photos/"),
+				}},
+				Objects: []s3response.Object{{
+					Key: backend.GetStringPtr("sample.jpg"),
+				}},
 			},
+			getobj: getObj,
 		},
 		{
 			// test case single dir/single file
 			fsys: fstest.MapFS{
 				"test/file": {},
 			},
-			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "single dir single file",
-					delimiter: "/",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("test/"),
-						}},
-						Objects: []s3response.Object{},
-					},
-				},
-			},
-		},
-		{
-			// non-standard delimiter
-			fsys: fstest.MapFS{
-				"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
-				"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
-				"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("test/"),
+				}},
+				Objects: []s3response.Object{},
 			},
 			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "different delimiter 1",
-					delimiter: "|",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 2",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 3",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 4",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 5",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|6/Januar|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 6",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|6/Januar|y/sampl|",
-					expected: backend.WalkResults{
-						Objects: []s3response.Object{
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
-							},
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
-							},
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
-							},
-						},
-					},
-				},
-			},
 		},
 	}
 
 	for _, tt := range tests {
-		for _, tc := range tt.cases {
-			res, err := backend.Walk(context.Background(),
-				tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
-				tt.getobj, []string{})
-			if err != nil {
-				t.Errorf("tc.name: walk: %v", err)
-			}
-
-			compareResults(tc.name, res, tc.expected, t)
+		res, err := backend.Walk(context.Background(), tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
+		if err != nil {
+			t.Fatalf("walk: %v", err)
 		}
+
+		compareResults(res, tt.expected, t)
 	}
 }
 
-func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
+func compareResults(got, wanted backend.WalkResults, t *testing.T) {
 	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
-		t.Errorf("%v: unexpected common prefix, got %v wanted %v",
-			name,
+		t.Errorf("unexpected common prefix, got %v wanted %v",
 			printCommonPrefixes(got.CommonPrefixes),
 			printCommonPrefixes(wanted.CommonPrefixes))
 	}
 
 	if !compareObjects(got.Objects, wanted.Objects) {
-		t.Errorf("%v: unexpected object, got %v wanted %v",
-			name,
+		t.Errorf("unexpected object, got %v wanted %v",
 			printObjects(got.Objects),
 			printObjects(wanted.Objects))
 	}
@@ -315,16 +202,10 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
 func printObjects(list []s3response.Object) string {
 	res := "["
 	for _, cp := range list {
-		var key string
-		if cp.Key == nil {
-			key = "<nil>"
-		} else {
-			key = *cp.Key
-		}
 		if res == "[" {
-			res = res + key
+			res = res + *cp.Key
 		} else {
-			res = res + ", " + key
+			res = res + ", " + *cp.Key
 		}
 	}
 	return res + "]"

cmd/versitygw/admin.go

@@ -19,7 +19,7 @@ import (
 	"crypto/sha256"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/xml"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,7 +29,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/aws/smithy-go"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3response"
@@ -38,7 +37,6 @@ import (
 var (
 	adminAccess   string
 	adminSecret   string
-	adminRegion   string
 	adminEndpoint string
 	allowInsecure bool
 )
@@ -173,14 +171,6 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminSecret,
 			},
-			&cli.StringFlag{
-				Name:        "region",
-				Usage:       "admin s3 region string",
-				EnvVars:     []string{"ADMIN_REGION"},
-				Value:       "us-east-1",
-				Destination: &adminRegion,
-				Aliases:     []string{"r"},
-			},
 			&cli.StringFlag{
 				Name:        "endpoint-url",
 				Usage:       "admin apis endpoint url",
@@ -225,24 +215,24 @@ func createUser(ctx *cli.Context) error {
 		GroupID: groupID,
 	}
 
-	accxml, err := xml.Marshal(acc)
+	accJson, err := json.Marshal(acc)
 	if err != nil {
 		return fmt.Errorf("failed to parse user data: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(accxml)
+	hashedPayload := sha256.Sum256(accJson)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -261,9 +251,11 @@ func createUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
+	fmt.Printf("%s\n", body)
+
 	return nil
 }
 
@@ -285,7 +277,7 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -304,9 +296,11 @@ func deleteUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
+	fmt.Printf("%s\n", body)
+
 	return nil
 }
 
@@ -323,24 +317,24 @@ func updateUser(ctx *cli.Context) error {
 		props.GroupID = &groupId
 	}
 
-	propsxml, err := xml.Marshal(props)
+	propsJSON, err := json.Marshal(props)
 	if err != nil {
 		return fmt.Errorf("failed to parse user attributes: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(propsxml)
+	hashedPayload := sha256.Sum256(propsJSON)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -359,9 +353,11 @@ func updateUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
+	fmt.Printf("%s\n", body)
+
 	return nil
 }
 
@@ -378,7 +374,7 @@ func listUsers(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -397,15 +393,15 @@ func listUsers(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
-	var accs auth.ListUserAccountsResult
-	if err := xml.Unmarshal(body, &accs); err != nil {
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
 		return err
 	}
 
-	printAcctTable(accs.Accounts)
+	printAcctTable(accs)
 
 	return nil
 }
@@ -445,7 +441,7 @@ func changeBucketOwner(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -464,9 +460,11 @@ func changeBucketOwner(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
+	fmt.Println(string(body))
+
 	return nil
 }
 
@@ -495,7 +493,7 @@ func listBuckets(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -514,26 +512,15 @@ func listBuckets(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
-	var result s3response.ListBucketsResult
-	if err := xml.Unmarshal(body, &result); err != nil {
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
 		return err
 	}
 
-	printBuckets(result.Buckets)
+	printBuckets(buckets)
 
 	return nil
 }
-
-func parseApiError(body []byte) error {
-	var apiErr smithy.GenericAPIError
-	err := xml.Unmarshal(body, &apiErr)
-	if err != nil {
-		apiErr.Code = "InternalServerError"
-		apiErr.Message = err.Error()
-	}
-
-	return &apiErr
-}

cmd/versitygw/gateway_test.go

@@ -7,7 +7,6 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
@@ -58,9 +57,7 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
-		NewDirPerm: 0755,
-	})
+	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}
@@ -78,9 +75,6 @@ func initPosix(ctx context.Context) {
 		}
 		wg.Done()
 	}()
-
-	// wait for server to start
-	time.Sleep(1 * time.Second)
 }
 
 func TestIntegration(t *testing.T) {

cmd/versitygw/posix.go

@@ -16,8 +16,6 @@ package main
 
 import (
 	"fmt"
-	"io/fs"
-	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/meta"
@@ -28,7 +26,6 @@ var (
 	chownuid, chowngid bool
 	bucketlinks        bool
 	versioningDir      string
-	dirPerms           uint
 )
 
 func posixCommand() *cli.Command {
@@ -71,14 +68,6 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_VERSIONING_DIR"},
 				Destination: &versioningDir,
 			},
-			&cli.UintFlag{
-				Name:        "dir-perms",
-				Usage:       "default directory permissions for new directories",
-				EnvVars:     []string{"VGW_DIR_PERMS"},
-				Destination: &dirPerms,
-				DefaultText: "0755",
-				Value:       0755,
-			},
 		},
 	}
 }
@@ -94,16 +83,11 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("posix xattr check: %v", err)
 	}
 
-	if dirPerms > math.MaxUint32 {
-		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
-	}
-
 	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
 		ChownUID:      chownuid,
 		ChownGID:      chowngid,
 		BucketLinks:   bucketlinks,
 		VersioningDir: versioningDir,
-		NewDirPerm:    fs.FileMode(dirPerms),
 	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)

cmd/versitygw/scoutfs.go

@@ -16,8 +16,6 @@ package main
 
 import (
 	"fmt"
-	"io/fs"
-	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/scoutfs"
@@ -71,14 +69,6 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_BUCKET_LINKS"},
 				Destination: &bucketlinks,
 			},
-			&cli.UintFlag{
-				Name:        "dir-perms",
-				Usage:       "default directory permissions for new directories",
-				EnvVars:     []string{"VGW_DIR_PERMS"},
-				Destination: &dirPerms,
-				DefaultText: "0755",
-				Value:       0755,
-			},
 		},
 	}
 }
@@ -88,16 +78,11 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	if dirPerms > math.MaxUint32 {
-		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
-	}
-
 	var opts scoutfs.ScoutfsOpts
 	opts.GlacierMode = glacier
 	opts.ChownUID = chownuid
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
-	opts.NewDirPerm = fs.FileMode(dirPerms)
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

cmd/versitygw/test.go

@@ -37,7 +37,6 @@ var (
 	pathStyle         bool
 	checksumDisable   bool
 	versioningEnabled bool
-	azureTests        bool
 )
 
 func testCommand() *cli.Command {
@@ -96,26 +95,12 @@ func initTestCommands() []*cli.Command {
 					Destination: &versioningEnabled,
 					Aliases:     []string{"vs"},
 				},
-				&cli.BoolFlag{
-					Name:        "azure-test-mode",
-					Usage:       "Skips tests that are not supported by Azure",
-					Destination: &azureTests,
-					Aliases:     []string{"azure"},
-				},
 			},
 		},
 		{
 			Name:   "posix",
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test posix when versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-			},
 		},
 		{
 			Name:   "iam",
@@ -303,9 +288,6 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if versioningEnabled {
 			opts = append(opts, integration.WithVersioningEnabled())
 		}
-		if azureTests {
-			opts = append(opts, integration.WithAzureMode())
-		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -337,22 +319,11 @@ func extractIntTests() (commands []*cli.Command) {
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
-				if versioningEnabled {
-					opts = append(opts, integration.WithVersioningEnabled())
-				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)
 				return err
 			},
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test the bucket object versioning, if the versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-			},
 		})
 	}
 	return

extra/example.conf

@@ -311,12 +311,6 @@ ROOT_SECRET_ACCESS_KEY=
 # to directories at the top level gateway directory as buckets.
 #VGW_BUCKET_LINKS=false
 
-# The default permissions mode when creating new directories is 0755. Use
-# VGW_DIR_PERMS option to set a different mode for any new directory that the
-# gateway creates. This applies to buckets created through the gateway as well
-# as any parent directories automatically created with object uploads.
-#VGW_DIR_PERMS=0755
-
 ###########
 # scoutfs #
 ###########
@@ -352,12 +346,6 @@ ROOT_SECRET_ACCESS_KEY=
 # to directories at the top level gateway directory as buckets.
 #VGW_BUCKET_LINKS=false
 
-# The default permissions mode when creating new directories is 0755. Use
-# VGW_DIR_PERMS option to set a different mode for any new directory that the
-# gateway creates. This applies to buckets created through the gateway as well
-# as any parent directories automatically created with object uploads.
-#VGW_DIR_PERMS=0755
-
 ######
 # s3 #
 ######

go.mod

@@ -3,13 +3,13 @@ module github.com/versity/versitygw
 go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.32.6
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0
-	github.com/aws/smithy-go v1.22.1
+	github.com/aws/aws-sdk-go-v2 v1.30.5
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2
+	github.com/aws/smithy-go v1.20.4
 	github.com/go-ldap/ldap/v3 v3.4.8
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/go-cmp v0.6.0
@@ -19,24 +19,23 @@ require (
 	github.com/oklog/ulid/v2 v2.1.0
 	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
-	github.com/smira/go-statsd v1.3.4
-	github.com/urfave/cli/v2 v2.27.5
-	github.com/valyala/fasthttp v1.57.0
+	github.com/smira/go-statsd v1.3.3
+	github.com/urfave/cli/v2 v2.27.4
+	github.com/valyala/fasthttp v1.55.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sync v0.9.0
-	golang.org/x/sys v0.27.0
+	golang.org/x/sys v0.25.0
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -45,32 +44,32 @@ require (
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.8 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.29.0 // indirect
-	golang.org/x/net v0.31.0 // indirect
-	golang.org/x/text v0.20.0 // indirect
-	golang.org/x/time v0.8.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.28.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.47
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.33
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.32
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect

go.sum

@@ -1,21 +1,17 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
 github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -23,55 +19,51 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4=
-github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
-github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo=
-github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42 h1:vEnk9vtjJ62OO2wOhEmgKMZgNcn1w0aF7XCiNXO5rK0=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42/go.mod h1:GUOPbPJWRZsdt1OJ355upCrry4d3ZFgdX6rhT7gtkto=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
+github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
+github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU=
+github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18 h1:9DIp7vhmOPmueCDwpXa45bEbLHHTt1kcxChdTJWWxvI=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18/go.mod h1:aJv/Fwz8r56ozwYFRC4bzoeL1L17GYQYemfblOBux1M=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0 h1:HrHFR8RoS4l4EvodRMFcJMYQ8o3UhmALn2nbInXaxZA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8=
-github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
-github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
-github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o=
+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
@@ -117,11 +109,9 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -135,8 +125,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
 github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
-github.com/nats-io/nkeys v0.4.8 h1:+wee30071y3vCZAYRsnrmIPaOe47A/SkK/UBDPdIV70=
-github.com/nats-io/nkeys v0.4.8/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
@@ -152,8 +142,6 @@ github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
 github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -164,8 +152,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
-github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
+github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
+github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
@@ -177,12 +165,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
-github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
+github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
-github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE=
+github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
+github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -195,8 +183,6 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -206,8 +192,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
-golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -223,14 +209,12 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
-golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
-golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -250,8 +234,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
-golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -267,10 +251,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
-golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
-golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
-golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
+golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

metrics/actions.go

@@ -72,14 +72,6 @@ var (
 	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
 	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
 	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
-
-	// Admin actions
-	ActionAdminCreateUser        = "admin_CreateUser"
-	ActionAdminUpdateUser        = "admin_UpdateUser"
-	ActionAdminDeleteUser        = "admin_DeleteUser"
-	ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
-	ActionAdminListUsers         = "admin_ListUsers"
-	ActionAdminListBuckets       = "admin_ListBuckets"
 )
 
 func init() {

runtests.sh

@@ -5,19 +5,17 @@ rm -rf /tmp/gw
 mkdir /tmp/gw
 rm -rf /tmp/covdata
 mkdir /tmp/covdata
-rm -rf /tmp/versioing.covdata
-mkdir /tmp/versioning.covdata
 rm -rf /tmp/versioningdir
 mkdir /tmp/versioningdir
 
-# run server in background not versioning-enabled
-# port: 7070(default)
-GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
+# run server in background
+GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
 GW_PID=$!
 
+# wait a second for server to start up
 sleep 1
 
-# check if versioning-enabled gateway process is still running
+# check if server is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -25,7 +23,7 @@ fi
 
 # run tests
 # full flow tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow -vs; then
 	echo "full flow tests failed"
 	kill $GW_PID
 	exit 1
@@ -43,39 +41,8 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
 	exit 1
 fi
 
-kill $GW_PID
-
-# run server in background versioning-enabled
-# port: 7071
-GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7071 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
-GW_VS_PID=$!
-
-# wait a second for server to start up
-sleep 1
-
-# check if versioning-enabled gateway process is still running
-if ! kill -0 $GW_VS_PID; then
-	echo "versioning-enabled server no longer running"
-	exit 1
-fi
-
-# run tests
-# full flow tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 full-flow -vs; then
-	echo "versioning-enabled full-flow tests failed"
-	kill $GW_VS_PID
-	exit 1
-fi
-# posix tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 posix -vs; then
-	echo "versiongin-enabled posix tests failed"
-	kill $GW_VS_PID
-	exit 1
-fi
-
 # kill off server
-kill $GW_VS_PID
-
+kill $GW_PID
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-server.go

@@ -53,9 +53,6 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
 	app.Use(middlewares.VerifyMD5Body(l))
 
-	// Admin role checker
-	app.Use(middlewares.IsAdmin(l))
-
 	server.router.Init(app, be, iam, l)
 
 	return server

s3api/controllers/admin.go

@@ -16,19 +16,15 @@ package controllers
 
 import (
 	"encoding/json"
-	"encoding/xml"
+	"errors"
 	"fmt"
-	"net/http"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/metrics"
-	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
-	"github.com/versity/versitygw/s3response"
 )
 
 type AdminController struct {
@@ -42,124 +38,187 @@ func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLo
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:CreateUser",
+			})
+	}
 	var usr auth.Account
-	err := xml.Unmarshal(ctx.Body(), &usr)
+	err := json.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminCreateUser,
+		return sendResponse(ctx, fmt.Errorf("failed to parse request body: %w", err), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusBadRequest,
+				action: "admin:CreateUser",
 			})
 	}
 
-	if !usr.Role.IsValid() {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminCreateUser,
+	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
+		return sendResponse(ctx, errors.New("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusBadRequest,
+				action: "admin:CreateUser",
 			})
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
+		status := fiber.StatusInternalServerError
+		err = fmt.Errorf("failed to create user: %w", err)
+
 		if strings.Contains(err.Error(), "user already exists") {
-			err = s3err.GetAPIError(s3err.ErrAdminUserExists)
+			status = fiber.StatusConflict
 		}
 
-		return SendResponse(ctx, err,
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminCreateUser,
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				status: status,
+				logger: c.l,
+				action: "admin:CreateUser",
 			})
 	}
 
-	return SendResponse(ctx, nil,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminCreateUser,
-			Status: http.StatusCreated,
-		})
+	return sendResponse(ctx, nil, "The user has been created successfully", &metaOptions{
+		status: fiber.StatusCreated,
+		logger: c.l,
+		action: "admin:CreateUser",
+	})
 }
 
 func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:UpdateUser",
+			})
+	}
+
 	access := ctx.Query("access")
 	if access == "" {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminUpdateUser,
+		return sendResponse(ctx, errors.New("missing user access parameter"), nil,
+			&metaOptions{
+				status: fiber.StatusBadRequest,
+				logger: c.l,
+				action: "admin:UpdateUser",
 			})
 	}
 
 	var props auth.MutableProps
-	if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminUpdateUser,
+	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
+		return sendResponse(ctx, fmt.Errorf("invalid request body %w", err), nil,
+			&metaOptions{
+				status: fiber.StatusBadRequest,
+				logger: c.l,
+				action: "admin:UpdateUser",
 			})
 	}
 
 	err := c.iam.UpdateUserAccount(access, props)
 	if err != nil {
+		status := fiber.StatusInternalServerError
+		err = fmt.Errorf("failed to update user account: %w", err)
+
 		if strings.Contains(err.Error(), "user not found") {
-			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
+			status = fiber.StatusNotFound
 		}
 
-		return SendResponse(ctx, err,
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminUpdateUser,
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				status: status,
+				logger: c.l,
+				action: "admin:UpdateUser",
 			})
 	}
 
-	return SendResponse(ctx, nil,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminUpdateUser,
+	return sendResponse(ctx, nil, "the user has been updated successfully",
+		&metaOptions{
+			logger: c.l,
+			action: "admin:UpdateUser",
 		})
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:DeleteUser",
+			})
+	}
 
 	err := c.iam.DeleteUserAccount(access)
-	return SendResponse(ctx, err,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminDeleteUser,
+	if err != nil {
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:DeleteUser",
+			})
+	}
+
+	return sendResponse(ctx, nil, "The user has been deleted successfully",
+		&metaOptions{
+			logger: c.l,
+			action: "admin:DeleteUser",
 		})
 }
 
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:ListUsers",
+			})
+	}
 	accs, err := c.iam.ListUserAccounts()
-	return SendXMLResponse(ctx,
-		auth.ListUserAccountsResult{
-			Accounts: accs,
-		}, err,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminListUsers,
+	return sendResponse(ctx, err, accs,
+		&metaOptions{
+			logger: c.l,
+			action: "admin:ListUsers",
 		})
 }
 
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:ChangeBucketOwner",
+			})
+	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
 
 	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
 	if err != nil {
-		return SendResponse(ctx, err,
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminChangeBucketOwner,
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:ChangeBucketOwner",
 			})
 	}
 	if len(accs) > 0 {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminChangeBucketOwner,
+		return sendResponse(ctx, errors.New("user specified as the new bucket owner does not exist"), nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:ChangeBucketOwner",
+				status: fiber.StatusNotFound,
 			})
 	}
 
@@ -176,28 +235,91 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 
 	aclParsed, err := json.Marshal(acl)
 	if err != nil {
-		return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminChangeBucketOwner,
+		return sendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err), nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:ChangeBucketOwner",
 			})
 	}
 
 	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
-	return SendResponse(ctx, err,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminChangeBucketOwner,
+	return sendResponse(ctx, err, "Bucket owner has been updated successfully",
+		&metaOptions{
+			logger: c.l,
+			action: "admin:ChangeBucketOwner",
 		})
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:ListBuckets",
+			})
+	}
+
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
-	return SendXMLResponse(ctx,
-		s3response.ListBucketsResult{
-			Buckets: buckets,
-		}, err, &MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminListBuckets,
+	return sendResponse(ctx, err, buckets,
+		&metaOptions{
+			logger: c.l,
+			action: "admin:ListBuckets",
 		})
 }
+
+type metaOptions struct {
+	action string
+	status int
+	logger s3log.AuditLogger
+}
+
+func sendResponse(ctx *fiber.Ctx, err error, data any, m *metaOptions) error {
+	status := m.status
+	if err != nil {
+		if status == 0 {
+			status = fiber.StatusInternalServerError
+		}
+		if m.logger != nil {
+			m.logger.Log(ctx, err, []byte(err.Error()), s3log.LogMeta{
+				Action:     m.action,
+				HttpStatus: status,
+			})
+		}
+
+		return ctx.Status(status).SendString(err.Error())
+	}
+
+	if status == 0 {
+		status = fiber.StatusOK
+	}
+
+	msg, ok := data.(string)
+	if ok {
+		if m.logger != nil {
+			m.logger.Log(ctx, nil, []byte(msg), s3log.LogMeta{
+				Action:     m.action,
+				HttpStatus: status,
+			})
+		}
+
+		return ctx.Status(status).SendString(msg)
+	}
+
+	dataJSON, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+
+	if m.logger != nil {
+		m.logger.Log(ctx, nil, dataJSON, s3log.LogMeta{
+			HttpStatus: status,
+			Action:     m.action,
+		})
+	}
+
+	ctx.Set(fiber.HeaderContentType, fiber.MIMEApplicationJSON)
+
+	return ctx.Status(status).Send(dataJSON)
+}

s3api/controllers/admin_test.go

@@ -15,11 +15,12 @@
 package controllers
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
@@ -42,26 +43,33 @@ func TestAdminController_CreateUser(t *testing.T) {
 
 	app := fiber.New()
 
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	app.Patch("/create-user", adminController.CreateUser)
 
-	succUser := `
-		<Account>
-			<Access>access</Access>
-			<Secret>secret</Secret>
-			<Role>admin</Role>
-			<UserID>0</UserID>
-			<GroupID>0</GroupID>
-		</Account>
-	`
-	invuser := `
-		<Account>
-			<Access>access</Access>
-			<Secret>secret</Secret>
-			<Role>invalid_role</Role>
-			<UserID>0</UserID>
-			<GroupID>0</GroupID>
-		</Account>
-	`
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	usr := auth.Account{
+		Access: "access",
+		Secret: "secret",
+		Role:   "invalid role",
+	}
+
+	user, _ := json.Marshal(&usr)
+
+	usr.Role = "admin"
+
+	succUsr, _ := json.Marshal(&usr)
+
+	appErr.Patch("/create-user", adminController.CreateUser)
 
 	tests := []struct {
 		name       string
@@ -71,31 +79,31 @@ func TestAdminController_CreateUser(t *testing.T) {
 		statusCode int
 	}{
 		{
-			name: "Admin-create-user-malformed-body",
+			name: "Admin-create-user-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
+			},
+			wantErr:    false,
+			statusCode: 201,
+		},
+		{
+			name: "Admin-create-user-invalid-user-role",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
 			},
 			wantErr:    false,
 			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
-			app:  app,
+			app:  appErr,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
-			statusCode: 400,
-		},
-		{
-			name: "Admin-create-user-success",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
-			},
-			wantErr:    false,
-			statusCode: 201,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -126,8 +134,24 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	app := fiber.New()
 
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	app.Patch("/update-user", adminController.UpdateUser)
 
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/update-user", adminController.UpdateUser)
+
+	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
+
 	adminControllerErr := AdminController{
 		iam: &IAMServiceMock{
 			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
@@ -138,15 +162,12 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	appNotFound := fiber.New()
 
-	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
+	appNotFound.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
 
-	succUser := `
-		<Account>
-			<Secret>secret</Secret>
-			<UserID>0</UserID>
-			<GroupID>0</GroupID>
-		</Account>
-	`
+	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
 
 	tests := []struct {
 		name       string
@@ -159,7 +180,7 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			name: "Admin-update-user-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
 			},
 			wantErr:    false,
 			statusCode: 200,
@@ -168,10 +189,10 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			name: "Admin-update-user-missing-access",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
 			},
 			wantErr:    false,
-			statusCode: 404,
+			statusCode: 400,
 		},
 		{
 			name: "Admin-update-user-invalid-request-body",
@@ -182,11 +203,20 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
+		{
+			name: "Admin-update-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
 		{
 			name: "Admin-update-user-not-found",
 			app:  appNotFound,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
 			},
 			wantErr:    false,
 			statusCode: 404,
@@ -220,8 +250,22 @@ func TestAdminController_DeleteUser(t *testing.T) {
 
 	app := fiber.New()
 
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	app.Patch("/delete-user", adminController.DeleteUser)
 
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/delete-user", adminController.DeleteUser)
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -238,6 +282,15 @@ func TestAdminController_DeleteUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Admin-delete-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -274,9 +327,30 @@ func TestAdminController_ListUsers(t *testing.T) {
 	}
 
 	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	appErr.Patch("/list-users", adminControllerErr.ListUsers)
 
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-users", adminController.ListUsers)
+
 	appSucc := fiber.New()
+
+	appSucc.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
@@ -286,6 +360,15 @@ func TestAdminController_ListUsers(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
+		{
+			name: "Admin-list-users-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
 		{
 			name: "Admin-list-users-iam-error",
 			app:  appErr,
@@ -352,12 +435,39 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 
 	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
 
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
 	appIamErr := fiber.New()
+
+	appIamErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
 
 	appIamNoSuchUser := fiber.New()
+
+	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
@@ -367,6 +477,15 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
+		{
+			name: "Change-bucket-owner-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
 			app:  appIamErr,
@@ -421,8 +540,23 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	}
 
 	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -430,6 +564,15 @@ func TestAdminController_ListBuckets(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
+		{
+			name: "List-buckets-incorrect-role",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
 		{
 			name: "List-buckets-success",
 			app:  app,

s3api/controllers/backend_moq_test.go

@@ -41,7 +41,7 @@ var _ backend.Backend = &BackendMock{}
 //			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 //				panic("mock out the CreateMultipartUpload method")
 //			},
-//			DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error {
+//			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
 //			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
@@ -74,7 +74,7 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 //				panic("mock out the GetBucketTagging method")
 //			},
-//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
 //				panic("mock out the GetBucketVersioning method")
 //			},
 //			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
@@ -83,7 +83,7 @@ var _ backend.Backend = &BackendMock{}
 //			GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 //				panic("mock out the GetObjectAcl method")
 //			},
-//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 //				panic("mock out the GetObjectAttributes method")
 //			},
 //			GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error) {
@@ -104,7 +104,7 @@ var _ backend.Backend = &BackendMock{}
 //			HeadObjectFunc: func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 //				panic("mock out the HeadObject method")
 //			},
-//			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+//			ListBucketsFunc: func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
 //				panic("mock out the ListBuckets method")
 //			},
 //			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
@@ -202,7 +202,7 @@ type BackendMock struct {
 	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 
 	// DeleteBucketFunc mocks the DeleteBucket method.
-	DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error
+	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
 	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
 	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
@@ -235,7 +235,7 @@ type BackendMock struct {
 	GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
 
 	// GetBucketVersioningFunc mocks the GetBucketVersioning method.
-	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
+	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 
 	// GetObjectFunc mocks the GetObject method.
 	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
@@ -244,7 +244,7 @@ type BackendMock struct {
 	GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 
 	// GetObjectAttributesFunc mocks the GetObjectAttributes method.
-	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
+	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
 
 	// GetObjectLegalHoldFunc mocks the GetObjectLegalHold method.
 	GetObjectLegalHoldFunc func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error)
@@ -265,7 +265,7 @@ type BackendMock struct {
 	HeadObjectFunc func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 
 	// ListBucketsFunc mocks the ListBuckets method.
-	ListBucketsFunc func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
+	ListBucketsFunc func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
 
 	// ListBucketsAndOwnersFunc mocks the ListBucketsAndOwners method.
 	ListBucketsAndOwnersFunc func(contextMoqParam context.Context) ([]s3response.Bucket, error)
@@ -388,8 +388,8 @@ type BackendMock struct {
 		DeleteBucket []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// Bucket is the bucket argument value.
-			Bucket string
+			// DeleteBucketInput is the deleteBucketInput argument value.
+			DeleteBucketInput *s3.DeleteBucketInput
 		}
 		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
 		DeleteBucketOwnershipControls []struct {
@@ -547,8 +547,10 @@ type BackendMock struct {
 		ListBuckets []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// ListBucketsInput is the listBucketsInput argument value.
-			ListBucketsInput s3response.ListBucketsInput
+			// Owner is the owner argument value.
+			Owner string
+			// IsAdmin is the isAdmin argument value.
+			IsAdmin bool
 		}
 		// ListBucketsAndOwners holds details about calls to the ListBucketsAndOwners method.
 		ListBucketsAndOwners []struct {
@@ -1010,21 +1012,21 @@ func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
 }
 
 // DeleteBucket calls DeleteBucketFunc.
-func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, bucket string) error {
+func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 	if mock.DeleteBucketFunc == nil {
 		panic("BackendMock.DeleteBucketFunc: method is nil but Backend.DeleteBucket was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam context.Context
-		Bucket          string
+		ContextMoqParam   context.Context
+		DeleteBucketInput *s3.DeleteBucketInput
 	}{
-		ContextMoqParam: contextMoqParam,
-		Bucket:          bucket,
+		ContextMoqParam:   contextMoqParam,
+		DeleteBucketInput: deleteBucketInput,
 	}
 	mock.lockDeleteBucket.Lock()
 	mock.calls.DeleteBucket = append(mock.calls.DeleteBucket, callInfo)
 	mock.lockDeleteBucket.Unlock()
-	return mock.DeleteBucketFunc(contextMoqParam, bucket)
+	return mock.DeleteBucketFunc(contextMoqParam, deleteBucketInput)
 }
 
 // DeleteBucketCalls gets all the calls that were made to DeleteBucket.
@@ -1032,12 +1034,12 @@ func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, bucket st
 //
 //	len(mockedBackend.DeleteBucketCalls())
 func (mock *BackendMock) DeleteBucketCalls() []struct {
-	ContextMoqParam context.Context
-	Bucket          string
+	ContextMoqParam   context.Context
+	DeleteBucketInput *s3.DeleteBucketInput
 } {
 	var calls []struct {
-		ContextMoqParam context.Context
-		Bucket          string
+		ContextMoqParam   context.Context
+		DeleteBucketInput *s3.DeleteBucketInput
 	}
 	mock.lockDeleteBucket.RLock()
 	calls = mock.calls.DeleteBucket
@@ -1410,7 +1412,7 @@ func (mock *BackendMock) GetBucketTaggingCalls() []struct {
 }
 
 // GetBucketVersioning calls GetBucketVersioningFunc.
-func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
 	if mock.GetBucketVersioningFunc == nil {
 		panic("BackendMock.GetBucketVersioningFunc: method is nil but Backend.GetBucketVersioning was just called")
 	}
@@ -1518,7 +1520,7 @@ func (mock *BackendMock) GetObjectAclCalls() []struct {
 }
 
 // GetObjectAttributes calls GetObjectAttributesFunc.
-func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 	if mock.GetObjectAttributesFunc == nil {
 		panic("BackendMock.GetObjectAttributesFunc: method is nil but Backend.GetObjectAttributes was just called")
 	}
@@ -1790,21 +1792,23 @@ func (mock *BackendMock) HeadObjectCalls() []struct {
 }
 
 // ListBuckets calls ListBucketsFunc.
-func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
 	if mock.ListBucketsFunc == nil {
 		panic("BackendMock.ListBucketsFunc: method is nil but Backend.ListBuckets was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam  context.Context
-		ListBucketsInput s3response.ListBucketsInput
+		ContextMoqParam context.Context
+		Owner           string
+		IsAdmin         bool
 	}{
-		ContextMoqParam:  contextMoqParam,
-		ListBucketsInput: listBucketsInput,
+		ContextMoqParam: contextMoqParam,
+		Owner:           owner,
+		IsAdmin:         isAdmin,
 	}
 	mock.lockListBuckets.Lock()
 	mock.calls.ListBuckets = append(mock.calls.ListBuckets, callInfo)
 	mock.lockListBuckets.Unlock()
-	return mock.ListBucketsFunc(contextMoqParam, listBucketsInput)
+	return mock.ListBucketsFunc(contextMoqParam, owner, isAdmin)
 }
 
 // ListBucketsCalls gets all the calls that were made to ListBuckets.
@@ -1812,12 +1816,14 @@ func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, listBucket
 //
 //	len(mockedBackend.ListBucketsCalls())
 func (mock *BackendMock) ListBucketsCalls() []struct {
-	ContextMoqParam  context.Context
-	ListBucketsInput s3response.ListBucketsInput
+	ContextMoqParam context.Context
+	Owner           string
+	IsAdmin         bool
 } {
 	var calls []struct {
-		ContextMoqParam  context.Context
-		ListBucketsInput s3response.ListBucketsInput
+		ContextMoqParam context.Context
+		Owner           string
+		IsAdmin         bool
 	}
 	mock.lockListBuckets.RLock()
 	calls = mock.calls.ListBuckets

s3api/controllers/base.go

@@ -51,9 +51,8 @@ type S3ApiController struct {
 }
 
 const (
-	iso8601Format             = "20060102T150405Z"
-	iso8601TimeFormatExtended = "Mon Jan _2 15:04:05 2006"
-	defaultContentType        = "binary/octet-stream"
+	iso8601Format      = "20060102T150405Z"
+	defaultContentType = "binary/octet-stream"
 )
 
 func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) S3ApiController {
@@ -69,32 +68,8 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs
 }
 
 func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
-	cToken := ctx.Query("continuation-token")
-	prefix := ctx.Query("prefix")
-	maxBucketsStr := ctx.Query("max-buckets")
 	acct := ctx.Locals("account").(auth.Account)
-
-	maxBuckets, err := utils.ParseUint(maxBucketsStr)
-	if err != nil || maxBuckets > 10000 {
-		if c.debug {
-			log.Printf("error parsing max-buckets %q: %v\n", maxBucketsStr, err)
-		}
-		return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
-			&MetaOpts{
-				Logger:     c.logger,
-				MetricsMng: c.mm,
-				Action:     metrics.ActionListAllMyBuckets,
-			})
-	}
-
-	res, err := c.be.ListBuckets(ctx.Context(),
-		s3response.ListBucketsInput{
-			Owner:             acct.Access,
-			IsAdmin:           acct.Role == auth.RoleAdmin,
-			MaxBuckets:        maxBuckets,
-			ContinuationToken: cToken,
-			Prefix:            prefix,
-		})
+	res, err := c.be.ListBuckets(ctx.Context(), acct.Access, acct.Role == "admin")
 	return SendXMLResponse(ctx, res, err,
 		&MetaOpts{
 			Logger:     c.logger,
@@ -108,6 +83,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
+	maxParts := int32(ctx.QueryInt("max-parts", -1))
 	partNumberMarker := ctx.Query("part-number-marker")
 	acceptRange := ctx.Get("Range")
 	acct := ctx.Locals("account").(auth.Account)
@@ -245,6 +221,16 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if uploadId != "" {
+		if maxParts < 0 && ctx.Request().URI().QueryArgs().Has("max-parts") {
+			return SendResponse(ctx,
+				s3err.GetAPIError(s3err.ErrInvalidMaxParts),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionListParts,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
 		if partNumberMarker != "" {
 			n, err := strconv.Atoi(partNumberMarker)
 			if err != nil || n < 0 {
@@ -262,24 +248,8 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					})
 			}
 		}
-		mxParts := ctx.Query("max-parts")
-		maxParts, err := utils.ParseUint(mxParts)
-		if err != nil {
-			if c.debug {
-				log.Printf("error parsing max parts %q: %v",
-					mxParts, err)
-			}
-			return SendResponse(ctx,
-				s3err.GetAPIError(s3err.ErrInvalidMaxParts),
-				&MetaOpts{
-					Logger:      c.logger,
-					MetricsMng:  c.mm,
-					Action:      metrics.ActionListParts,
-					BucketOwner: parsedAcl.Owner,
-				})
-		}
 
-		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
 			AclPermission: types.PermissionRead,
@@ -298,13 +268,17 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
+		var mxParts *int32
+		if ctx.Request().URI().QueryArgs().Has("max-parts") {
+			mxParts = &maxParts
+		}
 
 		res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
 			Bucket:           &bucket,
 			Key:              &key,
 			UploadId:         &uploadId,
 			PartNumberMarker: &partNumberMarker,
-			MaxParts:         &maxParts,
+			MaxParts:         mxParts,
 		})
 		return SendXMLResponse(ctx, res, err,
 			&MetaOpts{
@@ -372,19 +346,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker")
 		maxPartsParsed, err := utils.ParseUint(maxParts)
 		if err != nil {
-			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxParts),
-				&MetaOpts{
-					Logger:      c.logger,
-					MetricsMng:  c.mm,
-					Action:      metrics.ActionGetObjectAttributes,
-					BucketOwner: parsedAcl.Owner,
-				})
-		}
-		attrs, err := utils.ParseObjectAttributes(ctx)
-		if err != nil {
-			if c.debug {
-				log.Printf("error parsing object attributes: %v", err)
-			}
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
 					Logger:      c.logger,
@@ -393,6 +354,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
+		attrs := utils.ParseObjectAttributes(ctx)
 
 		res, err := c.be.GetObjectAttributes(ctx.Context(),
 			&s3.GetObjectAttributesInput{
@@ -403,22 +365,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 				VersionId:        &versionId,
 			})
 		if err != nil {
-			hdrs := []utils.CustomHeader{}
-
-			if res.DeleteMarker != nil {
-				hdrs = append(hdrs, utils.CustomHeader{
-					Key:   "x-amz-delete-marker",
-					Value: "true",
-				})
-			}
-			if getstring(res.VersionId) != "" {
-				hdrs = append(hdrs, utils.CustomHeader{
-					Key:   "x-amz-version-id",
-					Value: getstring(res.VersionId),
-				})
-			}
-
-			utils.SetResponseHeaders(ctx, hdrs)
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
 					Logger:      c.logger,
@@ -427,31 +373,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
-
-		hdrs := []utils.CustomHeader{}
-
-		if getstring(res.VersionId) != "" {
-			hdrs = append(hdrs, utils.CustomHeader{
-				Key:   "x-amz-version-id",
-				Value: getstring(res.VersionId),
-			})
-		}
-		if res.DeleteMarker != nil && *res.DeleteMarker {
-			hdrs = append(hdrs, utils.CustomHeader{
-				Key:   "x-amz-delete-marker",
-				Value: "true",
-			})
-		}
-		if res.LastModified != nil {
-			hdrs = append(hdrs, utils.CustomHeader{
-				Key:   "Last-Modified",
-				Value: res.LastModified.UTC().Format(iso8601TimeFormatExtended),
-			})
-		}
-
-		utils.SetResponseHeaders(ctx, hdrs)
-
-		return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), nil,
+		return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), err,
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
@@ -814,7 +736,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max keys %q: %v",
 					maxkeysStr, err)
 			}
-			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
+			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -947,13 +869,12 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max uploads %q: %v",
 					maxUploadsStr, err)
 			}
-			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxUploads),
-				&MetaOpts{
-					Logger:      c.logger,
-					MetricsMng:  c.mm,
-					Action:      metrics.ActionListMultipartUploads,
-					BucketOwner: parsedAcl.Owner,
-				})
+			return SendXMLResponse(ctx, nil, err, &MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionListMultipartUploads,
+				BucketOwner: parsedAcl.Owner,
+			})
 		}
 		res, err := c.be.ListMultipartUploads(ctx.Context(),
 			&s3.ListMultipartUploadsInput{
@@ -998,7 +919,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max keys %q: %v",
 					maxkeysStr, err)
 			}
-			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
+			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -1049,7 +970,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			log.Printf("error parsing max keys %q: %v",
 				maxkeysStr, err)
 		}
-		return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
+		return SendXMLResponse(ctx, nil, err,
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
@@ -1150,7 +1071,6 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				MetricsMng:  c.mm,
 				Action:      metrics.ActionPutBucketTagging,
 				BucketOwner: parsedAcl.Owner,
-				Status:      http.StatusNoContent,
 			})
 	}
 
@@ -2323,7 +2243,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &keyStart}}, true, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{keyStart}, true, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2548,7 +2468,10 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = c.be.DeleteBucket(ctx.Context(), bucket)
+	err = c.be.DeleteBucket(ctx.Context(),
+		&s3.DeleteBucketInput{
+			Bucket: &bucket,
+		})
 	return SendResponse(ctx, err,
 		&MetaOpts{
 			Logger:      c.logger,
@@ -2604,7 +2527,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 	// The AWS CLI sends 'True', while Go SDK sends 'true'
 	bypass := strings.EqualFold(bypassHdr, "true")
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, utils.ParseDeleteObjects(dObj.Objects), bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2757,7 +2680,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	// The AWS CLI sends 'True', while Go SDK sends 'true'
 	bypass := strings.EqualFold(bypassHdr, "true")
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key, VersionId: &versionId}}, bypass, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{key}, bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2954,6 +2877,15 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 				BucketOwner: parsedAcl.Owner,
 			})
 	}
+	if res == nil {
+		return SendResponse(ctx, fmt.Errorf("head object nil response"),
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionHeadObject,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
 
 	utils.SetMetaHeaders(ctx, res.Metadata)
 	headers := []utils.CustomHeader{

s3api/controllers/base_test.go

@@ -92,7 +92,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, nil
 			},
 		},
@@ -109,7 +109,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr := fiber.New()
 	s3ApiControllerErr := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
 			},
 		},
@@ -187,8 +187,8 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 				return &s3.GetObjectAclOutput{}, nil
 			},
-			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
-				return s3response.GetObjectAttributesResponse{}, nil
+			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+				return s3response.GetObjectAttributesResult{}, nil
 			},
 			GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
@@ -382,8 +382,8 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return map[string]string{}, nil
 			},
-			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
-				return s3response.GetBucketVersioningOutput{}, nil
+			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+				return &s3.GetBucketVersioningOutput{}, nil
 			},
 			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
 				return s3response.ListVersionsResult{}, nil
@@ -750,7 +750,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
 			},
 			wantErr:    false,
-			statusCode: 204,
+			statusCode: 200,
 		},
 		{
 			name: "Put-bucket-ownership-controls-invalid-ownership",
@@ -1217,7 +1217,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			DeleteBucketFunc: func(_ context.Context, bucket string) error {
+			DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
 				return nil
 			},
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
@@ -1628,7 +1628,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 				return acldata, nil
 			},
 			HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+				return nil, s3err.GetAPIError(42)
 			},
 		},
 	}

s3api/middlewares/admin.go

@@ -1,59 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package middlewares
-
-import (
-	"strings"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/metrics"
-	"github.com/versity/versitygw/s3api/controllers"
-	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3log"
-)
-
-func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
-	return func(ctx *fiber.Ctx) error {
-		acct := ctx.Locals("account").(auth.Account)
-		if acct.Role != auth.RoleAdmin {
-			path := ctx.Path()
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
-				&controllers.MetaOpts{
-					Logger: logger,
-					Action: detectAction(path),
-				})
-		}
-
-		return ctx.Next()
-	}
-}
-
-func detectAction(path string) (action string) {
-	if strings.Contains(path, "create-user") {
-		action = metrics.ActionAdminCreateUser
-	} else if strings.Contains(path, "update-user") {
-		action = metrics.ActionAdminUpdateUser
-	} else if strings.Contains(path, "delete-user") {
-		action = metrics.ActionAdminDeleteUser
-	} else if strings.Contains(path, "list-user") {
-		action = metrics.ActionAdminListUsers
-	} else if strings.Contains(path, "list-buckets") {
-		action = metrics.ActionAdminListBuckets
-	} else if strings.Contains(path, "change-bucket-owner") {
-		action = metrics.ActionAdminChangeBucketOwner
-	}
-	return action
-}

s3api/middlewares/url-decoder.go

@@ -26,11 +26,12 @@ import (
 
 func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
+		reqURL := ctx.Request().URI().String()
+		decoded, err := url.Parse(reqURL)
 		if err != nil {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}
-		ctx.Path(unescp)
+		ctx.Path(decoded.Path)
 		return ctx.Next()
 	}
 }

s3api/router.go

@@ -20,7 +20,6 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
-	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
 )
@@ -36,22 +35,22 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		adminController := controllers.NewAdminController(iam, be, aLogger)
 
 		// CreateUser admin api
-		app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
+		app.Patch("/create-user", adminController.CreateUser)
 
 		// DeleteUsers admin api
-		app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
+		app.Patch("/delete-user", adminController.DeleteUser)
 
 		// UpdateUser admin api
-		app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
+		app.Patch("update-user", adminController.UpdateUser)
 
 		// ListUsers admin api
-		app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
+		app.Patch("/list-users", adminController.ListUsers)
 
 		// ChangeBucketOwner admin api
-		app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
+		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
 
 		// ListBucketsAndOwners admin api
-		app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
+		app.Patch("/list-buckets", adminController.ListBuckets)
 	}
 
 	// ListBuckets action

s3api/utils/utils.go

@@ -39,10 +39,6 @@ var (
 	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
-const (
-	upperhex = "0123456789ABCDEF"
-)
-
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
 	headers.DisableNormalizing()
@@ -68,9 +64,7 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		body = bytes.NewReader(req.Body())
 	}
 
-	escapedURI := escapeOriginalURI(ctx)
-
-	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
+	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -180,7 +174,7 @@ func ParseUint(str string) (int32, error) {
 	}
 	num, err := strconv.ParseUint(str, 10, 16)
 	if err != nil {
-		return 1000, fmt.Errorf("invalid uint: %w", err)
+		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
 	}
 	return int32(num), nil
 }
@@ -254,51 +248,35 @@ func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
 	return
 }
 
-func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResponse) s3response.GetObjectAttributesResponse {
-	// These properties shouldn't appear in the final response body
-	output.LastModified = nil
-	output.VersionId = nil
-	output.DeleteMarker = nil
-
-	if _, ok := attrs[s3response.ObjectAttributesEtag]; !ok {
+func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResult) s3response.GetObjectAttributesResult {
+	if _, ok := attrs[types.ObjectAttributesEtag]; !ok {
 		output.ETag = nil
 	}
-	if _, ok := attrs[s3response.ObjectAttributesObjectParts]; !ok {
+	if _, ok := attrs[types.ObjectAttributesObjectParts]; !ok {
 		output.ObjectParts = nil
 	}
-	if _, ok := attrs[s3response.ObjectAttributesObjectSize]; !ok {
+	if _, ok := attrs[types.ObjectAttributesObjectSize]; !ok {
 		output.ObjectSize = nil
 	}
-	if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
+	if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
 		output.StorageClass = ""
 	}
-	fmt.Printf("%+v\n", output)
 
 	return output
 }
 
-func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
-	attrs := map[s3response.ObjectAttributes]struct{}{}
-	var err error
+func ParseObjectAttributes(ctx *fiber.Ctx) map[types.ObjectAttributes]struct{} {
+	attrs := map[types.ObjectAttributes]struct{}{}
 	ctx.Request().Header.VisitAll(func(key, value []byte) {
 		if string(key) == "X-Amz-Object-Attributes" {
 			oattrs := strings.Split(string(value), ",")
 			for _, a := range oattrs {
-				attr := s3response.ObjectAttributes(a)
-				if !attr.IsValid() {
-					err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
-					break
-				}
-				attrs[attr] = struct{}{}
+				attrs[types.ObjectAttributes(a)] = struct{}{}
 			}
 		}
 	})
 
-	if len(attrs) == 0 {
-		return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
-	}
-
-	return attrs, err
+	return attrs
 }
 
 type objLockCfg struct {
@@ -361,74 +339,3 @@ func IsValidOwnership(val types.ObjectOwnership) bool {
 		return false
 	}
 }
-
-func escapeOriginalURI(ctx *fiber.Ctx) string {
-	path := ctx.Path()
-
-	// Escape the URI original path
-	escapedURI := escapePath(path)
-
-	// Add the URI query params
-	query := string(ctx.Request().URI().QueryArgs().QueryString())
-	if query != "" {
-		escapedURI = escapedURI + "?" + query
-	}
-
-	return escapedURI
-}
-
-// Escapes the path string
-// Most of the parts copied from std url
-func escapePath(s string) string {
-	hexCount := 0
-	for i := 0; i < len(s); i++ {
-		c := s[i]
-		if shouldEscape(c) {
-			hexCount++
-		}
-	}
-
-	if hexCount == 0 {
-		return s
-	}
-
-	var buf [64]byte
-	var t []byte
-
-	required := len(s) + 2*hexCount
-	if required <= len(buf) {
-		t = buf[:required]
-	} else {
-		t = make([]byte, required)
-	}
-
-	j := 0
-	for i := 0; i < len(s); i++ {
-		switch c := s[i]; {
-		case shouldEscape(c):
-			t[j] = '%'
-			t[j+1] = upperhex[c>>4]
-			t[j+2] = upperhex[c&15]
-			j += 3
-		default:
-			t[j] = s[i]
-			j++
-		}
-	}
-
-	return string(t)
-}
-
-// Checks if the character needs to be escaped
-func shouldEscape(c byte) bool {
-	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
-		return false
-	}
-
-	switch c {
-	case '-', '_', '.', '~', '/':
-		return false
-	}
-
-	return true
-}

s3api/utils/utils_test.go

@@ -19,12 +19,10 @@ import (
 	"net/http"
 	"reflect"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
-	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3response"
 )
 
@@ -285,68 +283,47 @@ func TestParseUint(t *testing.T) {
 
 func TestFilterObjectAttributes(t *testing.T) {
 	type args struct {
-		attrs  map[s3response.ObjectAttributes]struct{}
-		output s3response.GetObjectAttributesResponse
+		attrs  map[types.ObjectAttributes]struct{}
+		output s3response.GetObjectAttributesResult
 	}
-
 	etag, objSize := "etag", int64(3222)
-	delMarker := true
-
 	tests := []struct {
 		name string
 		args args
-		want s3response.GetObjectAttributesResponse
+		want s3response.GetObjectAttributesResult
 	}{
 		{
 			name: "keep only ETag",
 			args: args{
-				attrs: map[s3response.ObjectAttributes]struct{}{
-					s3response.ObjectAttributesEtag: {},
+				attrs: map[types.ObjectAttributes]struct{}{
+					types.ObjectAttributesEtag: {},
 				},
-				output: s3response.GetObjectAttributesResponse{
+				output: s3response.GetObjectAttributesResult{
 					ObjectSize: &objSize,
 					ETag:       &etag,
 				},
 			},
-			want: s3response.GetObjectAttributesResponse{ETag: &etag},
+			want: s3response.GetObjectAttributesResult{ETag: &etag},
 		},
 		{
 			name: "keep multiple props",
 			args: args{
-				attrs: map[s3response.ObjectAttributes]struct{}{
-					s3response.ObjectAttributesEtag:         {},
-					s3response.ObjectAttributesObjectSize:   {},
-					s3response.ObjectAttributesStorageClass: {},
+				attrs: map[types.ObjectAttributes]struct{}{
+					types.ObjectAttributesEtag:         {},
+					types.ObjectAttributesObjectSize:   {},
+					types.ObjectAttributesStorageClass: {},
 				},
-				output: s3response.GetObjectAttributesResponse{
+				output: s3response.GetObjectAttributesResult{
 					ObjectSize:  &objSize,
 					ETag:        &etag,
 					ObjectParts: &s3response.ObjectParts{},
 					VersionId:   &etag,
 				},
 			},
-			want: s3response.GetObjectAttributesResponse{
+			want: s3response.GetObjectAttributesResult{
 				ETag:       &etag,
 				ObjectSize: &objSize,
-			},
-		},
-		{
-			name: "make sure LastModified, DeleteMarker and VersionId are removed",
-			args: args{
-				attrs: map[s3response.ObjectAttributes]struct{}{
-					s3response.ObjectAttributesEtag: {},
-				},
-				output: s3response.GetObjectAttributesResponse{
-					ObjectSize:   &objSize,
-					ETag:         &etag,
-					ObjectParts:  &s3response.ObjectParts{},
-					VersionId:    &etag,
-					LastModified: backend.GetTimePtr(time.Now()),
-					DeleteMarker: &delMarker,
-				},
-			},
-			want: s3response.GetObjectAttributesResponse{
-				ETag: &etag,
+				VersionId:  &etag,
 			},
 		},
 	}
@@ -405,125 +382,3 @@ func TestIsValidOwnership(t *testing.T) {
 		})
 	}
 }
-
-func Test_shouldEscape(t *testing.T) {
-	type args struct {
-		c byte
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			name: "shouldn't-escape-alphanum",
-			args: args{
-				c: 'h',
-			},
-			want: false,
-		},
-		{
-			name: "shouldn't-escape-unreserved-char",
-			args: args{
-				c: '_',
-			},
-			want: false,
-		},
-		{
-			name: "shouldn't-escape-unreserved-number",
-			args: args{
-				c: '0',
-			},
-			want: false,
-		},
-		{
-			name: "shouldn't-escape-path-separator",
-			args: args{
-				c: '/',
-			},
-			want: false,
-		},
-		{
-			name: "should-escape-special-char-1",
-			args: args{
-				c: '&',
-			},
-			want: true,
-		},
-		{
-			name: "should-escape-special-char-2",
-			args: args{
-				c: '*',
-			},
-			want: true,
-		},
-		{
-			name: "should-escape-special-char-3",
-			args: args{
-				c: '(',
-			},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := shouldEscape(tt.args.c); got != tt.want {
-				t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_escapePath(t *testing.T) {
-	type args struct {
-		s string
-	}
-	tests := []struct {
-		name string
-		args args
-		want string
-	}{
-		{
-			name: "empty-string",
-			args: args{
-				s: "",
-			},
-			want: "",
-		},
-		{
-			name: "alphanum-path",
-			args: args{
-				s: "/test-bucket/test-key",
-			},
-			want: "/test-bucket/test-key",
-		},
-		{
-			name: "path-with-unescapable-chars",
-			args: args{
-				s: "/test~bucket/test.key",
-			},
-			want: "/test~bucket/test.key",
-		},
-		{
-			name: "path-with-escapable-chars",
-			args: args{
-				s: "/bucket-*(/test=key&",
-			},
-			want: "/bucket-%2A%28/test%3Dkey%26",
-		},
-		{
-			name: "path-with-space",
-			args: args{
-				s: "/test-bucket/my key",
-			},
-			want: "/test-bucket/my%20key",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := escapePath(tt.args.s); got != tt.want {
-				t.Errorf("escapePath() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}

s3err/s3err.go

@@ -57,7 +57,6 @@ const (
 	ErrAccessDenied
 	ErrMethodNotAllowed
 	ErrBucketNotEmpty
-	ErrVersionedBucketNotEmpty
 	ErrBucketAlreadyExists
 	ErrBucketAlreadyOwnedByYou
 	ErrNoSuchBucket
@@ -66,11 +65,9 @@ const (
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidMaxKeys
-	ErrInvalidMaxBuckets
 	ErrInvalidMaxUploads
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
-	ErrInvalidObjectAttributes
 	ErrInvalidPart
 	ErrInvalidPartNumber
 	ErrInternalError
@@ -125,7 +122,6 @@ const (
 	ErrNoSuchBucketPolicy
 	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
-	ErrObjectAttributesInvalidHeader
 	ErrRequestTimeTooSkewed
 	ErrInvalidBucketAclWithObjectOwnership
 	ErrBothCannedAndHeaderGrants
@@ -138,22 +134,12 @@ const (
 	ErrKeyTooLong
 	ErrInvalidVersionId
 	ErrNoSuchVersion
-	ErrSuspendedVersioningNotAllowed
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
-	ErrDirectoryNotEmpty
 	ErrQuotaExceeded
-	ErrVersioningNotConfigured
-
-	// Admin api errors
-	ErrAdminAccessDenied
-	ErrAdminUserNotFound
-	ErrAdminUserExists
-	ErrAdminInvalidUserRole
-	ErrAdminMissingUserAcess
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -172,11 +158,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty.",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	ErrVersionedBucketNotEmpty: {
-		Code:           "BucketNotEmpty",
-		Description:    "The bucket you tried to delete is not empty. You must delete all versions in the bucket.",
-		HTTPStatusCode: http.StatusConflict,
-	},
 	ErrBucketAlreadyExists: {
 		Code:           "BucketAlreadyExists",
 		Description:    "The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again.",
@@ -197,11 +178,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The Content-Md5 you specified is not valid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidMaxBuckets: {
-		Code:           "InvalidArgument",
-		Description:    "Argument max-buckets must be an integer between 1 and 10000.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrInvalidMaxUploads: {
 		Code:           "InvalidArgument",
 		Description:    "Argument max-uploads must be an integer between 0 and 2147483647.",
@@ -222,11 +198,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Argument partNumberMarker must be an integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidObjectAttributes: {
-		Code:           "InvalidArgument",
-		Description:    "Invalid attribute name specified.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrNoSuchBucket: {
 		Code:           "NoSuchBucket",
 		Description:    "The specified bucket does not exist.",
@@ -384,7 +355,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
-		Description:    "The AWS Access Key Id you provided does not exist in our records.",
+		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrRequestNotReadyYet: {
@@ -464,12 +435,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
-		Description:    "The specified object does not have a ObjectLock configuration.",
+		Description:    "The specified object does not have an ObjectLock configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidBucketObjectLockConfiguration: {
 		Code:           "InvalidRequest",
-		Description:    "Bucket is missing Object Lock Configuration.",
+		Description:    "Bucket is missing ObjectLockConfiguration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectLockConfigurationNotAllowed: {
@@ -507,11 +478,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrObjectAttributesInvalidHeader: {
-		Code:           "InvalidRequest",
-		Description:    "The x-amz-object-attributes header specifying the attributes to be retrieved is either missing or empty",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrRequestTimeTooSkewed: {
 		Code:           "RequestTimeTooSkewed",
 		Description:    "The difference between the request time and the server's time is too large.",
@@ -553,9 +519,8 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrInvalidMetadataDirective: {
-		Code:           "InvalidArgument",
-		Description:    "Unknown metadata directive.",
-		HTTPStatusCode: http.StatusBadRequest,
+		Code:        "InvalidArgument",
+		Description: "Unknown metadata directive.",
 	},
 	ErrInvalidVersionId: {
 		Code:           "InvalidArgument",
@@ -572,11 +537,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The specified version does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrSuspendedVersioningNotAllowed: {
-		Code:           "InvalidBucketState",
-		Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {
@@ -594,48 +554,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Directory object contains data payload.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrDirectoryNotEmpty: {
-		Code:           "ErrDirectoryNotEmpty",
-		Description:    "Directory object not empty.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrQuotaExceeded: {
 		Code:           "QuotaExceeded",
 		Description:    "Your request was denied due to quota exceeded.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	ErrVersioningNotConfigured: {
-		Code:           "VersioningNotConfigured",
-		Description:    "Versioning has not been configured for the gateway.",
-		HTTPStatusCode: http.StatusNotImplemented,
-	},
-
-	// Admin api errors
-	ErrAdminAccessDenied: {
-		Code:           "XAdminAccessDenied",
-		Description:    "Only admin users have access to this resource.",
-		HTTPStatusCode: http.StatusForbidden,
-	},
-	ErrAdminUserNotFound: {
-		Code:           "XAdminUserNotFound",
-		Description:    "No user exists with the provided access key ID.",
-		HTTPStatusCode: http.StatusNotFound,
-	},
-	ErrAdminUserExists: {
-		Code:           "XAdminUserExists",
-		Description:    "A user with the provided access key ID already exists.",
-		HTTPStatusCode: http.StatusConflict,
-	},
-	ErrAdminInvalidUserRole: {
-		Code:           "XAdminInvalidArgument",
-		Description:    "User role has to be one of the following: 'user', 'admin', 'userplus'.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrAdminMissingUserAcess: {
-		Code:           "XAdminInvalidArgument",
-		Description:    "User access key ID is missing.",
-		HTTPStatusCode: http.StatusNotFound,
-	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3response/s3response.go

@@ -19,14 +19,9 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/s3err"
 )
 
-const (
-	iso8601TimeFormat         = "2006-01-02T15:04:05.000Z"
-	iso8601TimeFormatExtended = "2006-01-02T15:04:05.000000Z"
-	iso8601TimeFormatWithTZ   = "2006-01-02T15:04:05-0700"
-)
+const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
 
 type PutObjectOutput struct {
 	ETag      string
@@ -50,7 +45,7 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 		Alias: (*Alias)(&p),
 	}
 
-	aux.LastModified = p.LastModified.UTC().Format(iso8601TimeFormat)
+	aux.LastModified = p.LastModified.UTC().Format(RFC3339TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -78,34 +73,30 @@ type ListPartsResult struct {
 	Parts []Part `xml:"Part"`
 }
 
-type ObjectAttributes string
-
-const (
-	ObjectAttributesEtag         ObjectAttributes = "ETag"
-	ObjectAttributesChecksum     ObjectAttributes = "Checksum"
-	ObjectAttributesObjectParts  ObjectAttributes = "ObjectParts"
-	ObjectAttributesStorageClass ObjectAttributes = "StorageClass"
-	ObjectAttributesObjectSize   ObjectAttributes = "ObjectSize"
-)
-
-func (o ObjectAttributes) IsValid() bool {
-	return o == ObjectAttributesChecksum ||
-		o == ObjectAttributesEtag ||
-		o == ObjectAttributesObjectParts ||
-		o == ObjectAttributesObjectSize ||
-		o == ObjectAttributesStorageClass
+type GetObjectAttributesResult struct {
+	ETag         *string
+	LastModified *time.Time
+	ObjectSize   *int64
+	StorageClass types.StorageClass
+	VersionId    *string
+	ObjectParts  *ObjectParts
 }
 
-type GetObjectAttributesResponse struct {
-	ETag         *string
-	ObjectSize   *int64
-	StorageClass types.StorageClass `xml:",omitempty"`
-	ObjectParts  *ObjectParts
+func (r GetObjectAttributesResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias GetObjectAttributesResult
+	aux := &struct {
+		LastModified *string `xml:"LastModified"`
+		*Alias
+	}{
+		Alias: (*Alias)(&r),
+	}
 
-	// Not included in the response body
-	VersionId    *string
-	LastModified *time.Time
-	DeleteMarker *bool
+	if r.LastModified != nil {
+		formattedTime := r.LastModified.UTC().Format(RFC3339TimeFormat)
+		aux.LastModified = &formattedTime
+	}
+
+	return e.EncodeElement(aux, start)
 }
 
 type ObjectParts struct {
@@ -188,7 +179,7 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	}
 
 	if o.LastModified != nil {
-		formattedTime := o.LastModified.UTC().Format(iso8601TimeFormat)
+		formattedTime := o.LastModified.UTC().Format(RFC3339TimeFormat)
 		aux.LastModified = &formattedTime
 	}
 
@@ -214,7 +205,7 @@ func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 		Alias: (*Alias)(&u),
 	}
 
-	aux.Initiated = u.Initiated.UTC().Format(iso8601TimeFormat)
+	aux.Initiated = u.Initiated.UTC().Format(RFC3339TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -281,20 +272,10 @@ type Bucket struct {
 	Owner string `json:"owner"`
 }
 
-type ListBucketsInput struct {
-	Owner             string
-	IsAdmin           bool
-	ContinuationToken string
-	Prefix            string
-	MaxBuckets        int32
-}
-
 type ListAllMyBucketsResult struct {
-	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
-	Owner             CanonicalUser
-	Buckets           ListAllMyBucketsList
-	ContinuationToken string `xml:"ContinuationToken,omitempty"`
-	Prefix            string `xml:"Prefix,omitempty"`
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	Owner   CanonicalUser
+	Buckets ListAllMyBucketsList
 }
 
 type ListAllMyBucketsEntry struct {
@@ -311,7 +292,7 @@ func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement
 		Alias: (*Alias)(&r),
 	}
 
-	aux.CreationDate = r.CreationDate.UTC().Format(iso8601TimeFormat)
+	aux.CreationDate = r.CreationDate.UTC().Format(RFC3339TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -341,7 +322,7 @@ func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) err
 		Alias: (*Alias)(&r),
 	}
 
-	aux.LastModified = r.LastModified.UTC().Format(iso8601TimeFormat)
+	aux.LastModified = r.LastModified.UTC().Format(RFC3339TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -399,71 +380,7 @@ type ListVersionsResult struct {
 	NextKeyMarker       *string
 	NextVersionIdMarker *string
 	Prefix              *string
+	RequestCharged      types.RequestCharged
 	VersionIdMarker     *string
 	Versions            []types.ObjectVersion `xml:"Version"`
 }
-
-type GetBucketVersioningOutput struct {
-	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
-	MFADelete *types.MFADeleteStatus
-	Status    *types.BucketVersioningStatus
-}
-
-type PutObjectRetentionInput struct {
-	XMLName         xml.Name `xml:"Retention"`
-	Mode            types.ObjectLockRetentionMode
-	RetainUntilDate AmzDate
-}
-
-type AmzDate struct {
-	time.Time
-}
-
-// Parses the date from xml string and validates for predefined date formats
-func (d *AmzDate) UnmarshalXML(e *xml.Decoder, startElement xml.StartElement) error {
-	var dateStr string
-	err := e.DecodeElement(&dateStr, &startElement)
-	if err != nil {
-		return err
-	}
-
-	retDate, err := d.ISO8601Parse(dateStr)
-	if err != nil {
-		return s3err.GetAPIError(s3err.ErrInvalidRequest)
-	}
-
-	*d = AmzDate{retDate}
-	return nil
-}
-
-// Encodes expiration date if it is non-zero
-// Encodes empty string if it's zero
-func (d AmzDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
-	if d.IsZero() {
-		return nil
-	}
-	return e.EncodeElement(d.UTC().Format(iso8601TimeFormat), startElement)
-}
-
-// Parses ISO8601 date string to time.Time by
-// validating different time layouts
-func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
-	for _, layout := range []string{
-		iso8601TimeFormat,
-		iso8601TimeFormatExtended,
-		iso8601TimeFormatWithTZ,
-		time.RFC3339,
-	} {
-		t, err = time.Parse(layout, date)
-		if err == nil {
-			return t, nil
-		}
-	}
-
-	return t, err
-}
-
-// Admin api response types
-type ListBucketsResult struct {
-	Buckets []Bucket
-}

tests/.env.default

@@ -27,6 +27,3 @@ USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 REMOVE_TEST_FILE_FOLDER=true
-VERSIONING_DIR=/tmp/versioning
-COMMAND_LOG=command.log
-TIME_LOG=time.log

tests/Dockerfile_direct

@@ -47,7 +47,7 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
 
 USER tester
 RUN mkdir -p /home/tester/tests
-COPY --chown=tester:tester . /home/tester
+COPY --chown=tester:tester . /home/tester/tests
 
 # add bats support libraries
 RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests

tests/Dockerfile_test_bats

@@ -86,5 +86,4 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
 ENV WORKSPACE=.
 ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
 
-ENTRYPOINT ["tests/run.sh"]
-CMD ["s3api,s3,s3cmd,mc,rest"]
+CMD ["tests/run_all.sh"]

tests/README.md

@@ -81,56 +81,4 @@ For the s3 backend, see the **S3 Backend** instructions above.
 
 If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries.
 
-A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`
-
-## Environment Parameters
-
-**AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**:  identical to the same parameters in **s3**.
-
-**VERSITY_EXE**:  location of the versity executable relative to test folder.
-
-**RUN_VERSITYGW**:  whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
-
-**BACKEND**:  the storage backend type for the gateway, e.g. **posix** or **s3**.
-
-**LOCAL_FOLDER**:  if running with a **posix** backend, the backend storage folder.
-
-**BUCKET_ONE_NAME**, **BUCKET_TWO_NAME**:  test bucket names.
-
-**RECREATE_BUCKETS**:  whether to delete buckets between tests.  If set to false, the bucket will be restored to an original state for the purpose of ensuring consistent tests, but not deleted.
-
-**CERT**, **KEY**:  certificate and key locations if using SSL.
-
-**S3CMD_CONFIG**:  location of **s3cmd** config file if running **s3cmd** tests.
-
-**SECRETS_FILE**:  file where sensitive values, such as **AWS_SECRET_ACCESS_KEY**, should be stored.
-
-**MC_ALIAS**:  Minio MC alias if running MC tests.
-
-**LOG_LEVEL**:  level for test logger (1 - only critical, 2 - errors, 3 - warnings, 4 - info, 5 - debug info, 6 - tracing)
-
-**GOCOVERDIR**:  folder to put golang coverage info in, if checking coverage info.
-
-**USERS_FOLDER**:  folder to use if storing IAM data in a folder.
-
-**IAM_TYPE**:  how to store IAM data (**s3** or **folder**).
-
-**TEST_LOG_FILE**:  log file location for these bats tests.
-
-**VERSITY_LOG_FILE**:  log file for versity application as it is tested by bats tests.
-
-**DIRECT**:  if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes).
-
-**DIRECT_DISPLAY_NAME**:  username if **DIRECT** is set to **true**.
-
-**COVERAGE_DB**:  database to store client command coverage info and usage counts, if using.
-
-**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  credentials for users created and tested for non-root user **versitygw** operations.
-
-**TEST_FILE_FOLDER**:  where to put temporary test files.
-
-**REMOVE_TEST_FILE_FOLDER**:  whether to delete the test file folder between tests, should be set to **true** unless checking the files after a single test, or not yet sure that the test folder is in a safe location to avoid deleting other files.
-
-**VERSIONING_DIR**:  where to put gateway file versioning info.
-
-**COMMAND_LOG**:  where to store list of client commands, which if using will be reported during test failures.
+A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`

tests/commands/abort_multipart_upload.sh

@@ -20,7 +20,7 @@ abort_multipart_upload() {
     log 2 "'abort multipart upload' command requires bucket, key, upload ID"
     return 1
   fi
-  if ! error=$(send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error aborting upload: $error"
     return 1
   fi
@@ -33,10 +33,10 @@ abort_multipart_upload_with_user() {
     return 1
   fi
   record_command "abort-multipart-upload" "client:s3api"
-  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error aborting upload: $abort_multipart_upload_error"
     export abort_multipart_upload_error
     return 1
   fi
   return 0
-}
+}

tests/commands/complete_multipart_upload.sh

@@ -21,7 +21,7 @@ complete_multipart_upload() {
   fi
   log 5 "complete multipart upload id: $3, parts: $4"
   record_command "complete-multipart-upload" "client:s3api"
-  error=$(send_command aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
+  error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
   if [[ $completed -ne 0 ]]; then
     log 2 "error completing multipart upload: $error"
     return 1

tests/commands/copy_object.sh

@@ -16,28 +16,28 @@
 
 copy_object() {
   if [ $# -ne 4 ]; then
-    log 2 "copy object command requires command type, source, bucket, key"
+    echo "copy object command requires command type, source, bucket, key"
     return 1
   fi
   local exit_code=0
   local error
   record_command "copy-object" "client:$1"
   if [[ $1 == 's3' ]]; then
-    error=$(send_command aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4"
-    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(send_command mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+    error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
   else
-    log 2 "'copy-object' not implemented for '$1'"
+    echo "'copy-object' not implemented for '$1'"
     return 1
   fi
   log 5 "copy object exit code: $exit_code"
   if [ $exit_code -ne 0 ]; then
-    log 2 "error copying object to bucket: $error"
+    echo "error copying object to bucket: $error"
     return 1
   fi
   return 0
@@ -45,7 +45,7 @@ copy_object() {
 
 copy_object_empty() {
   record-command "copy-object" "client:s3api"
-  error=$(send_command aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
+  error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
   if [[ $result -eq 0 ]]; then
     log 2 "copy object with empty parameters returned no error"
     return 1

tests/commands/create_bucket.sh

@@ -30,14 +30,14 @@ create_bucket() {
   local error
   log 6 "create bucket"
   if [[ $1 == 's3' ]]; then
-    error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
+    error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
-    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -56,11 +56,11 @@ create_bucket_with_user() {
   fi
   local exit_code=0
   if [[ $1 == "aws" ]] || [[ $1 == "s3api" ]]; then
-    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -80,7 +80,7 @@ create_bucket_object_lock_enabled() {
   fi
 
   local exit_code=0
-  error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
+  error=$(aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
   if [ $exit_code -ne 0 ]; then
     log 2 "error creating bucket: $error"
     return 1

tests/commands/create_multipart_upload.sh

@@ -24,7 +24,7 @@ create_multipart_upload() {
     return 1
   fi
 
-  if ! multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+  if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
     log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
@@ -44,7 +44,7 @@ create_multipart_upload_with_user() {
     return 1
   fi
 
-  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
     log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
@@ -54,7 +54,6 @@ create_multipart_upload_with_user() {
     return 1
   fi
   upload_id="${upload_id//\"/}"
-  echo "$upload_id"
   return 0
 }
 
@@ -66,7 +65,7 @@ create_multipart_upload_params() {
     return 1
   fi
   local multipart_data
-  multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload \
+  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload \
     --bucket "$1" \
     --key "$2" \
     --content-type "$3" \
@@ -97,7 +96,7 @@ create_multipart_upload_custom() {
   done
   log 5 "${*:3}"
   log 5 "aws --no-verify-ssl s3api create-multipart-upload --bucket $1 --key $2 ${*:3}"
-  multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
+  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
   if [[ $result -ne 0 ]]; then
     log 2 "error creating custom multipart data command: $multipart_data"
     return 1
@@ -108,19 +107,3 @@ create_multipart_upload_custom() {
   log 5 "upload id: $upload_id"
   return 0
 }
-
-create_multipart_upload_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'create_multipart_upload_rest' requires bucket name, key"
-    return 1
-  fi
-  if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then
-    log 2 "error creating multipart upload: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")"
-    return 1
-  fi
-  return 0
-}

tests/commands/create_presigned_url.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-create_presigned_url() {
-  if [[ $# -ne 3 ]]; then
-    log 2 "create presigned url function requires command type, bucket, and filename"
-    return 1
-  fi
-
-  local presign_result=0
-  if [[ $1 == 's3api' ]]; then
-    presigned_url=$(send_command aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
-  elif [[ $1 == 's3cmd' ]]; then
-    presigned_url=$(send_command s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
-  elif [[ $1 == 'mc' ]]; then
-    presigned_url_data=$(send_command mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
-    presigned_url="${presigned_url_data#*Share: }"
-  else
-    log 2 "unrecognized command type $1"
-    return 1
-  fi
-  if [[ $presign_result -ne 0 ]]; then
-    log 2 "error generating presigned url: $presigned_url"
-    return 1
-  fi
-  export presigned_url
-  return 0
-}

tests/commands/delete_bucket.sh

@@ -31,13 +31,13 @@ delete_bucket() {
 
   exit_code=0
   if [[ $1 == 's3' ]]; then
-    error=$(send_command aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
-  elif [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
+    error=$(aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
+  elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(send_command mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
+    error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
   else
     log 2 "Invalid command type $1"
     return 1

tests/commands/delete_bucket_policy.sh

@@ -21,12 +21,12 @@ delete_bucket_policy() {
     return 1
   fi
   local delete_result=0
-  if [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+    error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(send_command mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
+    error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
   else
     log 2 "command 'delete bucket policy' not implemented for '$1'"
     return 1
@@ -44,7 +44,7 @@ delete_bucket_policy_with_user() {
     log 2 "'delete bucket policy with user' command requires bucket, username, password"
     return 1
   fi
-  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
+  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
     log 2 "error deleting bucket policy: $delete_bucket_policy_error"
     export delete_bucket_policy_error
     return 1

tests/commands/delete_bucket_tagging.sh

@@ -21,10 +21,10 @@ delete_bucket_tagging() {
     return 1
   fi
   local result
-  if [[ $1 == 's3api' ]]; then
-    tags=$(send_command aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    tags=$(send_command mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
+    tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -43,7 +43,7 @@ delete_bucket_tagging_with_user() {
     log 2 "delete bucket tagging command missing username, password, bucket name"
     return 1
   fi
-  if ! error=$(send_command AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
+  if ! error=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
     log 2 "error deleting bucket tagging with user: $error"
     return 1
   fi

tests/commands/delete_object.sh

@@ -24,15 +24,13 @@ delete_object() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    delete_object_error=$(send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]]; then
-    delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
+    delete_object_error=$(aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    delete_object_error=$(send_command mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
-  elif [[ $1 == 'rest' ]]; then
-    delete_object_rest "$2" "$3" || exit_code=$?
+    delete_object_error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -51,37 +49,13 @@ delete_object_bypass_retention() {
     log 2 "'delete-object with bypass retention' requires bucket, key, user, password"
     return 1
   fi
-  if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
+  if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
     log 2 "error deleting object with bypass retention: $delete_object_error"
     return 1
   fi
   return 0
 }
 
-delete_object_version() {
-  if [[ $# -ne 3 ]]; then
-    log 2 "'delete_object_version' requires bucket, key, version ID"
-    return 1
-  fi
-  if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then
-    log 2 "error deleting object version: $delete_object_error"
-    return 1
-  fi
-  return 0
-}
-
-delete_object_version_bypass_retention() {
-  if [[ $# -ne 3 ]]; then
-    log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID"
-    return 1
-  fi
-  if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then
-    log 2 "error deleting object version with bypass retention: $delete_object_error"
-    return 1
-  fi
-  return 0
-}
-
 delete_object_with_user() {
   record_command "delete-object" "client:$1"
   if [ $# -ne 5 ]; then
@@ -90,11 +64,11 @@ delete_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
   else
     log 2 "command 'delete object with user' not implemented for '$1'"
     return 1
@@ -105,44 +79,4 @@ delete_object_with_user() {
     return 1
   fi
   return 0
-}
-
-delete_object_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'delete_object_rest' requires bucket name, object name"
-    return 1
-  fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="DELETE
-/$1/$2
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER"/delete_object_error.txt 2>&1)
-  if [[ "$reply" != "204" ]]; then
-    log 2 "delete object command returned error: $(cat "$TEST_FILE_FOLDER"/delete_object_error.txt)"
-    return 1
-  fi
-  return 0
 }

tests/commands/delete_object_tagging.sh

@@ -17,65 +17,20 @@
 delete_object_tagging() {
   record_command "delete-object-tagging" "client:$1"
   if [[ $# -ne 3 ]]; then
-    log 2 "delete object tagging command missing command type, bucket, key"
+    echo "delete object tagging command missing command type, bucket, key"
     return 1
   fi
-  delete_result=0
-  if [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(send_command mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
-  elif [ "$1" == 'rest' ]; then
-    delete_object_tagging_rest "$2" "$3" || delete_result=$?
+    error=$(mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
   else
-    log 2 "delete-object-tagging command not implemented for '$1'"
+    echo "delete-object-tagging command not implemented for '$1'"
     return 1
   fi
   if [[ $delete_result -ne 0 ]]; then
-    log 2 "error deleting object tagging: $error"
+    echo "error deleting object tagging: $error"
     return 1
   fi
   return 0
-}
-
-delete_object_tagging_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'delete_object_tagging' requires bucket, key"
-    return 1
-  fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="DELETE
-/$1/$2
-tagging=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" \
-    -d "$tagging" -o "$TEST_FILE_FOLDER"/delete_tagging_error.txt 2>&1)
-  log 5 "reply status code: $reply"
-  if [[ "$reply" != "204" ]]; then
-    log 2 "reply error: $reply"
-    log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/delete_tagging_error.txt)"
-    return 1
-  fi
-  return 0
-}
+}

tests/commands/delete_objects.sh

@@ -20,7 +20,7 @@ delete_objects() {
     log 2 "'delete-objects' command requires bucket name, two object keys"
     return 1
   fi
-  if ! error=$(send_command aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
+  if ! error=$(aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
       \"Objects\": [
         {\"Key\": \"$2\"},
         {\"Key\": \"$3\"}

tests/commands/get_bucket_acl.sh

@@ -21,10 +21,10 @@ get_bucket_acl() {
     return 1
   fi
   local exit_code=0
-  if [[ $1 == 's3api' ]]; then
-    acl=$(send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    acl=$(aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
-    acl=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
+    acl=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
   else
     log 2 "command 'get bucket acl' not implemented for $1"
     return 1
@@ -42,7 +42,7 @@ get_bucket_acl_with_user() {
     log 2 "'get bucket ACL with user' command requires bucket name, username, password"
     return 1
   fi
-  if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
+  if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
     log 2 "error getting bucket ACLs: $bucket_acl"
     return 1
   fi

tests/commands/get_bucket_location.sh

@@ -17,18 +17,17 @@
 get_bucket_location() {
   record_command "get-bucket-location" "client:$1"
   if [[ $# -ne 2 ]]; then
-    log 2 "get bucket location command requires command type, bucket name"
+    echo "get bucket location command requires command type, bucket name"
     return 1
   fi
-  get_result=0
-  if [[ $1 == 's3api' ]]; then
+  if [[ $1 == 'aws' ]]; then
     get_bucket_location_aws "$2" || get_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_location_s3cmd "$2" || get_result=$?
   elif [[ $1 == 'mc' ]]; then
     get_bucket_location_mc "$2" || get_result=$?
   else
-    log 2 "command type '$1' not implemented for get_bucket_location"
+    echo "command type '$1' not implemented for get_bucket_location"
     return 1
   fi
   if [[ $get_result -ne 0 ]]; then
@@ -40,10 +39,10 @@ get_bucket_location() {
 get_bucket_location_aws() {
   record_command "get-bucket-location" "client:s3api"
   if [[ $# -ne 1 ]]; then
-    log 2 "get bucket location (aws) requires bucket name"
+    echo "get bucket location (aws) requires bucket name"
     return 1
   fi
-  location_json=$(send_command aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
+  location_json=$(aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
   if [[ $location_result -ne 0 ]]; then
     echo "error getting bucket location: $location"
     return 1
@@ -58,9 +57,9 @@ get_bucket_location_s3cmd() {
     echo "get bucket location (s3cmd) requires bucket name"
     return 1
   fi
-  info=$(send_command s3cmd --no-check-certificate info "s3://$1") || results=$?
+  info=$(s3cmd --no-check-certificate info "s3://$1") || results=$?
   if [[ $results -ne 0 ]]; then
-    log 2 "error getting bucket location: $location"
+    echo "error getting s3cmd info: $info"
     return 1
   fi
   bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}')
@@ -70,12 +69,12 @@ get_bucket_location_s3cmd() {
 get_bucket_location_mc() {
   record_command "get-bucket-location" "client:mc"
   if [[ $# -ne 1 ]]; then
-    log 2 "get bucket location (mc) requires bucket name"
+    echo "get bucket location (mc) requires bucket name"
     return 1
   fi
-  info=$(send_command mc --insecure stat "$MC_ALIAS/$1") || results=$?
+  info=$(mc --insecure stat "$MC_ALIAS/$1") || results=$?
   if [[ $results -ne 0 ]]; then
-    log 2 "error getting s3cmd info: $info"
+    echo "error getting s3cmd info: $info"
     return 1
   fi
   # shellcheck disable=SC2034

tests/commands/get_bucket_ownership_controls.sh

@@ -26,7 +26,7 @@ get_bucket_ownership_controls() {
     return 1
   fi
 
-  if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
+  if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
     log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
     return 1
   fi

tests/commands/get_bucket_policy.sh

@@ -21,7 +21,7 @@ get_bucket_policy() {
     return 1
   fi
   local get_bucket_policy_result=0
-  if [[ $1 == 's3api' ]]; then
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
     get_bucket_policy_aws "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
@@ -44,7 +44,7 @@ get_bucket_policy_aws() {
     log 2 "aws 'get bucket policy' command requires bucket"
     return 1
   fi
-  policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
+  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
   policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
   log 5 "$policy_json"
   if [[ $get_result -ne 0 ]]; then
@@ -66,7 +66,7 @@ get_bucket_policy_with_user() {
     log 2 "'get bucket policy with user' command requires bucket, username, password"
     return 1
   fi
-  if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
+  if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
     policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
     bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
   else
@@ -87,7 +87,7 @@ get_bucket_policy_s3cmd() {
     return 1
   fi
 
-  if ! info=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
+  if ! info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
     log 2 "error getting bucket policy: $info"
     return 1
   fi
@@ -97,57 +97,41 @@ get_bucket_policy_s3cmd() {
   policy_brackets=false
   # NOTE:  versitygw sends policies back in multiple lines here, direct in single line
   while IFS= read -r line; do
-    if check_and_load_policy_info; then
-      break
+    if [[ $policy_brackets == false ]]; then
+      policy_line=$(echo "$line" | grep 'Policy: ')
+      if [[ $policy_line != "" ]]; then
+        if [[ $policy_line != *'{'* ]]; then
+          break
+        fi
+        if [[ $policy_line == *'}'* ]]; then
+          log 5 "policy on single line"
+          bucket_policy=${policy_line//Policy:/}
+          break
+        else
+          policy_brackets=true
+          bucket_policy+="{"
+        fi
+      fi
+    else
+      bucket_policy+=$line
+      if [[ $line == "" ]]; then
+        break
+      fi
     fi
   done <<< "$info"
   log 5 "bucket policy: $bucket_policy"
   return 0
 }
 
-# return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading
-check_and_load_policy_info() {
-  if [[ $policy_brackets == false ]]; then
-    if search_for_first_policy_line_or_full_policy; then
-      return 0
-    fi
-  else
-    bucket_policy+=$line
-    if [[ $line == "}" ]]; then
-      return 0
-    fi
-  fi
-  return 1
-}
-
-# return 0 for empty or single-line policy, 1 for other cases
-search_for_first_policy_line_or_full_policy() {
-  policy_line=$(echo "$line" | grep 'Policy: ')
-  if [[ $policy_line != "" ]]; then
-    if [[ $policy_line != *'{'* ]]; then
-      return 0
-    fi
-    if [[ $policy_line == *'}'* ]]; then
-      log 5 "policy on single line"
-      bucket_policy=${policy_line//Policy:/}
-      return 0
-    else
-      policy_brackets=true
-      bucket_policy+="{"
-    fi
-  fi
-  return 1
-}
-
 get_bucket_policy_mc() {
   record_command "get-bucket-policy" "client:mc"
   if [[ $# -ne 1 ]]; then
-    log 2 "aws 'get bucket policy' command requires bucket"
+    echo "aws 'get bucket policy' command requires bucket"
     return 1
   fi
-  bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
+  bucket_policy=$(mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
   if [[ $get_result -ne 0 ]]; then
-    log 2 "error getting policy: $bucket_policy"
+    echo "error getting policy: $bucket_policy"
     return 1
   fi
   return 0

tests/commands/get_bucket_tagging.sh

@@ -21,10 +21,10 @@ get_bucket_tagging() {
   assert [ $# -eq 2 ]
   record_command "get-bucket-tagging" "client:$1"
   local result
-  if [[ $1 == 's3api' ]]; then
-    tags=$(send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
   else
     fail "invalid command type $1"
   fi
@@ -35,7 +35,7 @@ get_bucket_tagging() {
       export tags=
       return 0
     fi
-    log 2 "error getting bucket tags: $tags"
+    echo "error getting bucket tags: $tags"
     return 1
   fi
   export tags
@@ -49,7 +49,7 @@ get_bucket_tagging_with_user() {
   fi
   record_command "get-bucket-tagging" "client:s3api"
   local result
-  if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then
+  if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then
     log 5 "tags error: $tags"
     if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
       export tags=

tests/commands/get_bucket_versioning.sh

@@ -17,32 +17,15 @@
 get_bucket_versioning() {
   record_command "get-bucket-versioning" "client:s3api"
   if [[ $# -ne 2 ]]; then
-    log 2 "get bucket versioning command requires command type, bucket name"
+    log 2 "put bucket versioning command requires command type, bucket name"
     return 1
   fi
   local get_result=0
   if [[ $1 == 's3api' ]]; then
-    versioning=$(send_command aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
+    error=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
   fi
   if [[ $get_result -ne 0 ]]; then
-    log 2 "error getting bucket versioning: $versioning"
-    return 1
-  fi
-  return 0
-}
-
-get_bucket_versioning_rest() {
-  log 6 "get_object_rest"
-  if [ $# -ne 1 ]; then
-    log 2 "'get_bucket_versioning_rest' requires bucket name"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OUTPUT_FILE="$TEST_FILE_FOLDER/versioning.txt" ./tests/rest_scripts/get_bucket_versioning.sh); then
-    log 2 "error getting bucket versioning: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "get-bucket-versioning returned code $result: $(cat "$TEST_FILE_FOLDER/versioning.txt")"
+    log 2 "error getting bucket versioning: $error"
     return 1
   fi
   return 0

tests/commands/get_object.sh

@@ -23,15 +23,13 @@ get_object() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    get_object_error=$(send_command aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]]; then
-    get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
-  elif [[ $1 == 'rest' ]]; then
-    get_object_rest "$2" "$3" "$4" || exit_code=$?
+    get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
   else
     log 2 "'get object' command not implemented for '$1'"
     return 1
@@ -50,7 +48,8 @@ get_object_with_range() {
     log 2 "'get object with range' requires bucket, key, range, outfile"
     return 1
   fi
-  if ! get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1); then
+  get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1) || local exit_code=$?
+  if [[ $exit_code -ne 0 ]]; then
     log 2 "error getting object with range: $get_object_error"
     return 1
   fi
@@ -66,13 +65,13 @@ get_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]] || [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
     log 5 "s3cmd filename: $3"
-    get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
     log 5 "save location: $4"
-    get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
   else
     log 2 "'get_object_with_user' not implemented for client '$1'"
     return 1
@@ -84,45 +83,3 @@ get_object_with_user() {
   fi
   return 0
 }
-
-get_object_rest() {
-  log 6 "get_object_rest"
-  if [ $# -ne 3 ]; then
-    log 2 "'get_object_rest' requires bucket name, object name, output file"
-    return 1
-  fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1/$2
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1/$2" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$3" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$3")"
-    return 1
-  fi
-  return 0
-}

tests/commands/get_object_attributes.sh

@@ -20,7 +20,7 @@ get_object_attributes() {
     log 2 "'get object attributes' command requires bucket, key"
     return 1
   fi
-  attributes=$(send_command aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$?
+  attributes=$(aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$?
   if [[ $get_result -ne 0 ]]; then
     log 2 "error getting object attributes: $attributes"
     return 1

tests/commands/get_object_legal_hold.sh

@@ -20,26 +20,10 @@ get_object_legal_hold() {
     return 1
   fi
   record_command "get-object-legal-hold" "client:s3api"
-  legal_hold=$(send_command aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
+  legal_hold=$(aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
   if [[ $get_result -ne 0 ]]; then
     log 2 "error getting object legal hold: $legal_hold"
     return 1
   fi
   return 0
-}
-
-get_object_legal_hold_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'get_object_legal_hold_rest' requires bucket, key"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/legal_hold.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then
-    log 2 "error getting object legal hold: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")"
-    return 1
-  fi
-  return 0
 }

tests/commands/get_object_lock_configuration.sh

@@ -20,7 +20,7 @@ get_object_lock_configuration() {
     log 2 "'get object lock configuration' command missing bucket name"
     return 1
   fi
-  if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
+  if ! lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
     log 2 "error obtaining lock config: $lock_config"
     # shellcheck disable=SC2034
     get_object_lock_config_err=$lock_config
@@ -28,44 +28,4 @@ get_object_lock_configuration() {
   fi
   lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning")
   return 0
-}
-
-get_object_lock_configuration_rest() {
-  log 6 "get_object_lock_configuration_rest"
-  if [ $# -ne 1 ]; then
-    log 2 "'get_object_lock_configuration_rest' requires bucket name"
-    return 1
-  fi
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-object-lock=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
-    return 1
-  fi
-  return 0
 }

tests/commands/get_object_retention.sh

@@ -20,57 +20,11 @@ get_object_retention() {
     log 2 "'get object retention' command requires bucket, key"
     return 1
   fi
-  if ! retention=$(send_command aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then
+  if ! retention=$(aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then
     log 2 "error getting object retention: $retention"
     get_object_retention_error=$retention
     export get_object_retention_error
     return 1
   fi
   return 0
-}
-
-get_object_retention_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'get_object_tagging_rest' requires bucket, key"
-    return 1
-  fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1/$2
-retention=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?retention" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER"/object_retention.txt 2>&1)
-  log 5 "reply status code: $reply"
-  if [[ "$reply" != "200" ]]; then
-    if [ "$reply" == "404" ]; then
-      return 1
-    fi
-    log 2 "reply error: $reply"
-    log 2 "get object retention command returned error: $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
-    return 2
-  fi
-  log 5 "object tags:  $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
-  return 0
-}
+}

tests/commands/get_object_tagging.sh

@@ -21,12 +21,10 @@ get_object_tagging() {
     return 1
   fi
   local result
-  if [[ $1 == 's3api' ]]; then
-    tags=$(send_command aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
-  elif [[ "$1" == 'mc' ]]; then
-    tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
-  elif [ "$1" == 'rest' ]; then
-    get_object_tagging_rest "$2" "$3" || result=$?
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -43,50 +41,4 @@ get_object_tagging() {
     tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
   fi
   export tags
-}
-
-get_object_tagging_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'get_object_tagging' requires bucket, key"
-    return 1
-  fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1/$2
-tagging=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?tagging" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER"/object_tags.txt 2>&1)
-  log 5 "reply status code: $reply"
-  if [[ "$reply" != "200" ]]; then
-    if [ "$reply" == "404" ]; then
-      return 1
-    fi
-    log 2 "reply error: $reply"
-    log 2 "get object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/object_tags.txt)"
-    return 2
-  fi
-  log 5 "object tags:  $(cat "$TEST_FILE_FOLDER"/object_tags.txt)"
-  return 0
 }

tests/commands/head_bucket.sh

@@ -29,22 +29,21 @@ head_bucket() {
     return 1
   fi
   local exit_code=0
-  if [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
-    bucket_info=$(send_command aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
+  if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+    bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    bucket_info=$(send_command s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
+    bucket_info=$(s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
-    log 2 "invalid command type $1"
+    fail "invalid command type $1"
   fi
   if [ $exit_code -ne 0 ]; then
-    log 2 "error getting bucket info: $bucket_info"
     if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
       return 1
     fi
+    log 2 "error getting bucket info: $bucket_info"
     return 2
   fi
-  echo "$bucket_info"
   return 0
 }

tests/commands/head_object.sh

@@ -21,12 +21,12 @@ head_object() {
     return 2
   fi
   local exit_code=0
-  if [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
-    metadata=$(send_command aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?"
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+    metadata=$(aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
-    metadata=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?"
+    metadata=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?"
   elif [[ $1 == 'mc' ]]; then
-    metadata=$(send_command mc --insecure stat "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+    metadata=$(mc --insecure stat "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 2

tests/commands/list_buckets.sh

@@ -18,27 +18,27 @@ list_buckets() {
   log 6 "list_buckets"
   record_command "list-buckets" "client:$1"
   if [ $# -ne 1 ]; then
-    log 2 "list buckets command missing command type"
+    echo "list buckets command missing command type"
     return 1
   fi
 
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    buckets=$(send_command aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
-  elif [[ $1 == 's3api' ]]; then
+    buckets=$(aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
     list_buckets_s3api "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    buckets=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
+    buckets=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
+    buckets=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
   elif [[ $1 == 'rest' ]]; then
     list_buckets_rest || exit_code=$?
   else
-    log 2 "list buckets command not implemented for '$1'"
+    echo "list buckets command not implemented for '$1'"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    log 2 "error listing buckets: $buckets"
+    echo "error listing buckets: $buckets"
     return 1
   fi
 
@@ -57,25 +57,25 @@ list_buckets() {
 list_buckets_with_user() {
   record_command "list-buckets" "client:$1"
   if [ $# -ne 3 ]; then
-    log 2 "'list buckets as user' command missing command type, username, password"
+    echo "'list buckets as user' command missing command type, username, password"
     return 1
   fi
 
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    buckets=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
+    buckets=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
     list_buckets_s3api "$2" "$3" || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    buckets=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$2" --secret_key="$3" ls s3:// 2>&1) || exit_code=$?
+    buckets=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$2" --secret_key="$3" ls s3:// 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
+    buckets=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
   else
-    log 2 "list buckets command not implemented for '$1'"
+    echo "list buckets command not implemented for '$1'"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    log 2 "error listing buckets: $buckets"
+    echo "error listing buckets: $buckets"
     return 1
   fi
 
@@ -96,8 +96,8 @@ list_buckets_s3api() {
     log 2 "'list_buckets_s3api' requires username, password"
     return 1
   fi
-  if ! output=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api list-buckets 2>&1); then
-    log 2 "error listing buckets: $output"
+  if ! output=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api list-buckets 2>&1); then
+    echo "error listing buckets: $output"
     return 1
   fi
   log 5 "bucket data: $output"
@@ -117,14 +117,30 @@ list_buckets_s3api() {
 }
 
 list_buckets_rest() {
-  if ! result=$(COMMAND_LOG=$COMMAND_LOG OUTPUT_FILE="$TEST_FILE_FOLDER/buckets.txt" ./tests/rest_scripts/list_buckets.sh); then
-    log 2 "error listing buckets: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "list-buckets returned code $result: $(cat "$TEST_FILE_FOLDER/buckets.txt")"
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/
+
+host:${AWS_ENDPOINT_URL#*//}
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
     return 1
   fi
+
+  get_signature
+  # shellcheck disable=SC2034,SC2154
+  reply=$(curl -ks "$AWS_ENDPOINT_URL" \
+         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+         -H "x-amz-content-sha256: $payload_hash" \
+         -H "x-amz-date: $current_date_time" 2>&1)
   parse_bucket_list
-  return 0
 }

tests/commands/list_multipart_uploads.sh

@@ -20,7 +20,7 @@ list_multipart_uploads() {
     log 2 "'list multipart uploads' command requires bucket name"
     return 1
   fi
-  if ! uploads=$(send_command aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+  if ! uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
     log 2 "error listing uploads: $uploads"
     return 1
   fi
@@ -32,7 +32,7 @@ list_multipart_uploads_with_user() {
     log 2 "'list multipart uploads' command requires bucket name, username, password"
     return 1
   fi
-  if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+  if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
     log 2 "error listing uploads: $uploads"
     # shellcheck disable=SC2034
     list_multipart_uploads_error=$uploads

tests/commands/list_object_versions.sh

@@ -20,44 +20,10 @@ list_object_versions() {
     log 2 "'list object versions' command requires bucket name"
     return 1
   fi
-  versions=$(send_command aws --no-verify-ssl s3api list-object-versions --bucket "$1" 2>&1) || local list_result=$?
+  versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1") || local list_result=$?
   if [[ $list_result -ne 0 ]]; then
     log 2 "error listing object versions: $versions"
     return 1
   fi
-  versions=$(echo "$versions" | grep -v "InsecureRequestWarning")
   return 0
-}
-
-list_object_versions_rest() {
-  if [ $# -ne 1 ]; then
-    log 2 "'list_object_versions_rest' requires bucket name"
-    return 1
-  fi
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-versions=
-host:${AWS_ENDPOINT_URL#*//}
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-
-  get_signature
-  # shellcheck disable=SC2034,SC2154
-  reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \
-         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-         -H "x-amz-content-sha256: $payload_hash" \
-         -H "x-amz-date: $current_date_time" \
-         -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1)
 }

tests/commands/list_objects.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
-source ./tests/util/util_list_objects.sh
-source ./tests/commands/command.sh
+source ./tests/util_list_objects.sh
 
 # Copyright 2024 Versity Software
 # This file is licensed under the Apache License, Version 2.0
@@ -29,15 +28,15 @@ list_objects() {
 
   local output
   local result=0
-  if [[ $1 == 's3' ]]; then
-    output=$(send_command aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$?
+  if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then
+    output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$?
   elif [[ $1 == 's3api' ]]; then
     list_objects_s3api "$2" || result=$?
     return $result
   elif [[ $1 == 's3cmd' ]]; then
-    output=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$?
+    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    output=$(send_command mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$?
+    output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$?
   elif [[ $1 == 'rest' ]]; then
     list_objects_rest "$2" || result=$?
     return $result
@@ -67,7 +66,7 @@ list_objects_s3api() {
     log 2 "'list_objects_s3api' requires bucket"
     return 1
   fi
-  if ! output=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then
+  if ! output=$(aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then
     log 2 "error listing objects: $output"
     return 1
   fi
@@ -91,16 +90,16 @@ list_objects_s3api() {
 # export objects on success, return 1 for failure
 list_objects_s3api_v1() {
   if [ $# -lt 1 ] || [ $# -gt 2 ]; then
-    log 2 "list objects command requires bucket, (optional) delimiter"
+    echo "list objects command requires bucket, (optional) delimiter"
     return 1
   fi
   if [ "$2" == "" ]; then
-    objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
   else
-    objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
   fi
   if [[ $result -ne 0 ]]; then
-    log 2 "error listing objects: $objects"
+    echo "error listing objects: $objects"
     return 1
   fi
   export objects
@@ -113,13 +112,13 @@ list_objects_with_prefix() {
   fi
   local result=0
   if [ "$1" == 's3' ]; then
-    objects=$(send_command aws --no-verify-ssl s3 ls s3://"$2/$3" 2>&1) || result=$?
+    objects=$(aws --no-verify-ssl s3 ls s3://"$2/$3" 2>&1) || result=$?
   elif [ "$1" == 's3api' ]; then
-    objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$2" --prefix "$3" 2>&1) || result=$?
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$2" --prefix "$3" 2>&1) || result=$?
   elif [ "$1" == 's3cmd' ]; then
-    objects=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || result=$?
+    objects=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || result=$?
   elif [[ "$1" == 'mc' ]]; then
-    objects=$(send_command mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || result=$?
+    objects=$(mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || result=$?
   else
     log 2 "invalid command type '$1'"
     return 1
@@ -163,7 +162,7 @@ $payload_hash"
   fi
   get_signature
   # shellcheck disable=SC2154
-  reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \
+  reply=$(curl -ks "$header://$aws_endpoint_url_address/$1" \
     -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
     -H "x-amz-content-sha256: $payload_hash" \
     -H "x-amz-date: $current_date_time" 2>&1)

tests/commands/list_objects_v2.sh

@@ -19,13 +19,13 @@
 # export objects on success, return 1 for failure
 list_objects_v2() {
   if [ $# -ne 1 ]; then
-    log 2 "list objects command missing bucket and/or path"
+    echo "list objects command missing bucket and/or path"
     return 1
   fi
   record_command "list-objects-v2 client:s3api"
-  objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$?
+  objects=$(aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$?
   if [[ $result -ne 0 ]]; then
-    log 2 "error listing objects: $objects"
+    echo "error listing objects: $objects"
     return 1
   fi
 }

tests/commands/list_parts.sh

@@ -20,7 +20,7 @@ list_parts() {
     return 1
   fi
   record_command "list-parts" "client:s3api"
-  if ! listed_parts=$(send_command aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! listed_parts=$(aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error listing multipart upload parts: $listed_parts"
     return 1
   fi
@@ -32,7 +32,7 @@ list_parts_with_user() {
     return 1
   fi
   record_command 'list-parts' 'client:s3api'
-  if ! listed_parts=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api list-parts --bucket "$3" --key "$4" --upload-id "$5" 2>&1); then
+  if ! listed_parts=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api list-parts --bucket "$3" --key "$4" --upload-id "$5" 2>&1); then
     log 2 "Error listing multipart upload parts: $listed_parts"
     return 1
   fi

tests/commands/put_bucket_acl.sh

@@ -14,8 +14,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-source ./tests/util/util_file.sh
-source ./tests/commands/command.sh
+source ./tests/util_file.sh
 
 put_bucket_acl_s3api() {
   log 6 "put_bucket_acl_s3api"
@@ -25,7 +24,7 @@ put_bucket_acl_s3api() {
     return 1
   fi
   log 5 "bucket name: $1, acls: $2"
-  if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
+  if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
     log 2 "error putting bucket acl: $error"
     return 1
   fi
@@ -40,7 +39,7 @@ put_bucket_acl_s3api_with_user() {
     return 1
   fi
   log 5 "bucket name: $1, acls: $2"
-  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
+  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
     log 2 "error putting bucket acl: $error"
     return 1
   fi
@@ -88,7 +87,7 @@ put_bucket_canned_acl_s3cmd() {
     log 2 "put bucket acl command requires bucket name, permission"
     return 1
   fi
-  if ! error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$1" "$2" 2>&1); then
+  if ! error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$1" "$2" 2>&1); then
     log 2 "error putting s3cmd canned ACL:  $error"
     return 1
   fi
@@ -101,7 +100,7 @@ put_bucket_canned_acl() {
     return 1
   fi
   record_command "put-bucket-acl" "client:s3api"
-  if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
+  if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
     log 2 "error re-setting bucket acls: $error"
     return 1
   fi
@@ -114,7 +113,7 @@ put_bucket_canned_acl_with_user() {
     return 1
   fi
   record_command "put-bucket-acl" "client:s3api"
-  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
+  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
     log 2 "error re-setting bucket acls: $error"
     return 1
   fi

tests/commands/put_bucket_ownership_controls.sh

@@ -22,13 +22,9 @@ put_bucket_ownership_controls() {
   fi
 
   log 6 "put_bucket_ownership_controls"
-  if [ $# -ne 2 ]; then
-    log 2 "'put_bucket_ownership_controls' requires bucket name, rule"
-    return 1
-  fi
   record_command "put-bucket-ownership-controls" "client:s3api"
-  if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" --ownership-controls="Rules=[{ObjectOwnership=$2}]" 2>&1); then
-    log 2 "error putting bucket ownership controls: $error"
-    return 1
-  fi
+  assert [ $# -eq 2 ]
+  run aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" --ownership-controls="Rules=[{ObjectOwnership=$2}]"
+  # shellcheck disable=SC2154
+  assert_success "error putting bucket ownership controls: $output"
 }

tests/commands/put_bucket_policy.sh

@@ -21,12 +21,12 @@ put_bucket_policy() {
     return 1
   fi
   local put_policy_result=0
-  if [[ $1 == 's3api' ]]; then
-    policy=$(send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$?
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    policy=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$?
+    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
-    policy=$(send_command mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$?
+    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$?
   else
     log 2 "command 'put bucket policy' not implemented for '$1'"
     return 1
@@ -46,7 +46,7 @@ put_bucket_policy_with_user() {
     log 2 "'put bucket policy with user' command requires bucket, policy file, username, password"
     return 1
   fi
-  if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then
+  if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then
     log 2 "error putting bucket policy with user $3: $policy"
     put_bucket_policy_error=$policy
     export put_bucket_policy_error

tests/commands/put_bucket_tagging.sh

@@ -23,10 +23,10 @@ put_bucket_tagging() {
   local error
   local result=0
   record_command "put-bucket-tagging" "client:$1"
-  if [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(send_command mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$?
+    error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -42,7 +42,7 @@ put_bucket_tagging_with_user() {
   log 6 "put_bucket_tagging_with_user"
   assert [ $# -eq 5 ]
   record_command "put-bucket-tagging" "client:$1"
-  if ! error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]"); then
+  if ! error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]"); then
     log 2 "error putting bucket tagging: $error"
     return 1
   fi

tests/commands/put_bucket_versioning.sh

@@ -22,27 +22,11 @@ put_bucket_versioning() {
   fi
   local put_result=0
   if [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api put-bucket-versioning --bucket "$2" --versioning-configuration "{ \"Status\": \"$3\"}" 2>&1) || put_result=$?
+    error=$(aws --no-verify-ssl s3api put-bucket-versioning --bucket "$2" --versioning-configuration "{ \"Status\": \"$3\"}" 2>&1) || put_result=$?
   fi
   if [[ $put_result -ne 0 ]]; then
     log 2 "error putting bucket versioning: $error"
     return 1
   fi
   return 0
-}
-
-put_bucket_versioning_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'put_bucket_versioning_rest' requires bucket, 'Enabled' or 'Suspended'"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 STATUS="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/error.txt" ./tests/rest_scripts/put_bucket_versioning.sh); then
-    log 2 "error putting bucket versioning: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "put-bucket-versioning returned code $result: $(cat "$TEST_FILE_FOLDER/error.txt")"
-    return 1
-  fi
-  return 0
 }

tests/commands/put_object.sh

@@ -26,15 +26,13 @@ put_object() {
   local exit_code=0
   local error
   if [[ $1 == 's3' ]]; then
-    error=$(send_command aws --no-verify-ssl s3 mv "$2" s3://"$3/$4" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    error=$(aws --no-verify-ssl s3 mv "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(send_command mc --insecure put "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
-  elif [[ $1 == 'rest' ]]; then
-    put_object_rest "$2" "$3" "$4" || exit_code=$?
+    error=$(mc --insecure put "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
   else
     log 2 "'put object' command not implemented for '$1'"
     return 1
@@ -55,7 +53,7 @@ put_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    put_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" send_command aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    put_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   else
     log 2 "'put object with user' command not implemented for '$1'"
     return 1
@@ -68,19 +66,3 @@ put_object_with_user() {
   fi
   return 0
 }
-
-put_object_rest() {
-  if [ $# -ne 3 ]; then
-    log 2 "'put_object_rest' requires local file, bucket name, key"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
-    log 2 "error sending object file: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "expected response code of '200', was '$result' (output: $(cat "$TEST_FILE_FOLDER/result.txt")"
-    return 1
-  fi
-  return 0
-}

tests/commands/put_object_legal_hold.sh

@@ -21,7 +21,7 @@ put_object_legal_hold() {
     return 1
   fi
   local error=""
-  error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1) || local put_hold_result=$?
+  error=$(aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1) || local put_hold_result=$?
   if [[ $put_hold_result -ne 0 ]]; then
     log 2 "error putting object legal hold: $error"
     return 1

tests/commands/put_object_lock_configuration.sh

@@ -20,7 +20,7 @@ put_object_lock_configuration() {
     return 1
   fi
   local config="{\"ObjectLockEnabled\": \"$2\", \"Rule\": {\"DefaultRetention\": {\"Mode\": \"$3\", \"Days\": $4}}}"
-  if ! error=$(send_command aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
+  if ! error=$(aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
     log 2 "error putting object lock configuration: $error"
     return 1
   fi
@@ -33,7 +33,7 @@ put_object_lock_configuration_disabled() {
     return 1
   fi
   local config="{\"ObjectLockEnabled\": \"Enabled\"}"
-  if ! error=$(send_command aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
+  if ! error=$(aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
     log 2 "error putting object lock configuration: $error"
     return 1
   fi

tests/commands/put_object_retention.sh

@@ -20,26 +20,10 @@ put_object_retention() {
     log 2 "'put object retention' command requires bucket, key, retention mode, retention date"
     return 1
   fi
-  error=$(send_command aws --no-verify-ssl s3api put-object-retention --bucket "$1" --key "$2" --retention "{\"Mode\": \"$3\", \"RetainUntilDate\": \"$4\"}" 2>&1) || local put_result=$?
+  error=$(aws --no-verify-ssl s3api put-object-retention --bucket "$1" --key "$2" --retention "{\"Mode\": \"$3\", \"RetainUntilDate\": \"$4\"}" 2>&1) || local put_result=$?
   if [[ $put_result -ne 0 ]]; then
     log 2 "error putting object retention:  $error"
     return 1
   fi
   return 0
-}
-
-put_object_retention_rest() {
-  if [ $# -ne 4 ]; then
-    log 2 "'put_object_retention_rest' requires bucket, key, retention mode, retention date"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME="$1" OBJECT_KEY="$2" RETENTION_MODE="$3" RETAIN_UNTIL_DATE="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/error.txt" ./tests/rest_scripts/put_object_retention.sh); then
-    log 2 "error putting object retention: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/error.txt")"
-    return 1
-  fi
-  return 0
-}
+}

tests/commands/put_object_tagging.sh

@@ -16,18 +16,16 @@
 
 put_object_tagging() {
   if [ $# -ne 5 ]; then
-    log 2 "'put-object-tagging' command missing command type, bucket, object name, file, key, and/or value"
+    log 2 "'put-object-tagging' command missing command type, object name, file, key, and/or value"
     return 1
   fi
   local error
   local result
   record_command "put-object-tagging" "client:$1"
-  if [[ $1 == 's3api' ]]; then
-    error=$(send_command aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(send_command mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$?
-  elif [[ $1 == 'rest' ]]; then
-    put_object_tagging_rest "$2" "$3" "$4" "$5" || result=$?
+    error=$(mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -37,56 +35,4 @@ put_object_tagging() {
     return 1
   fi
   return 0
-}
-
-put_object_tagging_rest() {
-  if [ $# -ne 4 ]; then
-    log 2 "'put_object_tagging' requires bucket, key, tag key, tag value"
-    return 1
-  fi
-
-  tagging="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
-<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
-  <TagSet>
-    <Tag>
-      <Key>$3</Key>
-      <Value>$4</Value>
-    </Tag>
-  </TagSet>
-</Tagging>"
-
-  generate_hash_for_payload "$tagging"
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="PUT
-/$1/$2
-tagging=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?tagging" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" \
-    -d "$tagging" -o "$TEST_FILE_FOLDER"/put_tagging_error.txt 2>&1)
-  log 5 "reply status code: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "reply error: $reply"
-    log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/put_tagging_error.txt)"
-    return 1
-  fi
-  return 0
-}
+}

tests/commands/put_public_access_block.sh

@@ -19,7 +19,7 @@ put_public_access_block() {
     log 2 "'put_public_access_block' command requires bucket, access block list"
     return 1
   fi
-  if ! error=$(send_command aws --no-verify-ssl s3api put-public-access-block --bucket "$1" --public-access-block-configuration "$2"); then
+  if ! error=$(aws --no-verify-ssl s3api put-public-access-block --bucket "$1" --public-access-block-configuration "$2"); then
     log 2 "error updating public access block: $error"
     return 1
   fi

tests/commands/select_object_content.sh

@@ -20,7 +20,7 @@ select_object_content() {
     log 2 "'select object content' command requires bucket, key, expression, expression type, input serialization, output serialization, outfile"
     return 1
   fi
-  error=$(send_command aws --no-verify-ssl s3api select-object-content \
+  error=$(aws --no-verify-ssl s3api select-object-content \
     --bucket "$1" \
     --key "$2" \
     --expression "$3" \

tests/commands/upload_part.sh

@@ -21,7 +21,7 @@ upload_part() {
   fi
   local etag_json
   record_command "upload-part" "client:s3api"
-  if ! etag_json=$(send_command aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then
+  if ! etag_json=$(aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then
     log 2 "Error uploading part $5: $etag_json"
     return 1
   fi
@@ -30,23 +30,4 @@ upload_part() {
     return 1
   fi
   export etag
-}
-
-upload_part_and_get_etag_rest() {
-  if [ $# -ne 5 ]; then
-    log 2 "'upload_part_rest' requires bucket name, key, part number, upload ID, part"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="$4" UPLOAD_ID="$3" DATA_FILE="$5" OUTPUT_FILE="$TEST_FILE_FOLDER/etag.txt" ./tests/rest_scripts/upload_part.sh); then
-    log 2 "error sending upload-part REST command: $result"
-    return 1
-  fi
-  if [[ "$result" != "200" ]]; then
-    log 2 "upload-part command returned error $result: $(cat "$TEST_FILE_FOLDER/etag.txt")"
-    return 1
-  fi
-  log 5 "$(cat "$TEST_FILE_FOLDER/etag.txt")"
-  etag=$(grep -i "etag" "$TEST_FILE_FOLDER/etag.txt" | awk '{print $2}' | tr -d '\r')
-  log 5 "etag:  $etag"
-  return 0
-}
+}

tests/commands/upload_part_copy.sh

@@ -17,14 +17,14 @@
 upload_part_copy() {
   record_command "upload-part-copy" "client:s3api"
   if [ $# -ne 5 ]; then
-    log 2 "upload multipart part copy function must have bucket, key, upload ID, file name, part number"
+    echo "upload multipart part copy function must have bucket, key, upload ID, file name, part number"
     return 1
   fi
   local etag_json
-  log 5 "parameters:  $1 $2 $3 $4 $5"
-  etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
+  echo "$1 $2 $3 $4 $5"
+  etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
   if [[ $uploaded -ne 0 ]]; then
-    log 2 "Error uploading part $5: $etag_json"
+    echo "Error uploading part $5: $etag_json"
     return 1
   fi
   etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag')
@@ -39,7 +39,7 @@ upload_part_copy_with_range() {
   fi
   local etag_json
   log 5 "bucket: $1, key: $2, upload ID: $3, file name: $4, range: $5, copy source range: $6"
-  etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$?
+  etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$?
   if [[ $uploaded -ne 0 ]]; then
     log 2 "Error uploading part $5: $etag_json"
     export upload_part_copy_error=$etag_json

tests/docker-compose-bats.yml

@@ -1,36 +1,34 @@
 services:
   no_certs:
     build:
-      dockerfile: tests/Dockerfile_test_bats
+      context: .
+      dockerfile: Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.nocerts
   static_buckets:
     build:
-      dockerfile: tests/Dockerfile_test_bats
+      context: .
+      dockerfile: Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.static
   posix_backend:
     build:
-      dockerfile: tests/Dockerfile_test_bats
+      context: .
+      dockerfile: Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.default
     image: bats_test
   s3_backend:
     build:
-      dockerfile: tests/Dockerfile_test_bats
+      context: .
+      dockerfile: Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.s3
         - SECRETS_FILE=tests/.secrets.s3
-  s3api_np_only:
-    build:
-      dockerfile: tests/Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.default
-    image: bats_test
-    command: ["s3api-bucket,s3api-object"]
   direct:
     build:
-      dockerfile: tests/Dockerfile_direct
+      context: .
+      dockerfile: Dockerfile_direct
     volumes:
       - ./.env.direct:/home/tester/tests/.env.direct
       - ./.secrets.direct:/home/tester/tests/.secrets.direct

tests/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   posix:
     build: 
       context: .
-      dockerfile: tests/Dockerfile.dev
+      dockerfile: ./Dockerfile.dev
       args:
         - IAM_DIR=${IAM_DIR}
         - SETUP_DIR=${SETUP_DIR}
@@ -14,7 +14,7 @@ services:
   proxy:
     build: 
       context: .
-      dockerfile: tests/Dockerfile.dev
+      dockerfile: ./Dockerfile.dev
     volumes:
       - ./:/app
     ports:
@@ -34,9 +34,9 @@ services:
   azuritegw:
     build:
       context: .
-      dockerfile: tests/Dockerfile.dev
+      dockerfile: ./Dockerfile.dev
     volumes:
       - ./:/app
     ports:
       - 7070:7070
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]