Merge pull request #918 from versity/ben/azure_listing

fix: azure list multipart uploads test failures

.github/workflows/azurite.yml

@@ -0,0 +1,37 @@
+name: azurite functional tests
+
+on: pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+              
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Set up Docker Compose
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
+
+      - name: Wait for Azurite to be ready
+        run: sleep 40
+
+      - name: Get Dependencies
+        run: |
+          go mod download
+  
+      - name: Build and Run
+        run: |
+          make
+          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
+  
+      - name: Shut down services
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/docker-bats.yaml

@@ -1,28 +0,0 @@
-name: docker bats tests
-
-on: pull_request
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build Docker Image
-        run: |
-          mv tests/.env.docker.default tests/.env.docker
-          mv tests/.secrets.default tests/.secrets
-          docker build --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
-            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" --build-arg="MC_FOLDER=linux-amd64" \
-            --progress=plain -f tests/Dockerfile_test_bats -t bats_test .
-
-      - name: Set up Docker Compose
-        run: sudo apt-get install -y docker-compose
-
-      - name: Run Docker Container
-        run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from posix_backend posix_backend

.github/workflows/docker-bats.yml

@@ -0,0 +1,28 @@
+name: docker bats tests
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        run: |
+          cp tests/.env.docker.default tests/.env.docker
+          cp tests/.secrets.default tests/.secrets
+          docker build \
+            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
+            --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain \
+            -f tests/Dockerfile_test_bats \
+            -t bats_test .
+
+      - name: Run Docker Container
+        run: |
+          docker compose -f tests/docker-compose-bats.yml --project-directory . \
+          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/functional.yml

@@ -1,7 +1,8 @@
 name: functional tests
-on: pull_request
-jobs:
 
+on: pull_request
+
+jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
@@ -18,7 +19,7 @@ jobs:
 
     - name: Get Dependencies
       run: |
-        go get -v -t -d ./...
+        go mod download
 
     - name: Build and Run
       run: |

.github/workflows/system.yml

@@ -30,7 +30,7 @@ jobs:
             RECREATE_BUCKETS: "true"
             PORT: 7071
             BACKEND: "posix"
-          - set: "s3api, posix"
+          - set: "s3api non-policy, posix"
             LOCAL_FOLDER: /tmp/gw3
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
             BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
@@ -74,7 +74,7 @@ jobs:
             RECREATE_BUCKETS: "false"
             PORT: 7075
             BACKEND: "posix"
-          - set: "s3api, s3 backend"
+          - set: "s3api non-policy, s3 backend"
             LOCAL_FOLDER: /tmp/gw7
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
             BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
@@ -118,6 +118,28 @@ jobs:
             RECREATE_BUCKETS: "false"
             PORT: 7079
             BACKEND: "posix"
+          - set: "s3api policy and user, posix"
+            LOCAL_FOLDER: /tmp/gw11
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-10
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-10
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam11
+            AWS_ENDPOINT_URL: https://127.0.0.1:7080
+            RUN_SET: "s3api-policy,s3api-user"
+            RECREATE_BUCKETS: "true"
+            PORT: 7080
+            BACKEND: "posix"
+          - set: "s3api policy and user, s3 backend"
+            LOCAL_FOLDER: /tmp/gw12
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-11
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-11
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam12
+            AWS_ENDPOINT_URL: https://127.0.0.1:7081
+            RUN_SET: "s3api-policy,s3api-user"
+            RECREATE_BUCKETS: "true"
+            PORT: 7081
+            BACKEND: "s3"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -152,7 +174,7 @@ jobs:
         run: |
           sudo apt-get install libxml2-utils
 
-      - name: Build and run, posix backend
+      - name: Build and run
         env:
           LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
           BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
@@ -179,6 +201,9 @@ jobs:
           USERNAME_TWO: HIJKLMN
           PASSWORD_TWO: 8901234
           TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
+          REMOVE_TEST_FILE_FOLDER: true
+          VERSIONING_DIR: ${{ github.workspace }}/versioning
+          COMMAND_LOG: command.log
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST

.gitignore

@@ -62,4 +62,8 @@ tests/!s3cfg.local.default
 *.patch
 
 # grafana's local database (kept on filesystem for survival between instantiations)
-metrics-exploration/grafana_data/**
+metrics-exploration/grafana_data/**
+
+# bats tools
+/tests/bats-assert
+/tests/bats-support

Makefile

@@ -18,6 +18,10 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 
+# docker-compose
+DCCMD=docker-compose
+DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
+
 BIN=versitygw
 
 VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -71,19 +75,19 @@ dist:
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
-	docker compose --env-file .env.dev up posix
+	$(DOCKERCOMPOSE) up posix
 
 # Creates and runs S3 gateway proxy instance in a docker container
 .PHONY: up-proxy
 up-proxy:
-	docker compose --env-file .env.dev up proxy
+	$(DOCKERCOMPOSE) up proxy
 
 # Creates and runs S3 gateway to azurite instance in a docker container
 .PHONY: up-azurite
 up-azurite:
-	docker compose --env-file .env.dev up azurite azuritegw
+	$(DOCKERCOMPOSE) up azurite azuritegw
 
 # Creates and runs both S3 gateway and proxy server instances in docker containers
 .PHONY: up-app
 up-app:
-	docker compose --env-file .env.dev up
+	$(DOCKERCOMPOSE) up

auth/iam.go

@@ -28,6 +28,19 @@ const (
 	RoleUserPlus Role = "userplus"
 )
 
+func (r Role) IsValid() bool {
+	switch r {
+	case RoleAdmin:
+		return true
+	case RoleUser:
+		return true
+	case RoleUserPlus:
+		return true
+	default:
+		return false
+	}
+}
+
 // Account is a gateway IAM account
 type Account struct {
 	Access  string `json:"access"`
@@ -37,6 +50,10 @@ type Account struct {
 	GroupID int    `json:"groupID"`
 }
 
+type ListUserAccountsResult struct {
+	Accounts []Account
+}
+
 // Mutable props, which could be changed when updating an IAM account
 type MutableProps struct {
 	Secret  *string `json:"secret"`

auth/object_lock.go

@@ -25,6 +25,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
 type BucketLockConfig struct {
@@ -92,12 +93,12 @@ func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfigur
 }
 
 func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
-	var retention types.ObjectLockRetention
+	var retention s3response.PutObjectRetentionInput
 	if err := xml.Unmarshal(input, &retention); err != nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
-	if retention.RetainUntilDate == nil || retention.RetainUntilDate.Before(time.Now()) {
+	if retention.RetainUntilDate.Before(time.Now()) {
 		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
 	}
 	switch retention.Mode {
@@ -135,7 +136,7 @@ func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
 	}
 }
 
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, bypass bool, be backend.Backend) error {
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
 	data, err := be.GetObjectLockConfiguration(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -171,8 +172,15 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 	}
 
 	for _, obj := range objects {
+		var key, versionId string
+		if obj.Key != nil {
+			key = *obj.Key
+		}
+		if obj.VersionId != nil {
+			versionId = *obj.VersionId
+		}
 		checkRetention := true
-		retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
+		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
 			continue
 		}
@@ -203,7 +211,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 							if err != nil {
 								return err
 							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
+							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
 							if err != nil {
 								return s3err.GetAPIError(s3err.ErrObjectLocked)
 							}
@@ -217,8 +225,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 
 		checkLegalHold := true
 
-		status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
+		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
 		if err != nil {
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+				continue
+			}
 			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
 				checkLegalHold = false
 			} else {
@@ -243,7 +254,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 					if err != nil {
 						return err
 					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
+					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
 					if err != nil {
 						return s3err.GetAPIError(s3err.ErrObjectLocked)
 					}

backend/azure/azure.go

@@ -183,12 +183,14 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 		var acl auth.ACL
 		if len(aclBytes) > 0 {
 			if err := json.Unmarshal(aclBytes, &acl); err != nil {
-				return fmt.Errorf("unmarshal bucket acl: %w", err)
+				return fmt.Errorf("unmarshal acl: %w", err)
 			}
 		}
+
 		if acl.Owner == acct.Access {
 			return s3err.GetAPIError(s3err.ErrBucketAlreadyOwnedByYou)
 		}
+		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
 	}
 	return azureErrToS3Err(err)
 }
@@ -248,8 +250,8 @@ func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3
 	return &s3.HeadBucketOutput{}, nil
 }
 
-func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
-	pager := az.client.NewListBlobsFlatPager(*input.Bucket, nil)
+func (az *Azure) DeleteBucket(ctx context.Context, bucket string) error {
+	pager := az.client.NewListBlobsFlatPager(bucket, nil)
 
 	pg, err := pager.NextPage(ctx)
 	if err != nil {
@@ -259,7 +261,7 @@ func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput)
 	if len(pg.Segment.BlobItems) > 0 {
 		return s3err.GetAPIError(s3err.ErrBucketNotEmpty)
 	}
-	_, err = az.client.DeleteContainer(ctx, *input.Bucket, nil)
+	_, err = az.client.DeleteContainer(ctx, bucket, nil)
 	return azureErrToS3Err(err)
 }
 
@@ -569,23 +571,28 @@ Pager:
 				isTruncated = true
 				break Pager
 			}
+
+			marker := getString(input.Marker)
+			pfx := strings.TrimSuffix(*v.Name, getString(input.Delimiter))
+			if marker != "" && strings.HasPrefix(marker, pfx) {
+				continue
+			}
+
 			cPrefixes = append(cPrefixes, types.CommonPrefix{
 				Prefix: v.Name,
 			})
 		}
 	}
 
-	// TODO: generate common prefixes when appropriate
-
 	return s3response.ListObjectsResult{
 		Contents:       objects,
-		Marker:         input.Marker,
+		Marker:         backend.GetPtrFromString(*input.Marker),
 		MaxKeys:        input.MaxKeys,
 		Name:           input.Bucket,
 		NextMarker:     nextMarker,
-		Prefix:         input.Prefix,
+		Prefix:         backend.GetPtrFromString(*input.Prefix),
 		IsTruncated:    &isTruncated,
-		Delimiter:      input.Delimiter,
+		Delimiter:      backend.GetPtrFromString(*input.Delimiter),
 		CommonPrefixes: cPrefixes,
 	}, nil
 }
@@ -646,6 +653,13 @@ Pager:
 				isTruncated = true
 				break Pager
 			}
+
+			marker := getString(input.ContinuationToken)
+			pfx := strings.TrimSuffix(*v.Name, getString(input.Delimiter))
+			if marker != "" && strings.HasPrefix(marker, pfx) {
+				continue
+			}
+
 			cPrefixes = append(cPrefixes, types.CommonPrefix{
 				Prefix: v.Name,
 			})
@@ -654,14 +668,15 @@ Pager:
 
 	return s3response.ListObjectsV2Result{
 		Contents:              objects,
-		ContinuationToken:     input.ContinuationToken,
+		ContinuationToken:     backend.GetPtrFromString(*input.ContinuationToken),
 		MaxKeys:               input.MaxKeys,
 		Name:                  input.Bucket,
 		NextContinuationToken: nextMarker,
-		Prefix:                input.Prefix,
+		Prefix:                backend.GetPtrFromString(*input.Prefix),
 		IsTruncated:           &isTruncated,
-		Delimiter:             input.Delimiter,
+		Delimiter:             backend.GetPtrFromString(*input.Delimiter),
 		CommonPrefixes:        cPrefixes,
+		StartAfter:            backend.GetPtrFromString(*input.StartAfter),
 	}, nil
 }
 
@@ -1021,7 +1036,8 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa
 	prefix := string(metaTmpMultipartPrefix)
 
 	pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-		Prefix: &prefix,
+		Include: container.ListBlobsInclude{Metadata: true},
+		Prefix:  &prefix,
 	})
 
 	for pager.More() {
@@ -1285,22 +1301,9 @@ func (az *Azure) GetObjectLockConfiguration(ctx context.Context, bucket string)
 }
 
 func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
+	err := az.isBucketObjectLockEnabled(ctx, bucket)
 	if err != nil {
-		return azureErrToS3Err(err)
-	}
-
-	if len(cfg) == 0 {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
-	}
-
-	var bucketLockConfig auth.BucketLockConfig
-	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse bucket lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
 
 	blobClient, err := az.getBlobClient(bucket, object)
@@ -1361,6 +1364,11 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
 		return nil, azureErrToS3Err(err)
 	}
 
+	err = az.isBucketObjectLockEnabled(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+
 	retentionPtr, ok := props.Metadata[string(keyObjRetention)]
 	if !ok {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1370,22 +1378,9 @@ func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, version
 }
 
 func (az *Azure) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
-	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
+	err := az.isBucketObjectLockEnabled(ctx, bucket)
 	if err != nil {
-		return azureErrToS3Err(err)
-	}
-
-	if len(cfg) == 0 {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
-	}
-
-	var bucketLockConfig auth.BucketLockConfig
-	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse bucket lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+		return err
 	}
 
 	blobClient, err := az.getBlobClient(bucket, object)
@@ -1432,6 +1427,11 @@ func (az *Azure) GetObjectLegalHold(ctx context.Context, bucket, object, version
 		return nil, azureErrToS3Err(err)
 	}
 
+	err = az.isBucketObjectLockEnabled(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+
 	retentionPtr, ok := props.Metadata[string(keyObjLegalHold)]
 	if !ok {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
@@ -1471,6 +1471,28 @@ func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response
 	return buckets, nil
 }
 
+func (az *Azure) isBucketObjectLockEnabled(ctx context.Context, bucket string) error {
+	cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	if len(cfg) == 0 {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	var bucketLockConfig auth.BucketLockConfig
+	if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse bucket lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	return nil
+}
+
 func (az *Azure) getContainerURL(cntr string) string {
 	return fmt.Sprintf("%v/%v", strings.TrimRight(az.serviceURL, "/"), cntr)
 }

backend/backend.go

@@ -37,9 +37,9 @@ type Backend interface {
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
 	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
-	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
+	DeleteBucket(_ context.Context, bucket string) error
 	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
-	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
@@ -123,14 +123,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
 func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)

backend/common.go

@@ -54,6 +54,13 @@ func GetStringPtr(s string) *string {
 	return &s
 }
 
+func GetPtrFromString(str string) *string {
+	if str == "" {
+		return nil
+	}
+	return &str
+}
+
 func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
@@ -95,7 +102,7 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 		return 0, 0, errInvalidRange
 	}
 
-	if endOffset <= startOffset {
+	if endOffset < startOffset {
 		return 0, 0, errInvalidRange
 	}
 
@@ -109,15 +116,13 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
 		copySourceHeader = copySourceHeader[1:]
 	}
 
-	cSplitted := strings.Split(copySourceHeader, "?")
-	copySource := cSplitted[0]
-	var versionId string
-	if len(cSplitted) > 1 {
-		versionIdParts := strings.Split(cSplitted[1], "=")
-		if len(versionIdParts) != 2 || versionIdParts[0] != "versionId" {
-			return "", "", "", s3err.GetAPIError(s3err.ErrInvalidRequest)
-		}
-		versionId = versionIdParts[1]
+	var copySource, versionId string
+	i := strings.LastIndex(copySourceHeader, "?versionId=")
+	if i == -1 {
+		copySource = copySourceHeader
+	} else {
+		copySource = copySourceHeader[:i]
+		versionId = copySourceHeader[i+11:]
 	}
 
 	srcBucket, srcObject, ok := strings.Cut(copySource, "/")

backend/meta/meta.go

@@ -14,17 +14,19 @@
 
 package meta
 
+import "os"
+
 // MetadataStorer defines the interface for managing metadata.
 // When object == "", the operation is on the bucket.
 type MetadataStorer interface {
 	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
 	// Returns the value of the attribute, or an error if the attribute does not exist.
-	RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
+	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
 
 	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
 	// If attribute already exists, new attribute should replace existing.
 	// Returns an error if the operation fails.
-	StoreAttribute(bucket, object, attribute string, value []byte) error
+	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
 
 	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
 	// Returns an error if the operation fails.

backend/meta/xattr.go

@@ -17,6 +17,7 @@ package meta
 import (
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
@@ -36,7 +37,15 @@ var (
 type XattrMeta struct{}
 
 // RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
+func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
+	if f != nil {
+		b, err := xattr.FGet(f, xattrPrefix+attribute)
+		if errors.Is(err, xattr.ENOATTR) {
+			return nil, ErrNoSuchKey
+		}
+		return b, err
+	}
+
 	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
 	if errors.Is(err, xattr.ENOATTR) {
 		return nil, ErrNoSuchKey
@@ -45,7 +54,11 @@ func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte,
 }
 
 // StoreAttribute stores the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
+func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
+	if f != nil {
+		return xattr.FSet(f, xattrPrefix+attribute, value)
+	}
+
 	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
 }
 

backend/mkdir.go

@@ -15,11 +15,6 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
-var (
-	// TODO: make this configurable
-	defaultDirPerm fs.FileMode = 0755
-)
-
 // MkdirAll is similar to os.MkdirAll but it will return
 // ErrObjectParentIsFile when appropriate
 // MkdirAll creates a directory named path,
@@ -32,7 +27,7 @@ var (
 // and returns nil.
 // Any directory created will be set to provided uid/gid ownership
 // if doChown is true.
-func MkdirAll(path string, uid, gid int, doChown bool) error {
+func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
 	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
 	dir, err := os.Stat(path)
 	if err == nil {
@@ -55,14 +50,14 @@ func MkdirAll(path string, uid, gid int, doChown bool) error {
 
 	if j > 1 {
 		// Create parent.
-		err = MkdirAll(path[:j-1], uid, gid, doChown)
+		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
 		if err != nil {
 			return err
 		}
 	}
 
 	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, defaultDirPerm)
+	err = os.Mkdir(path, dirPerm)
 	if err != nil {
 		// Handle arguments like "foo/." by
 		// double-checking that directory doesn't exist.

backend/posix/with_otmpfile.go

@@ -43,6 +43,7 @@ type tmpfile struct {
 	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
 }
 
 var (
@@ -62,7 +63,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown)
+		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -108,6 +109,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
+		newDirPerm: p.newDirPerm,
 	}
 
 	// falloc is best effort, its fine if this fails
@@ -134,6 +136,9 @@ func (tmp *tmpfile) falloc() error {
 }
 
 func (tmp *tmpfile) link() error {
+	// make sure this is cleaned up in all error cases
+	defer tmp.f.Close()
+
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
 	// with any other simultaneous uploads. The final operation is to move the
@@ -148,7 +153,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -170,11 +175,21 @@ func (tmp *tmpfile) link() error {
 	}
 	defer dirf.Close()
 
-	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-	if err != nil {
-		return fmt.Errorf("link tmpfile (%q in %q): %w",
-			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
+	for {
+		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		if errors.Is(err, syscall.EEXIST) {
+			err := os.Remove(objPath)
+			if err != nil && !errors.Is(err, fs.ErrNotExist) {
+				return fmt.Errorf("remove stale path: %w", err)
+			}
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("link tmpfile (fd %q as %q): %w",
+				filepath.Base(tmp.f.Name()), objPath, err)
+		}
+		break
 	}
 
 	err = tmp.f.Close()

backend/posix/without_otmpfile.go

@@ -41,7 +41,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 
 	// Create a temp file for upload while in progress (see link comments below).
 	var err error
-	err = backend.MkdirAll(dir, uid, gid, doChown)
+	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}

backend/s3proxy/s3.go

@@ -125,8 +125,10 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 	return handleError(err)
 }
 
-func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
-	_, err := s.client.DeleteBucket(ctx, input)
+func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: &bucket,
+	})
 	return handleError(err)
 }
 
@@ -172,12 +174,15 @@ func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status
 	return handleError(err)
 }
 
-func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
 	out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
 		Bucket: &bucket,
 	})
 
-	return out, handleError(err)
+	return s3response.GetBucketVersioningOutput{
+		Status:    &out.Status,
+		MFADelete: &out.MFADelete,
+	}, handleError(err)
 }
 
 func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {

backend/scoutfs/scoutfs.go

@@ -44,6 +44,7 @@ type ScoutfsOpts struct {
 	ChownGID    bool
 	GlacierMode bool
 	BucketLinks bool
+	NewDirPerm  fs.FileMode
 }
 
 type ScoutFS struct {
@@ -74,6 +75,9 @@ type ScoutFS struct {
 	// used to determine if chowning is needed
 	euid int
 	egid int
+
+	// newDirPerm is the permissions to use when creating new directories
+	newDirPerm fs.FileMode
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -226,7 +230,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
-		b, err := s.meta.RetrieveAttribute(bucket, partObjPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
@@ -262,7 +266,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = moveData(pf, f.f)
+		err = moveData(pf, f.File())
 		pf.Close()
 		if err != nil {
 			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
@@ -277,83 +281,76 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	dir := filepath.Dir(objname)
 	if dir != "" {
 		uid, gid, doChown := s.getChownIDs(acct)
-		err = backend.MkdirAll(dir, uid, gid, doChown)
+		err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
 		if err != nil {
 			return nil, err
 		}
 	}
-	err = f.link()
-	if err != nil {
-		return nil, fmt.Errorf("link object in namespace: %w", err)
-	}
 
 	for k, v := range userMetaData {
-		err = s.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
+		err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
-			// cleanup object if returning error
-			os.Remove(objname)
 			return nil, fmt.Errorf("set user attr %q: %w", k, err)
 		}
 	}
 
 	// load and set tagging
-	tagging, err := s.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
-	if err == nil {
-		if err := s.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
-			// cleanup object
-			os.Remove(objname)
-			return nil, fmt.Errorf("set object tagging: %w", err)
-		}
-	}
+	tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
 		return nil, fmt.Errorf("get object tagging: %w", err)
 	}
+	if err == nil {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
+		if err != nil {
+			return nil, fmt.Errorf("set object tagging: %w", err)
+		}
+	}
 
 	// set content-type
 	if cType != "" {
-		if err := s.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
-			// cleanup object
-			os.Remove(objname)
+		err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
+		if err != nil {
 			return nil, fmt.Errorf("set object content type: %w", err)
 		}
 	}
 
 	// load and set legal hold
-	lHold, err := s.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
-	if err == nil {
-		if err := s.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
-			// cleanup object
-			os.Remove(objname)
-			return nil, fmt.Errorf("set object legal hold: %w", err)
-		}
-	}
+	lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
 		return nil, fmt.Errorf("get object legal hold: %w", err)
 	}
-
-	// load and set retention
-	ret, err := s.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
 	if err == nil {
-		if err := s.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
-			// cleanup object
-			os.Remove(objname)
-			return nil, fmt.Errorf("set object retention: %w", err)
+		err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
+		if err != nil {
+			return nil, fmt.Errorf("set object legal hold: %w", err)
 		}
 	}
+
+	// load and set retention
+	ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
 		return nil, fmt.Errorf("get object retention: %w", err)
 	}
+	if err == nil {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
+		if err != nil {
+			return nil, fmt.Errorf("set object retention: %w", err)
+		}
+	}
 
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
-	err = s.meta.StoreAttribute(bucket, object, etagkey, []byte(s3MD5))
+	err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
 	if err != nil {
-		// cleanup object if returning error
-		os.Remove(objname)
 		return nil, fmt.Errorf("set etag attr: %w", err)
 	}
 
+	err = f.link()
+	if err != nil {
+		return nil, fmt.Errorf("link object in namespace: %w", err)
+	}
+
 	// cleanup tmp dirs
 	os.RemoveAll(upiddir)
 	// use Remove for objdir in case there are still other uploads
@@ -392,7 +389,7 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
 		if !isValidMeta(e) {
 			continue
 		}
-		b, err := s.meta.RetrieveAttribute(bucket, object, e)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, object, e)
 		if err != nil {
 			continue
 		}
@@ -404,13 +401,13 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
 	}
 
 	var contentType, contentEncoding string
-	b, _ := s.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
+	b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
 	contentType = string(b)
 	if contentType != "" {
 		m[contentTypeHdr] = contentType
 	}
 
-	b, _ = s.meta.RetrieveAttribute(bucket, object, contentEncHdr)
+	b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
 	contentEncoding = string(b)
 	if contentEncoding != "" {
 		m[contentEncHdr] = contentEncoding
@@ -466,7 +463,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 			return nil, fmt.Errorf("stat part: %w", err)
 		}
 
-		b, err := s.meta.RetrieveAttribute(bucket, partPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
@@ -514,7 +511,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 		contentType = "application/x-directory"
 	}
 
-	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
+	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -554,7 +551,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	contentLength := fi.Size()
 
 	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
-	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, "")
+	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
 	if err == nil {
 		if *status {
 			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
@@ -565,7 +562,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 
 	var objectLockMode types.ObjectLockMode
 	var objectLockRetainUntilDate *time.Time
-	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, "")
+	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
 	if err == nil {
 		var config types.ObjectLockRetention
 		if err := json.Unmarshal(retention, &config); err == nil {
@@ -685,7 +682,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 
 	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
+	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -840,7 +837,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			etagBytes, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
+			etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
 			if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
 				return s3response.Object{}, backend.ErrSkipObj
 			}
@@ -869,7 +866,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		}
 
 		// file object, get object info and fill out object data
-		b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
 			return s3response.Object{}, backend.ErrSkipObj
 		}

backend/scoutfs/scoutfs_compat.go

@@ -40,6 +40,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		ChownUID:    opts.ChownUID,
 		ChownGID:    opts.ChownGID,
 		BucketLinks: opts.BucketLinks,
+		NewDirPerm:  opts.NewDirPerm,
 	})
 	if err != nil {
 		return nil, err
@@ -58,6 +59,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		chownuid:    opts.ChownUID,
 		chowngid:    opts.ChownGID,
 		glaciermode: opts.GlacierMode,
+		newDirPerm:  opts.NewDirPerm,
 	}, nil
 }
 
@@ -71,10 +73,10 @@ type tmpfile struct {
 	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
 }
 
 var (
-	// TODO: make this configurable
 	defaultFilePerm uint32 = 0644
 )
 
@@ -102,6 +104,7 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
+		newDirPerm: s.newDirPerm,
 	}
 
 	if doChown {
@@ -129,7 +132,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -174,6 +177,10 @@ func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}
+
 func moveData(from *os.File, to *os.File) error {
 	return scoutfs.MoveData(from, to)
 }

backend/scoutfs/scoutfs_incompat.go

@@ -28,9 +28,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
-type tmpfile struct {
-	f *os.File
-}
+type tmpfile struct{}
 
 var (
 	errNotSupported = errors.New("not supported")
@@ -56,6 +54,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return nil
+}
+
 func moveData(_, _ *os.File) error {
 	return errNotSupported
 }

cmd/versitygw/admin.go

@@ -19,7 +19,7 @@ import (
 	"crypto/sha256"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,6 +29,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3response"
@@ -37,6 +38,7 @@ import (
 var (
 	adminAccess   string
 	adminSecret   string
+	adminRegion   string
 	adminEndpoint string
 	allowInsecure bool
 )
@@ -171,6 +173,14 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminSecret,
 			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "admin s3 region string",
+				EnvVars:     []string{"ADMIN_REGION"},
+				Value:       "us-east-1",
+				Destination: &adminRegion,
+				Aliases:     []string{"r"},
+			},
 			&cli.StringFlag{
 				Name:        "endpoint-url",
 				Usage:       "admin apis endpoint url",
@@ -215,24 +225,24 @@ func createUser(ctx *cli.Context) error {
 		GroupID: groupID,
 	}
 
-	accJson, err := json.Marshal(acc)
+	accxml, err := xml.Marshal(acc)
 	if err != nil {
 		return fmt.Errorf("failed to parse user data: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(accJson)
+	hashedPayload := sha256.Sum256(accxml)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -251,11 +261,9 @@ func createUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -277,7 +285,7 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -296,11 +304,9 @@ func deleteUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -317,24 +323,24 @@ func updateUser(ctx *cli.Context) error {
 		props.GroupID = &groupId
 	}
 
-	propsJSON, err := json.Marshal(props)
+	propsxml, err := xml.Marshal(props)
 	if err != nil {
 		return fmt.Errorf("failed to parse user attributes: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(propsJSON)
+	hashedPayload := sha256.Sum256(propsxml)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -353,11 +359,9 @@ func updateUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -374,7 +378,7 @@ func listUsers(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -393,15 +397,15 @@ func listUsers(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var accs []auth.Account
-	if err := json.Unmarshal(body, &accs); err != nil {
+	var accs auth.ListUserAccountsResult
+	if err := xml.Unmarshal(body, &accs); err != nil {
 		return err
 	}
 
-	printAcctTable(accs)
+	printAcctTable(accs.Accounts)
 
 	return nil
 }
@@ -441,7 +445,7 @@ func changeBucketOwner(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -460,11 +464,9 @@ func changeBucketOwner(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Println(string(body))
-
 	return nil
 }
 
@@ -493,7 +495,7 @@ func listBuckets(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -512,15 +514,26 @@ func listBuckets(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var buckets []s3response.Bucket
-	if err := json.Unmarshal(body, &buckets); err != nil {
+	var result s3response.ListBucketsResult
+	if err := xml.Unmarshal(body, &result); err != nil {
 		return err
 	}
 
-	printBuckets(buckets)
+	printBuckets(result.Buckets)
 
 	return nil
 }
+
+func parseApiError(body []byte) error {
+	var apiErr smithy.GenericAPIError
+	err := xml.Unmarshal(body, &apiErr)
+	if err != nil {
+		apiErr.Code = "InternalServerError"
+		apiErr.Message = err.Error()
+	}
+
+	return &apiErr
+}

cmd/versitygw/gateway_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
@@ -57,7 +58,9 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
+	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
+		NewDirPerm: 0755,
+	})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}
@@ -75,6 +78,9 @@ func initPosix(ctx context.Context) {
 		}
 		wg.Done()
 	}()
+
+	// wait for server to start
+	time.Sleep(1 * time.Second)
 }
 
 func TestIntegration(t *testing.T) {

cmd/versitygw/posix.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/meta"
@@ -26,6 +28,7 @@ var (
 	chownuid, chowngid bool
 	bucketlinks        bool
 	versioningDir      string
+	dirPerms           uint
 )
 
 func posixCommand() *cli.Command {
@@ -68,6 +71,14 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_VERSIONING_DIR"},
 				Destination: &versioningDir,
 			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
 		},
 	}
 }
@@ -83,11 +94,16 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("posix xattr check: %v", err)
 	}
 
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
 	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
 		ChownUID:      chownuid,
 		ChownGID:      chowngid,
 		BucketLinks:   bucketlinks,
 		VersioningDir: versioningDir,
+		NewDirPerm:    fs.FileMode(dirPerms),
 	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)

cmd/versitygw/scoutfs.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/scoutfs"
@@ -69,6 +71,14 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_BUCKET_LINKS"},
 				Destination: &bucketlinks,
 			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
 		},
 	}
 }
@@ -78,11 +88,16 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
 	var opts scoutfs.ScoutfsOpts
 	opts.GlacierMode = glacier
 	opts.ChownUID = chownuid
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
+	opts.NewDirPerm = fs.FileMode(dirPerms)
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

cmd/versitygw/test.go

@@ -37,6 +37,7 @@ var (
 	pathStyle         bool
 	checksumDisable   bool
 	versioningEnabled bool
+	azureTests        bool
 )
 
 func testCommand() *cli.Command {
@@ -95,12 +96,26 @@ func initTestCommands() []*cli.Command {
 					Destination: &versioningEnabled,
 					Aliases:     []string{"vs"},
 				},
+				&cli.BoolFlag{
+					Name:        "azure-test-mode",
+					Usage:       "Skips tests that are not supported by Azure",
+					Destination: &azureTests,
+					Aliases:     []string{"azure"},
+				},
 			},
 		},
 		{
 			Name:   "posix",
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test posix when versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		},
 		{
 			Name:   "iam",
@@ -288,6 +303,9 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if versioningEnabled {
 			opts = append(opts, integration.WithVersioningEnabled())
 		}
+		if azureTests {
+			opts = append(opts, integration.WithAzureMode())
+		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -319,11 +337,22 @@ func extractIntTests() (commands []*cli.Command) {
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
+				if versioningEnabled {
+					opts = append(opts, integration.WithVersioningEnabled())
+				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)
 				return err
 			},
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		})
 	}
 	return

go.mod

@@ -3,13 +3,13 @@ module github.com/versity/versitygw
 go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.30.5
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2
-	github.com/aws/smithy-go v1.20.4
+	github.com/aws/aws-sdk-go-v2 v1.32.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0
+	github.com/aws/smithy-go v1.22.0
 	github.com/go-ldap/ldap/v3 v3.4.8
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/go-cmp v0.6.0
@@ -20,10 +20,11 @@ require (
 	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.3
-	github.com/urfave/cli/v2 v2.27.4
-	github.com/valyala/fasthttp v1.55.0
+	github.com/urfave/cli/v2 v2.27.5
+	github.com/valyala/fasthttp v1.56.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sys v0.25.0
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.26.0
 )
 
 require (
@@ -31,11 +32,11 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -49,27 +50,27 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.27.0 // indirect
-	golang.org/x/net v0.29.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
-	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.33
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.32
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.28.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.41
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect

go.sum

@@ -1,15 +1,19 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
@@ -19,51 +23,55 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
-github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
-github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
-github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU=
-github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18 h1:9DIp7vhmOPmueCDwpXa45bEbLHHTt1kcxChdTJWWxvI=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18/go.mod h1:aJv/Fwz8r56ozwYFRC4bzoeL1L17GYQYemfblOBux1M=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
+github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
+github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ=
+github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33 h1:X+4YY5kZRI/cOoSMVMGTqFXHAMg1bvvay7IBcqHpybQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.33/go.mod h1:DPynzu+cn92k5UQ6tZhX+wfTB4ah6QDU/NgdHqatmvk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE=
-github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o=
-github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
-github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo=
+github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
+github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
@@ -109,9 +117,11 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -142,6 +152,8 @@ github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
 github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
+github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -165,12 +177,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
-github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
-github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
+github.com/valyala/fasthttp v1.56.0 h1:bEZdJev/6LCBlpdORfrLu/WOZXXxvrUQSiyniuaoW8U=
+github.com/valyala/fasthttp v1.56.0/go.mod h1:sReBt3XZVnudxuLOx4J/fMrJVorWRiWY2koQKgABiVI=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -183,6 +195,8 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -192,8 +206,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
-golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -209,12 +223,14 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
-golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -234,8 +250,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
-golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -251,10 +267,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
-golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
-golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

metrics/actions.go

@@ -72,6 +72,14 @@ var (
 	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
 	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
 	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
+
+	// Admin actions
+	ActionAdminCreateUser        = "admin_CreateUser"
+	ActionAdminUpdateUser        = "admin_UpdateUser"
+	ActionAdminDeleteUser        = "admin_DeleteUser"
+	ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
+	ActionAdminListUsers         = "admin_ListUsers"
+	ActionAdminListBuckets       = "admin_ListBuckets"
 )
 
 func init() {

runtests.sh

@@ -5,17 +5,19 @@ rm -rf /tmp/gw
 mkdir /tmp/gw
 rm -rf /tmp/covdata
 mkdir /tmp/covdata
+rm -rf /tmp/versioing.covdata
+mkdir /tmp/versioning.covdata
 rm -rf /tmp/versioningdir
 mkdir /tmp/versioningdir
 
-# run server in background
-GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+# run server in background not versioning-enabled
+# port: 7070(default)
+GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
 GW_PID=$!
 
-# wait a second for server to start up
 sleep 1
 
-# check if server is still running
+# check if versioning-enabled gateway process is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -23,7 +25,7 @@ fi
 
 # run tests
 # full flow tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow -vs; then
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
 	echo "full flow tests failed"
 	kill $GW_PID
 	exit 1
@@ -41,8 +43,39 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
 	exit 1
 fi
 
-# kill off server
 kill $GW_PID
+
+# run server in background versioning-enabled
+# port: 7071
+GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7071 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+GW_VS_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if versioning-enabled gateway process is still running
+if ! kill -0 $GW_VS_PID; then
+	echo "versioning-enabled server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 full-flow -vs; then
+	echo "versioning-enabled full-flow tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 posix -vs; then
+	echo "versiongin-enabled posix tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_VS_PID
+
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-server.go

@@ -53,6 +53,9 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
 	app.Use(middlewares.VerifyMD5Body(l))
 
+	// Admin role checker
+	app.Use(middlewares.IsAdmin(l))
+
 	server.router.Init(app, be, iam, l)
 
 	return server

s3api/controllers/admin.go

@@ -16,15 +16,19 @@ package controllers
 
 import (
 	"encoding/json"
-	"errors"
+	"encoding/xml"
 	"fmt"
+	"net/http"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
+	"github.com/versity/versitygw/s3response"
 )
 
 type AdminController struct {
@@ -38,187 +42,124 @@ func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLo
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:CreateUser",
-			})
-	}
 	var usr auth.Account
-	err := json.Unmarshal(ctx.Body(), &usr)
+	err := xml.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return sendResponse(ctx, fmt.Errorf("failed to parse request body: %w", err), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusBadRequest,
-				action: "admin:CreateUser",
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
 			})
 	}
 
-	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return sendResponse(ctx, errors.New("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusBadRequest,
-				action: "admin:CreateUser",
+	if !usr.Role.IsValid() {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
 			})
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		status := fiber.StatusInternalServerError
-		err = fmt.Errorf("failed to create user: %w", err)
-
 		if strings.Contains(err.Error(), "user already exists") {
-			status = fiber.StatusConflict
+			err = s3err.GetAPIError(s3err.ErrAdminUserExists)
 		}
 
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				status: status,
-				logger: c.l,
-				action: "admin:CreateUser",
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
 			})
 	}
 
-	return sendResponse(ctx, nil, "The user has been created successfully", &metaOptions{
-		status: fiber.StatusCreated,
-		logger: c.l,
-		action: "admin:CreateUser",
-	})
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminCreateUser,
+			Status: http.StatusCreated,
+		})
 }
 
 func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:UpdateUser",
-			})
-	}
-
 	access := ctx.Query("access")
 	if access == "" {
-		return sendResponse(ctx, errors.New("missing user access parameter"), nil,
-			&metaOptions{
-				status: fiber.StatusBadRequest,
-				logger: c.l,
-				action: "admin:UpdateUser",
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
 			})
 	}
 
 	var props auth.MutableProps
-	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
-		return sendResponse(ctx, fmt.Errorf("invalid request body %w", err), nil,
-			&metaOptions{
-				status: fiber.StatusBadRequest,
-				logger: c.l,
-				action: "admin:UpdateUser",
+	if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
 			})
 	}
 
 	err := c.iam.UpdateUserAccount(access, props)
 	if err != nil {
-		status := fiber.StatusInternalServerError
-		err = fmt.Errorf("failed to update user account: %w", err)
-
 		if strings.Contains(err.Error(), "user not found") {
-			status = fiber.StatusNotFound
+			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
 		}
 
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				status: status,
-				logger: c.l,
-				action: "admin:UpdateUser",
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
 			})
 	}
 
-	return sendResponse(ctx, nil, "the user has been updated successfully",
-		&metaOptions{
-			logger: c.l,
-			action: "admin:UpdateUser",
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminUpdateUser,
 		})
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:DeleteUser",
-			})
-	}
 
 	err := c.iam.DeleteUserAccount(access)
-	if err != nil {
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:DeleteUser",
-			})
-	}
-
-	return sendResponse(ctx, nil, "The user has been deleted successfully",
-		&metaOptions{
-			logger: c.l,
-			action: "admin:DeleteUser",
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminDeleteUser,
 		})
 }
 
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:ListUsers",
-			})
-	}
 	accs, err := c.iam.ListUserAccounts()
-	return sendResponse(ctx, err, accs,
-		&metaOptions{
-			logger: c.l,
-			action: "admin:ListUsers",
+	return SendXMLResponse(ctx,
+		auth.ListUserAccountsResult{
+			Accounts: accs,
+		}, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListUsers,
 		})
 }
 
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:ChangeBucketOwner",
-			})
-	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
 
 	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
 	if err != nil {
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:ChangeBucketOwner",
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
 			})
 	}
 	if len(accs) > 0 {
-		return sendResponse(ctx, errors.New("user specified as the new bucket owner does not exist"), nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:ChangeBucketOwner",
-				status: fiber.StatusNotFound,
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
 			})
 	}
 
@@ -235,91 +176,28 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 
 	aclParsed, err := json.Marshal(acl)
 	if err != nil {
-		return sendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err), nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:ChangeBucketOwner",
+		return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
 			})
 	}
 
 	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
-	return sendResponse(ctx, err, "Bucket owner has been updated successfully",
-		&metaOptions{
-			logger: c.l,
-			action: "admin:ChangeBucketOwner",
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminChangeBucketOwner,
 		})
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:ListBuckets",
-			})
-	}
-
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
-	return sendResponse(ctx, err, buckets,
-		&metaOptions{
-			logger: c.l,
-			action: "admin:ListBuckets",
+	return SendXMLResponse(ctx,
+		s3response.ListBucketsResult{
+			Buckets: buckets,
+		}, err, &MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListBuckets,
 		})
 }
-
-type metaOptions struct {
-	action string
-	status int
-	logger s3log.AuditLogger
-}
-
-func sendResponse(ctx *fiber.Ctx, err error, data any, m *metaOptions) error {
-	status := m.status
-	if err != nil {
-		if status == 0 {
-			status = fiber.StatusInternalServerError
-		}
-		if m.logger != nil {
-			m.logger.Log(ctx, err, []byte(err.Error()), s3log.LogMeta{
-				Action:     m.action,
-				HttpStatus: status,
-			})
-		}
-
-		return ctx.Status(status).SendString(err.Error())
-	}
-
-	if status == 0 {
-		status = fiber.StatusOK
-	}
-
-	msg, ok := data.(string)
-	if ok {
-		if m.logger != nil {
-			m.logger.Log(ctx, nil, []byte(msg), s3log.LogMeta{
-				Action:     m.action,
-				HttpStatus: status,
-			})
-		}
-
-		return ctx.Status(status).SendString(msg)
-	}
-
-	dataJSON, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-
-	if m.logger != nil {
-		m.logger.Log(ctx, nil, dataJSON, s3log.LogMeta{
-			HttpStatus: status,
-			Action:     m.action,
-		})
-	}
-
-	ctx.Set(fiber.HeaderContentType, fiber.MIMEApplicationJSON)
-
-	return ctx.Status(status).Send(dataJSON)
-}

s3api/controllers/admin_test.go

@@ -15,12 +15,11 @@
 package controllers
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
@@ -43,33 +42,26 @@ func TestAdminController_CreateUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/create-user", adminController.CreateUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	usr := auth.Account{
-		Access: "access",
-		Secret: "secret",
-		Role:   "invalid role",
-	}
-
-	user, _ := json.Marshal(&usr)
-
-	usr.Role = "admin"
-
-	succUsr, _ := json.Marshal(&usr)
-
-	appErr.Patch("/create-user", adminController.CreateUser)
+	succUser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>admin</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+	invuser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>invalid_role</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
 
 	tests := []struct {
 		name       string
@@ -79,31 +71,31 @@ func TestAdminController_CreateUser(t *testing.T) {
 		statusCode int
 	}{
 		{
-			name: "Admin-create-user-success",
+			name: "Admin-create-user-malformed-body",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
-			},
-			wantErr:    false,
-			statusCode: 201,
-		},
-		{
-			name: "Admin-create-user-invalid-user-role",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
 			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
-			app:  appErr,
+			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
 			},
 			wantErr:    false,
-			statusCode: 403,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-create-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
+			},
+			wantErr:    false,
+			statusCode: 201,
 		},
 	}
 	for _, tt := range tests {
@@ -134,24 +126,8 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/update-user", adminController.UpdateUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appErr.Patch("/update-user", adminController.UpdateUser)
-
-	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
-
 	adminControllerErr := AdminController{
 		iam: &IAMServiceMock{
 			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
@@ -162,13 +138,16 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	appNotFound := fiber.New()
 
-	appNotFound.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
 
+	succUser := `
+		<Account>
+			<Secret>secret</Secret>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -180,7 +159,7 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			name: "Admin-update-user-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
 			statusCode: 200,
@@ -189,10 +168,10 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			name: "Admin-update-user-missing-access",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
-			statusCode: 400,
+			statusCode: 404,
 		},
 		{
 			name: "Admin-update-user-invalid-request-body",
@@ -203,20 +182,11 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "Admin-update-user-invalid-requester-role",
-			app:  appErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Admin-update-user-not-found",
 			app:  appNotFound,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
 			statusCode: 404,
@@ -250,22 +220,8 @@ func TestAdminController_DeleteUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/delete-user", adminController.DeleteUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appErr.Patch("/delete-user", adminController.DeleteUser)
-
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -282,15 +238,6 @@ func TestAdminController_DeleteUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
-		{
-			name: "Admin-delete-user-invalid-requester-role",
-			app:  appErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -327,30 +274,9 @@ func TestAdminController_ListUsers(t *testing.T) {
 	}
 
 	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appErr.Patch("/list-users", adminControllerErr.ListUsers)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-users", adminController.ListUsers)
-
 	appSucc := fiber.New()
-
-	appSucc.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
@@ -360,15 +286,6 @@ func TestAdminController_ListUsers(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
-		{
-			name: "Admin-list-users-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Admin-list-users-iam-error",
 			app:  appErr,
@@ -435,39 +352,12 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
-
 	appIamErr := fiber.New()
-
-	appIamErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
 
 	appIamNoSuchUser := fiber.New()
-
-	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
@@ -477,15 +367,6 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
-		{
-			name: "Change-bucket-owner-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
 			app:  appIamErr,
@@ -540,23 +421,8 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
-
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -564,15 +430,6 @@ func TestAdminController_ListBuckets(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
-		{
-			name: "List-buckets-incorrect-role",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "List-buckets-success",
 			app:  app,

s3api/controllers/backend_moq_test.go

@@ -41,7 +41,7 @@ var _ backend.Backend = &BackendMock{}
 //			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 //				panic("mock out the CreateMultipartUpload method")
 //			},
-//			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
+//			DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucket method")
 //			},
 //			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
@@ -74,7 +74,7 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 //				panic("mock out the GetBucketTagging method")
 //			},
-//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
 //				panic("mock out the GetBucketVersioning method")
 //			},
 //			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
@@ -202,7 +202,7 @@ type BackendMock struct {
 	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 
 	// DeleteBucketFunc mocks the DeleteBucket method.
-	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
+	DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error
 
 	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
 	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
@@ -235,7 +235,7 @@ type BackendMock struct {
 	GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
 
 	// GetBucketVersioningFunc mocks the GetBucketVersioning method.
-	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
 
 	// GetObjectFunc mocks the GetObject method.
 	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
@@ -388,8 +388,8 @@ type BackendMock struct {
 		DeleteBucket []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// DeleteBucketInput is the deleteBucketInput argument value.
-			DeleteBucketInput *s3.DeleteBucketInput
+			// Bucket is the bucket argument value.
+			Bucket string
 		}
 		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
 		DeleteBucketOwnershipControls []struct {
@@ -1012,21 +1012,21 @@ func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
 }
 
 // DeleteBucket calls DeleteBucketFunc.
-func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
+func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketFunc == nil {
 		panic("BackendMock.DeleteBucketFunc: method is nil but Backend.DeleteBucket was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam   context.Context
-		DeleteBucketInput *s3.DeleteBucketInput
+		ContextMoqParam context.Context
+		Bucket          string
 	}{
-		ContextMoqParam:   contextMoqParam,
-		DeleteBucketInput: deleteBucketInput,
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
 	}
 	mock.lockDeleteBucket.Lock()
 	mock.calls.DeleteBucket = append(mock.calls.DeleteBucket, callInfo)
 	mock.lockDeleteBucket.Unlock()
-	return mock.DeleteBucketFunc(contextMoqParam, deleteBucketInput)
+	return mock.DeleteBucketFunc(contextMoqParam, bucket)
 }
 
 // DeleteBucketCalls gets all the calls that were made to DeleteBucket.
@@ -1034,12 +1034,12 @@ func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBuc
 //
 //	len(mockedBackend.DeleteBucketCalls())
 func (mock *BackendMock) DeleteBucketCalls() []struct {
-	ContextMoqParam   context.Context
-	DeleteBucketInput *s3.DeleteBucketInput
+	ContextMoqParam context.Context
+	Bucket          string
 } {
 	var calls []struct {
-		ContextMoqParam   context.Context
-		DeleteBucketInput *s3.DeleteBucketInput
+		ContextMoqParam context.Context
+		Bucket          string
 	}
 	mock.lockDeleteBucket.RLock()
 	calls = mock.calls.DeleteBucket
@@ -1412,7 +1412,7 @@ func (mock *BackendMock) GetBucketTaggingCalls() []struct {
 }
 
 // GetBucketVersioning calls GetBucketVersioningFunc.
-func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
 	if mock.GetBucketVersioningFunc == nil {
 		panic("BackendMock.GetBucketVersioningFunc: method is nil but Backend.GetBucketVersioning was just called")
 	}

s3api/controllers/base.go

@@ -83,7 +83,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	maxParts := int32(ctx.QueryInt("max-parts", -1))
 	partNumberMarker := ctx.Query("part-number-marker")
 	acceptRange := ctx.Get("Range")
 	acct := ctx.Locals("account").(auth.Account)
@@ -221,16 +220,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if uploadId != "" {
-		if maxParts < 0 && ctx.Request().URI().QueryArgs().Has("max-parts") {
-			return SendResponse(ctx,
-				s3err.GetAPIError(s3err.ErrInvalidMaxParts),
-				&MetaOpts{
-					Logger:      c.logger,
-					MetricsMng:  c.mm,
-					Action:      metrics.ActionListParts,
-					BucketOwner: parsedAcl.Owner,
-				})
-		}
 		if partNumberMarker != "" {
 			n, err := strconv.Atoi(partNumberMarker)
 			if err != nil || n < 0 {
@@ -248,8 +237,24 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					})
 			}
 		}
+		mxParts := ctx.Query("max-parts")
+		maxParts, err := utils.ParseUint(mxParts)
+		if err != nil {
+			if c.debug {
+				log.Printf("error parsing max parts %q: %v",
+					mxParts, err)
+			}
+			return SendResponse(ctx,
+				s3err.GetAPIError(s3err.ErrInvalidMaxParts),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionListParts,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
 
-		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
 			AclPermission: types.PermissionRead,
@@ -268,17 +273,13 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
-		var mxParts *int32
-		if ctx.Request().URI().QueryArgs().Has("max-parts") {
-			mxParts = &maxParts
-		}
 
 		res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
 			Bucket:           &bucket,
 			Key:              &key,
 			UploadId:         &uploadId,
 			PartNumberMarker: &partNumberMarker,
-			MaxParts:         mxParts,
+			MaxParts:         &maxParts,
 		})
 		return SendXMLResponse(ctx, res, err,
 			&MetaOpts{
@@ -346,7 +347,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker")
 		maxPartsParsed, err := utils.ParseUint(maxParts)
 		if err != nil {
-			return SendXMLResponse(ctx, nil, err,
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxParts),
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -736,7 +737,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max keys %q: %v",
 					maxkeysStr, err)
 			}
-			return SendXMLResponse(ctx, nil, err,
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -869,12 +870,13 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max uploads %q: %v",
 					maxUploadsStr, err)
 			}
-			return SendXMLResponse(ctx, nil, err, &MetaOpts{
-				Logger:      c.logger,
-				MetricsMng:  c.mm,
-				Action:      metrics.ActionListMultipartUploads,
-				BucketOwner: parsedAcl.Owner,
-			})
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxUploads),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionListMultipartUploads,
+					BucketOwner: parsedAcl.Owner,
+				})
 		}
 		res, err := c.be.ListMultipartUploads(ctx.Context(),
 			&s3.ListMultipartUploadsInput{
@@ -919,7 +921,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max keys %q: %v",
 					maxkeysStr, err)
 			}
-			return SendXMLResponse(ctx, nil, err,
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -970,7 +972,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			log.Printf("error parsing max keys %q: %v",
 				maxkeysStr, err)
 		}
-		return SendXMLResponse(ctx, nil, err,
+		return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
@@ -2243,7 +2245,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{keyStart}, true, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &keyStart}}, true, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2468,10 +2470,7 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = c.be.DeleteBucket(ctx.Context(),
-		&s3.DeleteBucketInput{
-			Bucket: &bucket,
-		})
+	err = c.be.DeleteBucket(ctx.Context(), bucket)
 	return SendResponse(ctx, err,
 		&MetaOpts{
 			Logger:      c.logger,
@@ -2527,7 +2526,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 	// The AWS CLI sends 'True', while Go SDK sends 'true'
 	bypass := strings.EqualFold(bypassHdr, "true")
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, utils.ParseDeleteObjects(dObj.Objects), bypass, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2680,7 +2679,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	// The AWS CLI sends 'True', while Go SDK sends 'true'
 	bypass := strings.EqualFold(bypassHdr, "true")
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{key}, bypass, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key, VersionId: &versionId}}, bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{

s3api/controllers/base_test.go

@@ -382,8 +382,8 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return map[string]string{}, nil
 			},
-			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-				return &s3.GetBucketVersioningOutput{}, nil
+			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+				return s3response.GetBucketVersioningOutput{}, nil
 			},
 			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
 				return s3response.ListVersionsResult{}, nil
@@ -1217,7 +1217,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
+			DeleteBucketFunc: func(_ context.Context, bucket string) error {
 				return nil
 			},
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {

s3api/middlewares/admin.go

@@ -0,0 +1,59 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		acct := ctx.Locals("account").(auth.Account)
+		if acct.Role != auth.RoleAdmin {
+			path := ctx.Path()
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
+				&controllers.MetaOpts{
+					Logger: logger,
+					Action: detectAction(path),
+				})
+		}
+
+		return ctx.Next()
+	}
+}
+
+func detectAction(path string) (action string) {
+	if strings.Contains(path, "create-user") {
+		action = metrics.ActionAdminCreateUser
+	} else if strings.Contains(path, "update-user") {
+		action = metrics.ActionAdminUpdateUser
+	} else if strings.Contains(path, "delete-user") {
+		action = metrics.ActionAdminDeleteUser
+	} else if strings.Contains(path, "list-user") {
+		action = metrics.ActionAdminListUsers
+	} else if strings.Contains(path, "list-buckets") {
+		action = metrics.ActionAdminListBuckets
+	} else if strings.Contains(path, "change-bucket-owner") {
+		action = metrics.ActionAdminChangeBucketOwner
+	}
+	return action
+}

s3api/middlewares/url-decoder.go

@@ -26,12 +26,11 @@ import (
 
 func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		reqURL := ctx.Request().URI().String()
-		decoded, err := url.Parse(reqURL)
+		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
 		if err != nil {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}
-		ctx.Path(decoded.Path)
+		ctx.Path(unescp)
 		return ctx.Next()
 	}
 }

s3api/router.go

@@ -20,6 +20,7 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
 )
@@ -35,22 +36,22 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		adminController := controllers.NewAdminController(iam, be, aLogger)
 
 		// CreateUser admin api
-		app.Patch("/create-user", adminController.CreateUser)
+		app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
 
 		// DeleteUsers admin api
-		app.Patch("/delete-user", adminController.DeleteUser)
+		app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
 
 		// UpdateUser admin api
-		app.Patch("update-user", adminController.UpdateUser)
+		app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
 
 		// ListUsers admin api
-		app.Patch("/list-users", adminController.ListUsers)
+		app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
 
 		// ChangeBucketOwner admin api
-		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+		app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
 
 		// ListBucketsAndOwners admin api
-		app.Patch("/list-buckets", adminController.ListBuckets)
+		app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
 	}
 
 	// ListBuckets action

s3api/utils/utils.go

@@ -39,6 +39,10 @@ var (
 	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
+const (
+	upperhex = "0123456789ABCDEF"
+)
+
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
 	headers.DisableNormalizing()
@@ -64,7 +68,9 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		body = bytes.NewReader(req.Body())
 	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
+	escapedURI := escapeOriginalURI(ctx)
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -174,7 +180,7 @@ func ParseUint(str string) (int32, error) {
 	}
 	num, err := strconv.ParseUint(str, 10, 16)
 	if err != nil {
-		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+		return 1000, fmt.Errorf("invalid uint: %w", err)
 	}
 	return int32(num), nil
 }
@@ -339,3 +345,74 @@ func IsValidOwnership(val types.ObjectOwnership) bool {
 		return false
 	}
 }
+
+func escapeOriginalURI(ctx *fiber.Ctx) string {
+	path := ctx.Path()
+
+	// Escape the URI original path
+	escapedURI := escapePath(path)
+
+	// Add the URI query params
+	query := string(ctx.Request().URI().QueryArgs().QueryString())
+	if query != "" {
+		escapedURI = escapedURI + "?" + query
+	}
+
+	return escapedURI
+}
+
+// Escapes the path string
+// Most of the parts copied from std url
+func escapePath(s string) string {
+	hexCount := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			hexCount++
+		}
+	}
+
+	if hexCount == 0 {
+		return s
+	}
+
+	var buf [64]byte
+	var t []byte
+
+	required := len(s) + 2*hexCount
+	if required <= len(buf) {
+		t = buf[:required]
+	} else {
+		t = make([]byte, required)
+	}
+
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case shouldEscape(c):
+			t[j] = '%'
+			t[j+1] = upperhex[c>>4]
+			t[j+2] = upperhex[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+
+	return string(t)
+}
+
+// Checks if the character needs to be escaped
+func shouldEscape(c byte) bool {
+	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '_', '.', '~', '/':
+		return false
+	}
+
+	return true
+}

s3api/utils/utils_test.go

@@ -382,3 +382,125 @@ func TestIsValidOwnership(t *testing.T) {
 		})
 	}
 }
+
+func Test_shouldEscape(t *testing.T) {
+	type args struct {
+		c byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "shouldn't-escape-alphanum",
+			args: args{
+				c: 'h',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-char",
+			args: args{
+				c: '_',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-number",
+			args: args{
+				c: '0',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-path-separator",
+			args: args{
+				c: '/',
+			},
+			want: false,
+		},
+		{
+			name: "should-escape-special-char-1",
+			args: args{
+				c: '&',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-2",
+			args: args{
+				c: '*',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-3",
+			args: args{
+				c: '(',
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldEscape(tt.args.c); got != tt.want {
+				t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_escapePath(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "empty-string",
+			args: args{
+				s: "",
+			},
+			want: "",
+		},
+		{
+			name: "alphanum-path",
+			args: args{
+				s: "/test-bucket/test-key",
+			},
+			want: "/test-bucket/test-key",
+		},
+		{
+			name: "path-with-unescapable-chars",
+			args: args{
+				s: "/test~bucket/test.key",
+			},
+			want: "/test~bucket/test.key",
+		},
+		{
+			name: "path-with-escapable-chars",
+			args: args{
+				s: "/bucket-*(/test=key&",
+			},
+			want: "/bucket-%2A%28/test%3Dkey%26",
+		},
+		{
+			name: "path-with-space",
+			args: args{
+				s: "/test-bucket/my key",
+			},
+			want: "/test-bucket/my%20key",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := escapePath(tt.args.s); got != tt.want {
+				t.Errorf("escapePath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -57,6 +57,7 @@ const (
 	ErrAccessDenied
 	ErrMethodNotAllowed
 	ErrBucketNotEmpty
+	ErrVersionedBucketNotEmpty
 	ErrBucketAlreadyExists
 	ErrBucketAlreadyOwnedByYou
 	ErrNoSuchBucket
@@ -134,12 +135,21 @@ const (
 	ErrKeyTooLong
 	ErrInvalidVersionId
 	ErrNoSuchVersion
+	ErrSuspendedVersioningNotAllowed
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
 	ErrQuotaExceeded
+	ErrVersioningNotConfigured
+
+	// Admin api errors
+	ErrAdminAccessDenied
+	ErrAdminUserNotFound
+	ErrAdminUserExists
+	ErrAdminInvalidUserRole
+	ErrAdminMissingUserAcess
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -158,6 +168,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrVersionedBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty. You must delete all versions in the bucket.",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrBucketAlreadyExists: {
 		Code:           "BucketAlreadyExists",
 		Description:    "The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again.",
@@ -355,7 +370,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
-		Description:    "The access key ID you provided does not exist in our records.",
+		Description:    "The AWS Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrRequestNotReadyYet: {
@@ -435,12 +450,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
-		Description:    "The specified object does not have an ObjectLock configuration.",
+		Description:    "The specified object does not have a ObjectLock configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidBucketObjectLockConfiguration: {
 		Code:           "InvalidRequest",
-		Description:    "Bucket is missing ObjectLockConfiguration.",
+		Description:    "Bucket is missing Object Lock Configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectLockConfigurationNotAllowed: {
@@ -519,8 +534,9 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrInvalidMetadataDirective: {
-		Code:        "InvalidArgument",
-		Description: "Unknown metadata directive.",
+		Code:           "InvalidArgument",
+		Description:    "Unknown metadata directive.",
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidVersionId: {
 		Code:           "InvalidArgument",
@@ -537,6 +553,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The specified version does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrSuspendedVersioningNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {
@@ -559,6 +580,38 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Your request was denied due to quota exceeded.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrVersioningNotConfigured: {
+		Code:           "VersioningNotConfigured",
+		Description:    "Versioning has not been configured for the gateway.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+
+	// Admin api errors
+	ErrAdminAccessDenied: {
+		Code:           "XAdminAccessDenied",
+		Description:    "Only admin users have access to this resource.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAdminUserNotFound: {
+		Code:           "XAdminUserNotFound",
+		Description:    "No user exists with the provided access key ID.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminUserExists: {
+		Code:           "XAdminUserExists",
+		Description:    "A user with the provided access key ID already exists.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAdminInvalidUserRole: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User role has to be one of the following: 'user', 'admin', 'userplus'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminMissingUserAcess: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User access key ID is missing.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3response/s3response.go

@@ -19,9 +19,14 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
 )
 
-const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+const (
+	iso8601TimeFormat         = "2006-01-02T15:04:05.000Z"
+	iso8601TimeFormatExtended = "2006-01-02T15:04:05.000000Z"
+	iso8601TimeFormatWithTZ   = "2006-01-02T15:04:05-0700"
+)
 
 type PutObjectOutput struct {
 	ETag      string
@@ -45,7 +50,7 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 		Alias: (*Alias)(&p),
 	}
 
-	aux.LastModified = p.LastModified.UTC().Format(RFC3339TimeFormat)
+	aux.LastModified = p.LastModified.UTC().Format(iso8601TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -92,7 +97,7 @@ func (r GetObjectAttributesResult) MarshalXML(e *xml.Encoder, start xml.StartEle
 	}
 
 	if r.LastModified != nil {
-		formattedTime := r.LastModified.UTC().Format(RFC3339TimeFormat)
+		formattedTime := r.LastModified.UTC().Format(iso8601TimeFormat)
 		aux.LastModified = &formattedTime
 	}
 
@@ -179,7 +184,7 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	}
 
 	if o.LastModified != nil {
-		formattedTime := o.LastModified.UTC().Format(RFC3339TimeFormat)
+		formattedTime := o.LastModified.UTC().Format(iso8601TimeFormat)
 		aux.LastModified = &formattedTime
 	}
 
@@ -205,7 +210,7 @@ func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 		Alias: (*Alias)(&u),
 	}
 
-	aux.Initiated = u.Initiated.UTC().Format(RFC3339TimeFormat)
+	aux.Initiated = u.Initiated.UTC().Format(iso8601TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -292,7 +297,7 @@ func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement
 		Alias: (*Alias)(&r),
 	}
 
-	aux.CreationDate = r.CreationDate.UTC().Format(RFC3339TimeFormat)
+	aux.CreationDate = r.CreationDate.UTC().Format(iso8601TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -322,7 +327,7 @@ func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) err
 		Alias: (*Alias)(&r),
 	}
 
-	aux.LastModified = r.LastModified.UTC().Format(RFC3339TimeFormat)
+	aux.LastModified = r.LastModified.UTC().Format(iso8601TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -380,7 +385,71 @@ type ListVersionsResult struct {
 	NextKeyMarker       *string
 	NextVersionIdMarker *string
 	Prefix              *string
-	RequestCharged      types.RequestCharged
 	VersionIdMarker     *string
 	Versions            []types.ObjectVersion `xml:"Version"`
 }
+
+type GetBucketVersioningOutput struct {
+	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
+	MFADelete *types.MFADeleteStatus
+	Status    *types.BucketVersioningStatus
+}
+
+type PutObjectRetentionInput struct {
+	XMLName         xml.Name `xml:"Retention"`
+	Mode            types.ObjectLockRetentionMode
+	RetainUntilDate AmzDate
+}
+
+type AmzDate struct {
+	time.Time
+}
+
+// Parses the date from xml string and validates for predefined date formats
+func (d *AmzDate) UnmarshalXML(e *xml.Decoder, startElement xml.StartElement) error {
+	var dateStr string
+	err := e.DecodeElement(&dateStr, &startElement)
+	if err != nil {
+		return err
+	}
+
+	retDate, err := d.ISO8601Parse(dateStr)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	*d = AmzDate{retDate}
+	return nil
+}
+
+// Encodes expiration date if it is non-zero
+// Encodes empty string if it's zero
+func (d AmzDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if d.IsZero() {
+		return nil
+	}
+	return e.EncodeElement(d.UTC().Format(iso8601TimeFormat), startElement)
+}
+
+// Parses ISO8601 date string to time.Time by
+// validating different time layouts
+func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
+	for _, layout := range []string{
+		iso8601TimeFormat,
+		iso8601TimeFormatExtended,
+		iso8601TimeFormatWithTZ,
+		time.RFC3339,
+	} {
+		t, err = time.Parse(layout, date)
+		if err == nil {
+			return t, nil
+		}
+	}
+
+	return t, err
+}
+
+// Admin api response types
+type ListBucketsResult struct {
+	Buckets []Bucket
+}

tests/.env.default

@@ -27,3 +27,5 @@ USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning
+COMMAND_LOG=command.log

tests/Dockerfile_direct

@@ -47,7 +47,7 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
 
 USER tester
 RUN mkdir -p /home/tester/tests
-COPY --chown=tester:tester . /home/tester/tests
+COPY --chown=tester:tester . /home/tester
 
 # add bats support libraries
 RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests

tests/Dockerfile_test_bats

@@ -86,4 +86,5 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
 ENV WORKSPACE=.
 ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
 
-CMD ["tests/run_all.sh"]
+ENTRYPOINT ["tests/run.sh"]
+CMD ["s3api,s3,s3cmd,mc,rest"]

tests/README.md

@@ -81,4 +81,56 @@ For the s3 backend, see the **S3 Backend** instructions above.
 
 If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries.
 
-A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`
+A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`
+
+## Environment Parameters
+
+**AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**:  identical to the same parameters in **s3**.
+
+**VERSITY_EXE**:  location of the versity executable relative to test folder.
+
+**RUN_VERSITYGW**:  whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
+
+**BACKEND**:  the storage backend type for the gateway, e.g. **posix** or **s3**.
+
+**LOCAL_FOLDER**:  if running with a **posix** backend, the backend storage folder.
+
+**BUCKET_ONE_NAME**, **BUCKET_TWO_NAME**:  test bucket names.
+
+**RECREATE_BUCKETS**:  whether to delete buckets between tests.  If set to false, the bucket will be restored to an original state for the purpose of ensuring consistent tests, but not deleted.
+
+**CERT**, **KEY**:  certificate and key locations if using SSL.
+
+**S3CMD_CONFIG**:  location of **s3cmd** config file if running **s3cmd** tests.
+
+**SECRETS_FILE**:  file where sensitive values, such as **AWS_SECRET_ACCESS_KEY**, should be stored.
+
+**MC_ALIAS**:  Minio MC alias if running MC tests.
+
+**LOG_LEVEL**:  level for test logger (1 - only critical, 2 - errors, 3 - warnings, 4 - info, 5 - debug info, 6 - tracing)
+
+**GOCOVERDIR**:  folder to put golang coverage info in, if checking coverage info.
+
+**USERS_FOLDER**:  folder to use if storing IAM data in a folder.
+
+**IAM_TYPE**:  how to store IAM data (**s3** or **folder**).
+
+**TEST_LOG_FILE**:  log file location for these bats tests.
+
+**VERSITY_LOG_FILE**:  log file for versity application as it is tested by bats tests.
+
+**DIRECT**:  if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes).
+
+**DIRECT_DISPLAY_NAME**:  username if **DIRECT** is set to **true**.
+
+**COVERAGE_DB**:  database to store client command coverage info and usage counts, if using.
+
+**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  credentials for users created and tested for non-root user **versitygw** operations.
+
+**TEST_FILE_FOLDER**:  where to put temporary test files.
+
+**REMOVE_TEST_FILE_FOLDER**:  whether to delete the test file folder between tests, should be set to **true** unless checking the files after a single test, or not yet sure that the test folder is in a safe location to avoid deleting other files.
+
+**VERSIONING_DIR**:  where to put gateway file versioning info.
+
+**COMMAND_LOG**:  where to store list of client commands, which if using will be reported during test failures.

tests/commands/abort_multipart_upload.sh

@@ -20,7 +20,7 @@ abort_multipart_upload() {
     log 2 "'abort multipart upload' command requires bucket, key, upload ID"
     return 1
   fi
-  if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! error=$(send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error aborting upload: $error"
     return 1
   fi
@@ -33,10 +33,10 @@ abort_multipart_upload_with_user() {
     return 1
   fi
   record_command "abort-multipart-upload" "client:s3api"
-  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error aborting upload: $abort_multipart_upload_error"
     export abort_multipart_upload_error
     return 1
   fi
   return 0
-}
+}

tests/commands/command.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/logger.sh
+
+send_command() {
+  if [ $# -eq 0 ]; then
+    return 1
+  fi
+  if [ -n "$COMMAND_LOG" ]; then
+    args=(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" "$@")
+    if ! mask_arg_array "${args[@]}"; then
+      return 1
+    fi
+    # shellcheck disable=SC2154
+    echo "${masked_args[*]}" >> "$COMMAND_LOG"
+    "$@"
+    return $?
+  fi
+  "$@"
+}

tests/commands/complete_multipart_upload.sh

@@ -21,7 +21,7 @@ complete_multipart_upload() {
   fi
   log 5 "complete multipart upload id: $3, parts: $4"
   record_command "complete-multipart-upload" "client:s3api"
-  error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
+  error=$(send_command aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
   if [[ $completed -ne 0 ]]; then
     log 2 "error completing multipart upload: $error"
     return 1

tests/commands/copy_object.sh

@@ -23,14 +23,14 @@ copy_object() {
   local error
   record_command "copy-object" "client:$1"
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4"
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
   else
     echo "'copy-object' not implemented for '$1'"
     return 1
@@ -45,7 +45,7 @@ copy_object() {
 
 copy_object_empty() {
   record-command "copy-object" "client:s3api"
-  error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
+  error=$(send_command aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
   if [[ $result -eq 0 ]]; then
     log 2 "copy object with empty parameters returned no error"
     return 1

tests/commands/create_bucket.sh

@@ -30,14 +30,14 @@ create_bucket() {
   local error
   log 6 "create bucket"
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -56,11 +56,11 @@ create_bucket_with_user() {
   fi
   local exit_code=0
   if [[ $1 == "aws" ]] || [[ $1 == "s3api" ]]; then
-    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -80,7 +80,7 @@ create_bucket_object_lock_enabled() {
   fi
 
   local exit_code=0
-  error=$(aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
+  error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
   if [ $exit_code -ne 0 ]; then
     log 2 "error creating bucket: $error"
     return 1

tests/commands/create_multipart_upload.sh

@@ -24,7 +24,7 @@ create_multipart_upload() {
     return 1
   fi
 
-  if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+  if ! multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
     log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
@@ -44,7 +44,7 @@ create_multipart_upload_with_user() {
     return 1
   fi
 
-  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
     log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
@@ -65,7 +65,7 @@ create_multipart_upload_params() {
     return 1
   fi
   local multipart_data
-  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload \
+  multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload \
     --bucket "$1" \
     --key "$2" \
     --content-type "$3" \
@@ -96,7 +96,7 @@ create_multipart_upload_custom() {
   done
   log 5 "${*:3}"
   log 5 "aws --no-verify-ssl s3api create-multipart-upload --bucket $1 --key $2 ${*:3}"
-  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
+  multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
   if [[ $result -ne 0 ]]; then
     log 2 "error creating custom multipart data command: $multipart_data"
     return 1
@@ -107,3 +107,19 @@ create_multipart_upload_custom() {
   log 5 "upload id: $upload_id"
   return 0
 }
+
+create_multipart_upload_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'create_multipart_upload_rest' requires bucket name, key"
+    return 1
+  fi
+  if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then
+    log 2 "error creating multipart upload: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")"
+    return 1
+  fi
+  return 0
+}

tests/commands/create_presigned_url.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+create_presigned_url() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "create presigned url function requires command type, bucket, and filename"
+    return 1
+  fi
+
+  local presign_result=0
+  if [[ $1 == 'aws' ]]; then
+    presigned_url=$(send_command aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    presigned_url=$(send_command s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    presigned_url_data=$(send_command mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
+    presigned_url="${presigned_url_data#*Share: }"
+  else
+    log 2 "unrecognized command type $1"
+    return 1
+  fi
+  if [[ $presign_result -ne 0 ]]; then
+    log 2 "error generating presigned url: $presigned_url"
+    return 1
+  fi
+  export presigned_url
+  return 0
+}

tests/commands/delete_bucket.sh

@@ -31,13 +31,13 @@ delete_bucket() {
 
   exit_code=0
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
   elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
   else
     log 2 "Invalid command type $1"
     return 1

tests/commands/delete_bucket_policy.sh

@@ -22,11 +22,11 @@ delete_bucket_policy() {
   fi
   local delete_result=0
   if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
+    error=$(send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
+    error=$(send_command mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
   else
     log 2 "command 'delete bucket policy' not implemented for '$1'"
     return 1
@@ -44,7 +44,7 @@ delete_bucket_policy_with_user() {
     log 2 "'delete bucket policy with user' command requires bucket, username, password"
     return 1
   fi
-  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
+  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
     log 2 "error deleting bucket policy: $delete_bucket_policy_error"
     export delete_bucket_policy_error
     return 1

tests/commands/delete_bucket_tagging.sh

@@ -22,9 +22,9 @@ delete_bucket_tagging() {
   fi
   local result
   if [[ $1 == 'aws' ]]; then
-    tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
+    tags=$(send_command aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
+    tags=$(send_command mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -43,7 +43,7 @@ delete_bucket_tagging_with_user() {
     log 2 "delete bucket tagging command missing username, password, bucket name"
     return 1
   fi
-  if ! error=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
+  if ! error=$(send_command AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
     log 2 "error deleting bucket tagging with user: $error"
     return 1
   fi

tests/commands/delete_object.sh

@@ -24,13 +24,15 @@ delete_object() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    delete_object_error=$(aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
+    delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    delete_object_error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(send_command mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 'rest' ]]; then
+    delete_object_rest "$2" "$3" || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -49,13 +51,37 @@ delete_object_bypass_retention() {
     log 2 "'delete-object with bypass retention' requires bucket, key, user, password"
     return 1
   fi
-  if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
+  if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
     log 2 "error deleting object with bypass retention: $delete_object_error"
     return 1
   fi
   return 0
 }
 
+delete_object_version() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete_object_version' requires bucket, key, version ID"
+    return 1
+  fi
+  if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then
+    log 2 "error deleting object version: $delete_object_error"
+    return 1
+  fi
+  return 0
+}
+
+delete_object_version_bypass_retention() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID"
+    return 1
+  fi
+  if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then
+    log 2 "error deleting object version with bypass retention: $delete_object_error"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_with_user() {
   record_command "delete-object" "client:$1"
   if [ $# -ne 5 ]; then
@@ -64,11 +90,11 @@ delete_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
   else
     log 2 "command 'delete object with user' not implemented for '$1'"
     return 1
@@ -79,4 +105,44 @@ delete_object_with_user() {
     return 1
   fi
   return 0
+}
+
+delete_object_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'delete_object_rest' requires bucket name, object name"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="DELETE
+/$1/$2
+
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER"/delete_object_error.txt 2>&1)
+  if [[ "$reply" != "204" ]]; then
+    log 2 "delete object command returned error: $(cat "$TEST_FILE_FOLDER"/delete_object_error.txt)"
+    return 1
+  fi
+  return 0
 }

tests/commands/delete_object_tagging.sh

@@ -20,10 +20,13 @@ delete_object_tagging() {
     echo "delete object tagging command missing command type, bucket, key"
     return 1
   fi
+  delete_result=0
   if [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
+    error=$(send_command aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
+    error=$(send_command mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
+  elif [ "$1" == 'rest' ]; then
+    delete_object_tagging_rest "$2" "$3" || delete_result=$?
   else
     echo "delete-object-tagging command not implemented for '$1'"
     return 1
@@ -33,4 +36,46 @@ delete_object_tagging() {
     return 1
   fi
   return 0
-}
+}
+
+delete_object_tagging_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'delete_object_tagging' requires bucket, key"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="DELETE
+/$1/$2
+tagging=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -d "$tagging" -o "$TEST_FILE_FOLDER"/delete_tagging_error.txt 2>&1)
+  log 5 "reply status code: $reply"
+  if [[ "$reply" != "204" ]]; then
+    log 2 "reply error: $reply"
+    log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/delete_tagging_error.txt)"
+    return 1
+  fi
+  return 0
+}

tests/commands/delete_objects.sh

@@ -20,7 +20,7 @@ delete_objects() {
     log 2 "'delete-objects' command requires bucket name, two object keys"
     return 1
   fi
-  if ! error=$(aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
+  if ! error=$(send_command aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
       \"Objects\": [
         {\"Key\": \"$2\"},
         {\"Key\": \"$3\"}

tests/commands/get_bucket_acl.sh

@@ -22,9 +22,9 @@ get_bucket_acl() {
   fi
   local exit_code=0
   if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    acl=$(aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
+    acl=$(send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
-    acl=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
+    acl=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
   else
     log 2 "command 'get bucket acl' not implemented for $1"
     return 1
@@ -42,7 +42,7 @@ get_bucket_acl_with_user() {
     log 2 "'get bucket ACL with user' command requires bucket name, username, password"
     return 1
   fi
-  if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
+  if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
     log 2 "error getting bucket ACLs: $bucket_acl"
     return 1
   fi

tests/commands/get_bucket_location.sh

@@ -42,7 +42,7 @@ get_bucket_location_aws() {
     echo "get bucket location (aws) requires bucket name"
     return 1
   fi
-  location_json=$(aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
+  location_json=$(send_command aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
   if [[ $location_result -ne 0 ]]; then
     echo "error getting bucket location: $location"
     return 1
@@ -57,7 +57,7 @@ get_bucket_location_s3cmd() {
     echo "get bucket location (s3cmd) requires bucket name"
     return 1
   fi
-  info=$(s3cmd --no-check-certificate info "s3://$1") || results=$?
+  info=$(send_command s3cmd --no-check-certificate info "s3://$1") || results=$?
   if [[ $results -ne 0 ]]; then
     echo "error getting s3cmd info: $info"
     return 1
@@ -72,7 +72,7 @@ get_bucket_location_mc() {
     echo "get bucket location (mc) requires bucket name"
     return 1
   fi
-  info=$(mc --insecure stat "$MC_ALIAS/$1") || results=$?
+  info=$(send_command mc --insecure stat "$MC_ALIAS/$1") || results=$?
   if [[ $results -ne 0 ]]; then
     echo "error getting s3cmd info: $info"
     return 1

tests/commands/get_bucket_ownership_controls.sh

@@ -26,7 +26,7 @@ get_bucket_ownership_controls() {
     return 1
   fi
 
-  if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
+  if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
     log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
     return 1
   fi

tests/commands/get_bucket_policy.sh

@@ -44,7 +44,7 @@ get_bucket_policy_aws() {
     log 2 "aws 'get bucket policy' command requires bucket"
     return 1
   fi
-  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
+  policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
   policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
   log 5 "$policy_json"
   if [[ $get_result -ne 0 ]]; then
@@ -66,7 +66,7 @@ get_bucket_policy_with_user() {
     log 2 "'get bucket policy with user' command requires bucket, username, password"
     return 1
   fi
-  if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
+  if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
     policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
     bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
   else
@@ -87,7 +87,7 @@ get_bucket_policy_s3cmd() {
     return 1
   fi
 
-  if ! info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
+  if ! info=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
     log 2 "error getting bucket policy: $info"
     return 1
   fi
@@ -129,7 +129,7 @@ get_bucket_policy_mc() {
     echo "aws 'get bucket policy' command requires bucket"
     return 1
   fi
-  bucket_policy=$(mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
+  bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
   if [[ $get_result -ne 0 ]]; then
     echo "error getting policy: $bucket_policy"
     return 1

tests/commands/get_bucket_tagging.sh

@@ -22,9 +22,9 @@ get_bucket_tagging() {
   record_command "get-bucket-tagging" "client:$1"
   local result
   if [[ $1 == 'aws' ]]; then
-    tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
+    tags=$(send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
+    tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
   else
     fail "invalid command type $1"
   fi
@@ -49,7 +49,7 @@ get_bucket_tagging_with_user() {
   fi
   record_command "get-bucket-tagging" "client:s3api"
   local result
-  if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then
+  if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then
     log 5 "tags error: $tags"
     if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
       export tags=

tests/commands/get_bucket_versioning.sh

@@ -17,15 +17,32 @@
 get_bucket_versioning() {
   record_command "get-bucket-versioning" "client:s3api"
   if [[ $# -ne 2 ]]; then
-    log 2 "put bucket versioning command requires command type, bucket name"
+    log 2 "get bucket versioning command requires command type, bucket name"
     return 1
   fi
   local get_result=0
   if [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
+    versioning=$(send_command aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
   fi
   if [[ $get_result -ne 0 ]]; then
-    log 2 "error getting bucket versioning: $error"
+    log 2 "error getting bucket versioning: $versioning"
+    return 1
+  fi
+  return 0
+}
+
+get_bucket_versioning_rest() {
+  log 6 "get_object_rest"
+  if [ $# -ne 1 ]; then
+    log 2 "'get_bucket_versioning_rest' requires bucket name"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OUTPUT_FILE="$TEST_FILE_FOLDER/versioning.txt" ./tests/rest_scripts/get_bucket_versioning.sh); then
+    log 2 "error getting bucket versioning: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "get-bucket-versioning returned code $result: $(cat "$TEST_FILE_FOLDER/versioning.txt")"
     return 1
   fi
   return 0

tests/commands/get_object.sh

@@ -23,13 +23,15 @@ get_object() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    get_object_error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(send_command aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 'rest' ]]; then
+    get_object_rest "$2" "$3" "$4" || exit_code=$?
   else
     log 2 "'get object' command not implemented for '$1'"
     return 1
@@ -48,8 +50,7 @@ get_object_with_range() {
     log 2 "'get object with range' requires bucket, key, range, outfile"
     return 1
   fi
-  get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1) || local exit_code=$?
-  if [[ $exit_code -ne 0 ]]; then
+  if ! get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1); then
     log 2 "error getting object with range: $get_object_error"
     return 1
   fi
@@ -65,13 +66,13 @@ get_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]] || [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
     log 5 "s3cmd filename: $3"
-    get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
     log 5 "save location: $4"
-    get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
   else
     log 2 "'get_object_with_user' not implemented for client '$1'"
     return 1
@@ -83,3 +84,45 @@ get_object_with_user() {
   fi
   return 0
 }
+
+get_object_rest() {
+  log 6 "get_object_rest"
+  if [ $# -ne 3 ]; then
+    log 2 "'get_object_rest' requires bucket name, object name, output file"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1/$2
+
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1/$2" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$3" 2>&1)
+  log 5 "reply: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "get object command returned error: $(cat "$3")"
+    return 1
+  fi
+  return 0
+}

tests/commands/get_object_attributes.sh

@@ -20,7 +20,7 @@ get_object_attributes() {
     log 2 "'get object attributes' command requires bucket, key"
     return 1
   fi
-  attributes=$(aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$?
+  attributes=$(send_command aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$?
   if [[ $get_result -ne 0 ]]; then
     log 2 "error getting object attributes: $attributes"
     return 1

tests/commands/get_object_legal_hold.sh

@@ -20,10 +20,26 @@ get_object_legal_hold() {
     return 1
   fi
   record_command "get-object-legal-hold" "client:s3api"
-  legal_hold=$(aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
+  legal_hold=$(send_command aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
   if [[ $get_result -ne 0 ]]; then
     log 2 "error getting object legal hold: $legal_hold"
     return 1
   fi
   return 0
+}
+
+get_object_legal_hold_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'get_object_legal_hold_rest' requires bucket, key"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/legal_hold.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then
+    log 2 "error getting object legal hold: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")"
+    return 1
+  fi
+  return 0
 }

tests/commands/get_object_lock_configuration.sh

@@ -20,7 +20,7 @@ get_object_lock_configuration() {
     log 2 "'get object lock configuration' command missing bucket name"
     return 1
   fi
-  if ! lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
+  if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
     log 2 "error obtaining lock config: $lock_config"
     # shellcheck disable=SC2034
     get_object_lock_config_err=$lock_config
@@ -28,4 +28,44 @@ get_object_lock_configuration() {
   fi
   lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning")
   return 0
+}
+
+get_object_lock_configuration_rest() {
+  log 6 "get_object_lock_configuration_rest"
+  if [ $# -ne 1 ]; then
+    log 2 "'get_object_lock_configuration_rest' requires bucket name"
+    return 1
+  fi
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1
+object-lock=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
+  log 5 "reply: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
+    return 1
+  fi
+  return 0
 }

tests/commands/get_object_retention.sh

@@ -20,11 +20,57 @@ get_object_retention() {
     log 2 "'get object retention' command requires bucket, key"
     return 1
   fi
-  if ! retention=$(aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then
+  if ! retention=$(send_command aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then
     log 2 "error getting object retention: $retention"
     get_object_retention_error=$retention
     export get_object_retention_error
     return 1
   fi
   return 0
-}
+}
+
+get_object_retention_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'get_object_tagging_rest' requires bucket, key"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1/$2
+retention=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?retention" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER"/object_retention.txt 2>&1)
+  log 5 "reply status code: $reply"
+  if [[ "$reply" != "200" ]]; then
+    if [ "$reply" == "404" ]; then
+      return 1
+    fi
+    log 2 "reply error: $reply"
+    log 2 "get object retention command returned error: $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
+    return 2
+  fi
+  log 5 "object tags:  $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
+  return 0
+}

tests/commands/get_object_tagging.sh

@@ -21,10 +21,12 @@ get_object_tagging() {
     return 1
   fi
   local result
-  if [[ $1 == 'aws' ]]; then
-    tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
-  elif [[ $1 == 'mc' ]]; then
-    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
+  if [[ "$1" == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    tags=$(send_command aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
+  elif [[ "$1" == 'mc' ]]; then
+    tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
+  elif [ "$1" == 'rest' ]; then
+    get_object_tagging_rest "$2" "$3" || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -41,4 +43,50 @@ get_object_tagging() {
     tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
   fi
   export tags
+}
+
+get_object_tagging_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'get_object_tagging' requires bucket, key"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1/$2
+tagging=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?tagging" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER"/object_tags.txt 2>&1)
+  log 5 "reply status code: $reply"
+  if [[ "$reply" != "200" ]]; then
+    if [ "$reply" == "404" ]; then
+      return 1
+    fi
+    log 2 "reply error: $reply"
+    log 2 "get object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/object_tags.txt)"
+    return 2
+  fi
+  log 5 "object tags:  $(cat "$TEST_FILE_FOLDER"/object_tags.txt)"
+  return 0
 }

tests/commands/head_bucket.sh

@@ -30,11 +30,11 @@ head_bucket() {
   fi
   local exit_code=0
   if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
-    bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
+    bucket_info=$(send_command aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    bucket_info=$(s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
+    bucket_info=$(send_command s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     fail "invalid command type $1"
   fi

tests/commands/head_object.sh

@@ -22,11 +22,11 @@ head_object() {
   fi
   local exit_code=0
   if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
-    metadata=$(aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?"
+    metadata=$(send_command aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
-    metadata=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?"
+    metadata=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?"
   elif [[ $1 == 'mc' ]]; then
-    metadata=$(mc --insecure stat "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+    metadata=$(send_command mc --insecure stat "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 2

tests/commands/list_buckets.sh

@@ -24,13 +24,13 @@ list_buckets() {
 
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    buckets=$(aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
+    buckets=$(send_command aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
     list_buckets_s3api "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    buckets=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
+    buckets=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    buckets=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
+    buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
   elif [[ $1 == 'rest' ]]; then
     list_buckets_rest || exit_code=$?
   else
@@ -63,13 +63,13 @@ list_buckets_with_user() {
 
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    buckets=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
+    buckets=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
     list_buckets_s3api "$2" "$3" || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    buckets=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$2" --secret_key="$3" ls s3:// 2>&1) || exit_code=$?
+    buckets=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$2" --secret_key="$3" ls s3:// 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    buckets=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
+    buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
   else
     echo "list buckets command not implemented for '$1'"
     return 1
@@ -96,7 +96,7 @@ list_buckets_s3api() {
     log 2 "'list_buckets_s3api' requires username, password"
     return 1
   fi
-  if ! output=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api list-buckets 2>&1); then
+  if ! output=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api list-buckets 2>&1); then
     echo "error listing buckets: $output"
     return 1
   fi
@@ -117,30 +117,14 @@ list_buckets_s3api() {
 }
 
 list_buckets_rest() {
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/
-
-host:${AWS_ENDPOINT_URL#*//}
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(COMMAND_LOG=$COMMAND_LOG OUTPUT_FILE="$TEST_FILE_FOLDER/buckets.txt" ./tests/rest_scripts/list_buckets.sh); then
+    log 2 "error listing buckets: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "list-buckets returned code $result: $(cat "$TEST_FILE_FOLDER/buckets.txt")"
     return 1
   fi
-
-  get_signature
-  # shellcheck disable=SC2034,SC2154
-  reply=$(curl -ks "$AWS_ENDPOINT_URL" \
-         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-         -H "x-amz-content-sha256: $payload_hash" \
-         -H "x-amz-date: $current_date_time" 2>&1)
   parse_bucket_list
+  return 0
 }

tests/commands/list_multipart_uploads.sh

@@ -20,7 +20,7 @@ list_multipart_uploads() {
     log 2 "'list multipart uploads' command requires bucket name"
     return 1
   fi
-  if ! uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+  if ! uploads=$(send_command aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
     log 2 "error listing uploads: $uploads"
     return 1
   fi
@@ -32,7 +32,7 @@ list_multipart_uploads_with_user() {
     log 2 "'list multipart uploads' command requires bucket name, username, password"
     return 1
   fi
-  if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+  if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
     log 2 "error listing uploads: $uploads"
     # shellcheck disable=SC2034
     list_multipart_uploads_error=$uploads

tests/commands/list_object_versions.sh

@@ -20,10 +20,44 @@ list_object_versions() {
     log 2 "'list object versions' command requires bucket name"
     return 1
   fi
-  versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1") || local list_result=$?
+  versions=$(send_command aws --no-verify-ssl s3api list-object-versions --bucket "$1" 2>&1) || local list_result=$?
   if [[ $list_result -ne 0 ]]; then
     log 2 "error listing object versions: $versions"
     return 1
   fi
+  versions=$(echo "$versions" | grep -v "InsecureRequestWarning")
   return 0
+}
+
+list_object_versions_rest() {
+  if [ $# -ne 1 ]; then
+    log 2 "'list_object_versions_rest' requires bucket name"
+    return 1
+  fi
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1
+versions=
+host:${AWS_ENDPOINT_URL#*//}
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+
+  get_signature
+  # shellcheck disable=SC2034,SC2154
+  reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \
+         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+         -H "x-amz-content-sha256: $payload_hash" \
+         -H "x-amz-date: $current_date_time" \
+         -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1)
 }

tests/commands/list_objects.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 source ./tests/util_list_objects.sh
+source ./tests/commands/command.sh
 
 # Copyright 2024 Versity Software
 # This file is licensed under the Apache License, Version 2.0
@@ -29,14 +30,14 @@ list_objects() {
   local output
   local result=0
   if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then
-    output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$?
+    output=$(send_command aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$?
   elif [[ $1 == 's3api' ]]; then
     list_objects_s3api "$2" || result=$?
     return $result
   elif [[ $1 == 's3cmd' ]]; then
-    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$?
+    output=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$?
+    output=$(send_command mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$?
   elif [[ $1 == 'rest' ]]; then
     list_objects_rest "$2" || result=$?
     return $result
@@ -66,7 +67,7 @@ list_objects_s3api() {
     log 2 "'list_objects_s3api' requires bucket"
     return 1
   fi
-  if ! output=$(aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then
+  if ! output=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then
     log 2 "error listing objects: $output"
     return 1
   fi
@@ -94,9 +95,9 @@ list_objects_s3api_v1() {
     return 1
   fi
   if [ "$2" == "" ]; then
-    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+    objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
   else
-    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
+    objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
   fi
   if [[ $result -ne 0 ]]; then
     echo "error listing objects: $objects"
@@ -112,13 +113,13 @@ list_objects_with_prefix() {
   fi
   local result=0
   if [ "$1" == 's3' ]; then
-    objects=$(aws --no-verify-ssl s3 ls s3://"$2/$3" 2>&1) || result=$?
+    objects=$(send_command aws --no-verify-ssl s3 ls s3://"$2/$3" 2>&1) || result=$?
   elif [ "$1" == 's3api' ]; then
-    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$2" --prefix "$3" 2>&1) || result=$?
+    objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$2" --prefix "$3" 2>&1) || result=$?
   elif [ "$1" == 's3cmd' ]; then
-    objects=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || result=$?
+    objects=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || result=$?
   elif [[ "$1" == 'mc' ]]; then
-    objects=$(mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || result=$?
+    objects=$(send_command mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || result=$?
   else
     log 2 "invalid command type '$1'"
     return 1
@@ -162,7 +163,7 @@ $payload_hash"
   fi
   get_signature
   # shellcheck disable=SC2154
-  reply=$(curl -ks "$header://$aws_endpoint_url_address/$1" \
+  reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \
     -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
     -H "x-amz-content-sha256: $payload_hash" \
     -H "x-amz-date: $current_date_time" 2>&1)

tests/commands/list_objects_v2.sh

@@ -23,7 +23,7 @@ list_objects_v2() {
     return 1
   fi
   record_command "list-objects-v2 client:s3api"
-  objects=$(aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$?
+  objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$?
   if [[ $result -ne 0 ]]; then
     echo "error listing objects: $objects"
     return 1

tests/commands/list_parts.sh

@@ -20,7 +20,7 @@ list_parts() {
     return 1
   fi
   record_command "list-parts" "client:s3api"
-  if ! listed_parts=$(aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! listed_parts=$(send_command aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error listing multipart upload parts: $listed_parts"
     return 1
   fi
@@ -32,7 +32,7 @@ list_parts_with_user() {
     return 1
   fi
   record_command 'list-parts' 'client:s3api'
-  if ! listed_parts=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api list-parts --bucket "$3" --key "$4" --upload-id "$5" 2>&1); then
+  if ! listed_parts=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api list-parts --bucket "$3" --key "$4" --upload-id "$5" 2>&1); then
     log 2 "Error listing multipart upload parts: $listed_parts"
     return 1
   fi

tests/commands/put_bucket_acl.sh

@@ -15,6 +15,7 @@
 # under the License.
 
 source ./tests/util_file.sh
+source ./tests/commands/command.sh
 
 put_bucket_acl_s3api() {
   log 6 "put_bucket_acl_s3api"
@@ -24,7 +25,7 @@ put_bucket_acl_s3api() {
     return 1
   fi
   log 5 "bucket name: $1, acls: $2"
-  if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
+  if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
     log 2 "error putting bucket acl: $error"
     return 1
   fi
@@ -39,7 +40,7 @@ put_bucket_acl_s3api_with_user() {
     return 1
   fi
   log 5 "bucket name: $1, acls: $2"
-  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
+  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
     log 2 "error putting bucket acl: $error"
     return 1
   fi
@@ -87,7 +88,7 @@ put_bucket_canned_acl_s3cmd() {
     log 2 "put bucket acl command requires bucket name, permission"
     return 1
   fi
-  if ! error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$1" "$2" 2>&1); then
+  if ! error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$1" "$2" 2>&1); then
     log 2 "error putting s3cmd canned ACL:  $error"
     return 1
   fi
@@ -100,7 +101,7 @@ put_bucket_canned_acl() {
     return 1
   fi
   record_command "put-bucket-acl" "client:s3api"
-  if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
+  if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
     log 2 "error re-setting bucket acls: $error"
     return 1
   fi
@@ -113,7 +114,7 @@ put_bucket_canned_acl_with_user() {
     return 1
   fi
   record_command "put-bucket-acl" "client:s3api"
-  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
+  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then
     log 2 "error re-setting bucket acls: $error"
     return 1
   fi

tests/commands/put_bucket_ownership_controls.sh

@@ -22,9 +22,13 @@ put_bucket_ownership_controls() {
   fi
 
   log 6 "put_bucket_ownership_controls"
+  if [ $# -ne 2 ]; then
+    log 2 "'put_bucket_ownership_controls' requires bucket name, rule"
+    return 1
+  fi
   record_command "put-bucket-ownership-controls" "client:s3api"
-  assert [ $# -eq 2 ]
-  run aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" --ownership-controls="Rules=[{ObjectOwnership=$2}]"
-  # shellcheck disable=SC2154
-  assert_success "error putting bucket ownership controls: $output"
+  if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" --ownership-controls="Rules=[{ObjectOwnership=$2}]" 2>&1); then
+    log 2 "error putting bucket ownership controls: $error"
+    return 1
+  fi
 }

tests/commands/put_bucket_policy.sh

@@ -22,11 +22,11 @@ put_bucket_policy() {
   fi
   local put_policy_result=0
   if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$?
+    policy=$(send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$?
+    policy=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
-    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$?
+    policy=$(send_command mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$?
   else
     log 2 "command 'put bucket policy' not implemented for '$1'"
     return 1
@@ -46,7 +46,7 @@ put_bucket_policy_with_user() {
     log 2 "'put bucket policy with user' command requires bucket, policy file, username, password"
     return 1
   fi
-  if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then
+  if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then
     log 2 "error putting bucket policy with user $3: $policy"
     put_bucket_policy_error=$policy
     export put_bucket_policy_error

tests/commands/put_bucket_tagging.sh

@@ -24,9 +24,9 @@ put_bucket_tagging() {
   local result=0
   record_command "put-bucket-tagging" "client:$1"
   if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
+    error=$(send_command aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$?
+    error=$(send_command mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -42,7 +42,7 @@ put_bucket_tagging_with_user() {
   log 6 "put_bucket_tagging_with_user"
   assert [ $# -eq 5 ]
   record_command "put-bucket-tagging" "client:$1"
-  if ! error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]"); then
+  if ! error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]"); then
     log 2 "error putting bucket tagging: $error"
     return 1
   fi

tests/commands/put_bucket_versioning.sh

@@ -22,11 +22,27 @@ put_bucket_versioning() {
   fi
   local put_result=0
   if [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api put-bucket-versioning --bucket "$2" --versioning-configuration "{ \"Status\": \"$3\"}" 2>&1) || put_result=$?
+    error=$(send_command aws --no-verify-ssl s3api put-bucket-versioning --bucket "$2" --versioning-configuration "{ \"Status\": \"$3\"}" 2>&1) || put_result=$?
   fi
   if [[ $put_result -ne 0 ]]; then
     log 2 "error putting bucket versioning: $error"
     return 1
   fi
   return 0
+}
+
+put_bucket_versioning_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'put_bucket_versioning_rest' requires bucket, 'Enabled' or 'Suspended'"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 STATUS="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/error.txt" ./tests/rest_scripts/put_bucket_versioning.sh); then
+    log 2 "error putting bucket versioning: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "put-bucket-versioning returned code $result: $(cat "$TEST_FILE_FOLDER/error.txt")"
+    return 1
+  fi
+  return 0
 }

tests/commands/put_object.sh

@@ -26,13 +26,15 @@ put_object() {
   local exit_code=0
   local error
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 mv "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3 mv "$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure put "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure put "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 'rest' ]]; then
+    put_object_rest "$2" "$3" "$4" || exit_code=$?
   else
     log 2 "'put object' command not implemented for '$1'"
     return 1
@@ -53,7 +55,7 @@ put_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    put_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    put_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" send_command aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   else
     log 2 "'put object with user' command not implemented for '$1'"
     return 1
@@ -66,3 +68,43 @@ put_object_with_user() {
   fi
   return 0
 }
+
+put_object_rest() {
+  if [ $# -ne 3 ]; then
+    log 2 "'put_object_rest' requires local file, bucket name, key"
+    return 1
+  fi
+
+  generate_hash_for_payload_file "$1"
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="PUT
+/$2/$3
+
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$2/$3" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -T "$1" -o "$TEST_FILE_FOLDER"/put_object_error.txt 2>&1)
+  if [[ "$reply" != "200" ]]; then
+    log 2 "put object command returned error: $(cat "$TEST_FILE_FOLDER"/put_object_error.txt)"
+    return 1
+  fi
+  return 0
+}

tests/commands/put_object_legal_hold.sh

@@ -21,7 +21,7 @@ put_object_legal_hold() {
     return 1
   fi
   local error=""
-  error=$(aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1) || local put_hold_result=$?
+  error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1) || local put_hold_result=$?
   if [[ $put_hold_result -ne 0 ]]; then
     log 2 "error putting object legal hold: $error"
     return 1

tests/commands/put_object_lock_configuration.sh

@@ -20,7 +20,7 @@ put_object_lock_configuration() {
     return 1
   fi
   local config="{\"ObjectLockEnabled\": \"$2\", \"Rule\": {\"DefaultRetention\": {\"Mode\": \"$3\", \"Days\": $4}}}"
-  if ! error=$(aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
+  if ! error=$(send_command aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
     log 2 "error putting object lock configuration: $error"
     return 1
   fi
@@ -33,7 +33,7 @@ put_object_lock_configuration_disabled() {
     return 1
   fi
   local config="{\"ObjectLockEnabled\": \"Enabled\"}"
-  if ! error=$(aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
+  if ! error=$(send_command aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
     log 2 "error putting object lock configuration: $error"
     return 1
   fi

tests/commands/put_object_retention.sh

@@ -20,10 +20,26 @@ put_object_retention() {
     log 2 "'put object retention' command requires bucket, key, retention mode, retention date"
     return 1
   fi
-  error=$(aws --no-verify-ssl s3api put-object-retention --bucket "$1" --key "$2" --retention "{\"Mode\": \"$3\", \"RetainUntilDate\": \"$4\"}" 2>&1) || local put_result=$?
+  error=$(send_command aws --no-verify-ssl s3api put-object-retention --bucket "$1" --key "$2" --retention "{\"Mode\": \"$3\", \"RetainUntilDate\": \"$4\"}" 2>&1) || local put_result=$?
   if [[ $put_result -ne 0 ]]; then
     log 2 "error putting object retention:  $error"
     return 1
   fi
   return 0
-}
+}
+
+put_object_retention_rest() {
+  if [ $# -ne 4 ]; then
+    log 2 "'put_object_retention_rest' requires bucket, key, retention mode, retention date"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME="$1" OBJECT_KEY="$2" RETENTION_MODE="$3" RETAIN_UNTIL_DATE="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/error.txt" ./tests/rest_scripts/put_object_retention.sh); then
+    log 2 "error putting object retention: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/error.txt")"
+    return 1
+  fi
+  return 0
+}

tests/commands/put_object_tagging.sh

@@ -16,16 +16,18 @@
 
 put_object_tagging() {
   if [ $# -ne 5 ]; then
-    log 2 "'put-object-tagging' command missing command type, object name, file, key, and/or value"
+    log 2 "'put-object-tagging' command missing command type, bucket, object name, file, key, and/or value"
     return 1
   fi
   local error
   local result
   record_command "put-object-tagging" "client:$1"
   if [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$?
+    error=$(send_command aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$?
+    error=$(send_command mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$?
+  elif [[ $1 == 'rest' ]]; then
+    put_object_tagging_rest "$2" "$3" "$4" "$5" || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -35,4 +37,56 @@ put_object_tagging() {
     return 1
   fi
   return 0
-}
+}
+
+put_object_tagging_rest() {
+  if [ $# -ne 4 ]; then
+    log 2 "'put_object_tagging' requires bucket, key, tag key, tag value"
+    return 1
+  fi
+
+  tagging="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+  <TagSet>
+    <Tag>
+      <Key>$3</Key>
+      <Value>$4</Value>
+    </Tag>
+  </TagSet>
+</Tagging>"
+
+  generate_hash_for_payload "$tagging"
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="PUT
+/$1/$2
+tagging=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?tagging" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -d "$tagging" -o "$TEST_FILE_FOLDER"/put_tagging_error.txt 2>&1)
+  log 5 "reply status code: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "reply error: $reply"
+    log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/put_tagging_error.txt)"
+    return 1
+  fi
+  return 0
+}

tests/commands/put_public_access_block.sh

@@ -19,7 +19,7 @@ put_public_access_block() {
     log 2 "'put_public_access_block' command requires bucket, access block list"
     return 1
   fi
-  if ! error=$(aws --no-verify-ssl s3api put-public-access-block --bucket "$1" --public-access-block-configuration "$2"); then
+  if ! error=$(send_command aws --no-verify-ssl s3api put-public-access-block --bucket "$1" --public-access-block-configuration "$2"); then
     log 2 "error updating public access block: $error"
     return 1
   fi

tests/commands/select_object_content.sh

@@ -20,7 +20,7 @@ select_object_content() {
     log 2 "'select object content' command requires bucket, key, expression, expression type, input serialization, output serialization, outfile"
     return 1
   fi
-  error=$(aws --no-verify-ssl s3api select-object-content \
+  error=$(send_command aws --no-verify-ssl s3api select-object-content \
     --bucket "$1" \
     --key "$2" \
     --expression "$3" \

tests/commands/upload_part.sh

@@ -21,7 +21,7 @@ upload_part() {
   fi
   local etag_json
   record_command "upload-part" "client:s3api"
-  if ! etag_json=$(aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then
+  if ! etag_json=$(send_command aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then
     log 2 "Error uploading part $5: $etag_json"
     return 1
   fi
@@ -30,4 +30,4 @@ upload_part() {
     return 1
   fi
   export etag
-}
+}

tests/commands/upload_part_copy.sh

@@ -22,7 +22,7 @@ upload_part_copy() {
   fi
   local etag_json
   echo "$1 $2 $3 $4 $5"
-  etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
+  etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
   if [[ $uploaded -ne 0 ]]; then
     echo "Error uploading part $5: $etag_json"
     return 1
@@ -39,7 +39,7 @@ upload_part_copy_with_range() {
   fi
   local etag_json
   log 5 "bucket: $1, key: $2, upload ID: $3, file name: $4, range: $5, copy source range: $6"
-  etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$?
+  etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$?
   if [[ $uploaded -ne 0 ]]; then
     log 2 "Error uploading part $5: $etag_json"
     export upload_part_copy_error=$etag_json

tests/docker-compose-bats.yml

@@ -1,34 +1,36 @@
 services:
   no_certs:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.nocerts
   static_buckets:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.static
   posix_backend:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.default
     image: bats_test
   s3_backend:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.s3
         - SECRETS_FILE=tests/.secrets.s3
+  s3api_np_only:
+    build:
+      dockerfile: tests/Dockerfile_test_bats
+      args:
+        - CONFIG_FILE=tests/.env.default
+    image: bats_test
+    command: ["s3api-non-policy"]
   direct:
     build:
-      context: .
-      dockerfile: Dockerfile_direct
+      dockerfile: tests/Dockerfile_direct
     volumes:
       - ./.env.direct:/home/tester/tests/.env.direct
       - ./.secrets.direct:/home/tester/tests/.secrets.direct

tests/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   posix:
     build: 
       context: .
-      dockerfile: ./Dockerfile.dev
+      dockerfile: tests/Dockerfile.dev
       args:
         - IAM_DIR=${IAM_DIR}
         - SETUP_DIR=${SETUP_DIR}
@@ -14,7 +14,7 @@ services:
   proxy:
     build: 
       context: .
-      dockerfile: ./Dockerfile.dev
+      dockerfile: tests/Dockerfile.dev
     volumes:
       - ./:/app
     ports:
@@ -34,9 +34,9 @@ services:
   azuritegw:
     build:
       context: .
-      dockerfile: ./Dockerfile.dev
+      dockerfile: tests/Dockerfile.dev
     volumes:
       - ./:/app
     ports:
       - 7070:7070
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]

tests/env.sh

@@ -62,6 +62,15 @@ check_universal_vars() {
   if [[ $BYPASS_ENV_FILE != "true" ]]; then
     source_config_file
   fi
+  if [ -n "$COMMAND_LOG" ]; then
+    if [ -e "$COMMAND_LOG" ]; then
+      if ! error=$(rm "$COMMAND_LOG"); then
+        log 3 "error removing command log: $error"
+        return 1
+      fi
+    fi
+    echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG"
+  fi
 
   if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
     # shellcheck source=./tests/.secrets
@@ -99,7 +108,6 @@ check_universal_vars() {
   if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
     fail "RUN_VERSITYGW must be 'true' or 'false'"
   fi
-<<<<<<< HEAD
 
   if [ -z "$BUCKET_ONE_NAME" ]; then
     log 1 "BUCKET_ONE_NAME missing"
@@ -114,29 +122,18 @@ check_universal_vars() {
     exit 1
   fi
   if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then
-<<<<<<< HEAD
     log 1 "RECREATE_BUCKETS must be 'true' or 'false'"
     exit 1
   fi
   if [ -z "$TEST_FILE_FOLDER" ]; then
     log 1 "TEST_FILE_FOLDER missing"
     exit 1
-=======
-    fail "RECREATE_BUCKETS must be 'true' or 'false'"
-=======
-  if [[ -n "$VERSITY_LOG_FILE" ]]; then
-    export VERSITY_LOG_FILE
   fi
-  if [[ -n "$DIRECT" ]]; then
-    export DIRECT
-  fi
-  if [[ -n "$DIRECT_DISPLAY_NAME" ]]; then
-    export DIRECT_DISPLAY_NAME
-  fi
-  if [[ -n "$COVERAGE_DB" ]]; then
-    export COVERAGE_DB
->>>>>>> 95e62d2 (fix: Merge conflicts resolved)
->>>>>>> 8595c19 (feat: Added integration tests for bucket object versioning. Made a couple of bug fixes in the versioning implementation)
+  if [ ! -d "$TEST_FILE_FOLDER" ]; then
+    if ! error=$(mkdir -p "$TEST_FILE_FOLDER"); then
+      log 2 "error creating test folder: $error"
+      exit 1
+    fi
   fi
   # exporting these since they're needed for subshells
   export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
@@ -147,6 +144,18 @@ check_versity_vars() {
     log 1 "LOCAL_FOLDER missing"
     exit 1
   fi
+  if [ ! -d "$LOCAL_FOLDER" ]; then
+    if ! error=$(mkdir -p "$LOCAL_FOLDER"); then
+      log 2 "error creating local posix folder: $error"
+      exit 1
+    fi
+  fi
+  if [ -n "$VERSIONING_DIR" ] && [ ! -d "$VERSIONING_DIR" ]; then
+    if ! error=$(mkdir -p "$VERSIONING_DIR"); then
+      log 2 "error creating versioning folder: $error"
+      return 1
+    fi
+  fi
   if [ -z "$VERSITY_EXE" ]; then
     log 1 "VERSITY_EXE missing"
     exit 1

tests/integration/group-tests.go

@@ -134,6 +134,9 @@ func TestPutObject(s *S3Conf) {
 	PutObject_missing_object_lock_retention_config(s)
 	PutObject_with_object_lock(s)
 	PutObject_success(s)
+	if !s.versioningEnabled {
+		PutObject_racey_success(s)
+	}
 	PutObject_invalid_credentials(s)
 }
 
@@ -142,6 +145,7 @@ func TestHeadObject(s *S3Conf) {
 	HeadObject_invalid_part_number(s)
 	HeadObject_non_existing_mp(s)
 	HeadObject_mp_success(s)
+	HeadObject_directory_object_noslash(s)
 	HeadObject_non_existing_dir_object(s)
 	HeadObject_with_contenttype(s)
 	HeadObject_success(s)
@@ -155,6 +159,7 @@ func TestGetObjectAttributes(s *S3Conf) {
 
 func TestGetObject(s *S3Conf) {
 	GetObject_non_existing_key(s)
+	GetObject_directory_object_noslash(s)
 	GetObject_invalid_ranges(s)
 	GetObject_with_meta(s)
 	GetObject_success(s)
@@ -168,11 +173,13 @@ func TestListObjects(s *S3Conf) {
 	ListObjects_non_existing_bucket(s)
 	ListObjects_with_prefix(s)
 	ListObjects_truncated(s)
+	ListObjects_paginated(s)
 	ListObjects_invalid_max_keys(s)
 	ListObjects_max_keys_0(s)
 	ListObjects_delimiter(s)
 	ListObjects_max_keys_none(s)
 	ListObjects_marker_not_from_obj_list(s)
+	ListObjects_list_all_objs(s)
 }
 
 func TestListObjectsV2(s *S3Conf) {
@@ -184,10 +191,17 @@ func TestListObjectsV2(s *S3Conf) {
 	ListObjectsV2_single_dir_object_with_delim_and_prefix(s)
 	ListObjectsV2_truncated_common_prefixes(s)
 	ListObjectsV2_all_objs_max_keys(s)
+	ListObjectsV2_list_all_objs(s)
+}
+
+// VD stands for Versioning Disabled
+func TestListObjectVersions_VD(s *S3Conf) {
+	ListObjectVersions_VD_success(s)
 }
 
 func TestDeleteObject(s *S3Conf) {
 	DeleteObject_non_existing_object(s)
+	DeleteObject_directory_object_noslash(s)
 	DeleteObject_non_existing_dir_object(s)
 	DeleteObject_success(s)
 	DeleteObject_success_status_code(s)
@@ -266,6 +280,8 @@ func TestUploadPartCopy(s *S3Conf) {
 func TestListParts(s *S3Conf) {
 	ListParts_incorrect_uploadId(s)
 	ListParts_incorrect_object_key(s)
+	ListParts_invalid_max_parts(s)
+	ListParts_default_max_parts(s)
 	ListParts_truncated(s)
 	ListParts_success(s)
 }
@@ -293,6 +309,9 @@ func TestCompleteMultipartUpload(s *S3Conf) {
 	CompleteMultipartUpload_invalid_part_number(s)
 	CompleteMultipartUpload_invalid_ETag(s)
 	CompleteMultipartUpload_success(s)
+	if !s.azureTests {
+		CompleteMultipartUpload_racey_success(s)
+	}
 }
 
 func TestPutBucketAcl(s *S3Conf) {
@@ -390,6 +409,7 @@ func TestPutObjectRetention(s *S3Conf) {
 func TestGetObjectRetention(s *S3Conf) {
 	GetObjectRetention_non_existing_bucket(s)
 	GetObjectRetention_non_existing_object(s)
+	GetObjectRetention_disabled_lock(s)
 	GetObjectRetention_unset_config(s)
 	GetObjectRetention_success(s)
 }
@@ -407,6 +427,7 @@ func TestPutObjectLegalHold(s *S3Conf) {
 func TestGetObjectLegalHold(s *S3Conf) {
 	GetObjectLegalHold_non_existing_bucket(s)
 	GetObjectLegalHold_non_existing_object(s)
+	GetObjectLegalHold_disabled_lock(s)
 	GetObjectLegalHold_unset_config(s)
 	GetObjectLegalHold_success(s)
 }
@@ -414,7 +435,8 @@ func TestGetObjectLegalHold(s *S3Conf) {
 func TestWORMProtection(s *S3Conf) {
 	WORMProtection_bucket_object_lock_configuration_compliance_mode(s)
 	WORMProtection_bucket_object_lock_configuration_governance_mode(s)
-	WORMProtection_bucket_object_lock_governance_bypass_delete(s)
+	// WORMProtection_bucket_object_lock_governance_bypass_delete(s)
+	// WORMProtection_bucket_object_lock_governance_bypass_delete_multiple
 	WORMProtection_object_lock_retention_compliance_locked(s)
 	WORMProtection_object_lock_retention_governance_locked(s)
 	WORMProtection_object_lock_retention_governance_bypass_overwrite(s)
@@ -443,6 +465,9 @@ func TestFullFlow(s *S3Conf) {
 	TestGetObject(s)
 	TestListObjects(s)
 	TestListObjectsV2(s)
+	if !s.versioningEnabled && !s.azureTests {
+		TestListObjectVersions_VD(s)
+	}
 	TestDeleteObject(s)
 	TestDeleteObjects(s)
 	TestCopyObject(s)
@@ -450,7 +475,9 @@ func TestFullFlow(s *S3Conf) {
 	TestDeleteObjectTagging(s)
 	TestCreateMultipartUpload(s)
 	TestUploadPart(s)
-	TestUploadPartCopy(s)
+	if !s.azureTests {
+		TestUploadPartCopy(s)
+	}
 	TestListParts(s)
 	TestListMultipartUploads(s)
 	TestAbortMultipartUpload(s)
@@ -476,11 +503,16 @@ func TestFullFlow(s *S3Conf) {
 func TestPosix(s *S3Conf) {
 	PutObject_overwrite_dir_obj(s)
 	PutObject_overwrite_file_obj(s)
+	PutObject_overwrite_file_obj_with_nested_obj(s)
 	PutObject_dir_obj_with_data(s)
 	CreateMultipartUpload_dir_obj(s)
 	PutObject_name_too_long(s)
 	HeadObject_name_too_long(s)
 	DeleteObject_name_too_long(s)
+	// posix specific versioning tests
+	if !s.versioningEnabled {
+		TestVersioningDisabled(s)
+	}
 }
 
 func TestIAM(s *S3Conf) {
@@ -509,15 +541,24 @@ func TestVersioning(s *S3Conf) {
 	// PutBucketVersioning action
 	PutBucketVersioning_non_existing_bucket(s)
 	PutBucketVersioning_invalid_status(s)
-	PutBucketVersioning_success(s)
+	PutBucketVersioning_success_enabled(s)
+	PutBucketVersioning_success_suspended(s)
 	// GetBucketVersioning action
 	GetBucketVersioning_non_existing_bucket(s)
+	GetBucketVersioning_empty_response(s)
 	GetBucketVersioning_success(s)
+	// DeleteBucket action
+	Versioning_DeleteBucket_not_empty(s)
+	// PutObject action
+	Versioning_PutObject_suspended_null_versionId_obj(s)
+	Versioning_PutObject_null_versionId_obj(s)
+	Versioning_PutObject_overwrite_null_versionId_obj(s)
 	Versioning_PutObject_success(s)
 	// CopyObject action
 	Versioning_CopyObject_success(s)
 	Versioning_CopyObject_non_existing_version_id(s)
 	Versioning_CopyObject_from_an_object_version(s)
+	Versioning_CopyObject_special_chars(s)
 	// HeadObject action
 	Versioning_HeadObject_invalid_versionId(s)
 	Versioning_HeadObject_success(s)
@@ -525,10 +566,15 @@ func TestVersioning(s *S3Conf) {
 	// GetObject action
 	Versioning_GetObject_invalid_versionId(s)
 	Versioning_GetObject_success(s)
+	Versioning_GetObject_delete_marker_without_versionId(s)
 	Versioning_GetObject_delete_marker(s)
+	Versioning_GetObject_null_versionId_obj(s)
 	// DeleteObject(s) actions
 	Versioning_DeleteObject_delete_object_version(s)
+	Versioning_DeleteObject_non_existing_object(s)
 	Versioning_DeleteObject_delete_a_delete_marker(s)
+	Versioning_Delete_null_versionId_object(s)
+	Versioning_DeleteObject_suspended(s)
 	Versioning_DeleteObjects_success(s)
 	Versioning_DeleteObjects_delete_deleteMarkers(s)
 	// ListObjectVersions
@@ -537,11 +583,35 @@ func TestVersioning(s *S3Conf) {
 	ListObjectVersions_list_multiple_object_versions(s)
 	ListObjectVersions_multiple_object_versions_truncated(s)
 	ListObjectVersions_with_delete_markers(s)
+	ListObjectVersions_containing_null_versionId_obj(s)
+	ListObjectVersions_single_null_versionId_object(s)
 	// Multipart upload
 	Versioning_Multipart_Upload_success(s)
 	Versioning_Multipart_Upload_overwrite_an_object(s)
 	Versioning_UploadPartCopy_non_existing_versionId(s)
 	Versioning_UploadPartCopy_from_an_object_version(s)
+	// Object lock configuration
+	Versioning_Enable_object_lock(s)
+	Versioning_status_switch_to_suspended_with_object_lock(s)
+	// Object-Lock Retention
+	Versioning_PutObjectRetention_invalid_versionId(s)
+	Versioning_GetObjectRetention_invalid_versionId(s)
+	Versioning_Put_GetObjectRetention_success(s)
+	// Object-Lock Legal hold
+	Versioning_PutObjectLegalHold_invalid_versionId(s)
+	Versioning_GetObjectLegalHold_invalid_versionId(s)
+	Versioning_Put_GetObjectLegalHold_success(s)
+	// WORM protection
+	Versioning_WORM_obj_version_locked_with_legal_hold(s)
+	Versioning_WORM_obj_version_locked_with_governance_retention(s)
+	Versioning_WORM_obj_version_locked_with_compliance_retention(s)
+	// Concurrent requests
+	//Versioninig_concurrent_upload_object(s)
+}
+
+func TestVersioningDisabled(s *S3Conf) {
+	VersioningDisabled_GetBucketVersioning_not_configured(s)
+	VersioningDisabled_PutBucketVersioning_not_configured(s)
 }
 
 type IntTests map[string]func(s *S3Conf) error
@@ -632,10 +702,12 @@ func GetIntTests() IntTests {
 		"PutObject_special_chars":                                             PutObject_special_chars,
 		"PutObject_invalid_long_tags":                                         PutObject_invalid_long_tags,
 		"PutObject_success":                                                   PutObject_success,
+		"PutObject_racey_success":                                             PutObject_racey_success,
 		"HeadObject_non_existing_object":                                      HeadObject_non_existing_object,
 		"HeadObject_invalid_part_number":                                      HeadObject_invalid_part_number,
 		"HeadObject_non_existing_mp":                                          HeadObject_non_existing_mp,
 		"HeadObject_mp_success":                                               HeadObject_mp_success,
+		"HeadObject_directory_object_noslash":                                 HeadObject_directory_object_noslash,
 		"HeadObject_non_existing_dir_object":                                  HeadObject_non_existing_dir_object,
 		"HeadObject_name_too_long":                                            HeadObject_name_too_long,
 		"HeadObject_with_contenttype":                                         HeadObject_with_contenttype,
@@ -644,6 +716,7 @@ func GetIntTests() IntTests {
 		"GetObjectAttributes_non_existing_object":                             GetObjectAttributes_non_existing_object,
 		"GetObjectAttributes_existing_object":                                 GetObjectAttributes_existing_object,
 		"GetObject_non_existing_key":                                          GetObject_non_existing_key,
+		"GetObject_directory_object_noslash":                                  GetObject_directory_object_noslash,
 		"GetObject_invalid_ranges":                                            GetObject_invalid_ranges,
 		"GetObject_with_meta":                                                 GetObject_with_meta,
 		"GetObject_success":                                                   GetObject_success,
@@ -654,11 +727,13 @@ func GetIntTests() IntTests {
 		"ListObjects_non_existing_bucket":                                     ListObjects_non_existing_bucket,
 		"ListObjects_with_prefix":                                             ListObjects_with_prefix,
 		"ListObjects_truncated":                                               ListObjects_truncated,
+		"ListObjects_paginated":                                               ListObjects_paginated,
 		"ListObjects_invalid_max_keys":                                        ListObjects_invalid_max_keys,
 		"ListObjects_max_keys_0":                                              ListObjects_max_keys_0,
 		"ListObjects_delimiter":                                               ListObjects_delimiter,
 		"ListObjects_max_keys_none":                                           ListObjects_max_keys_none,
 		"ListObjects_marker_not_from_obj_list":                                ListObjects_marker_not_from_obj_list,
+		"ListObjects_list_all_objs":                                           ListObjects_list_all_objs,
 		"ListObjectsV2_start_after":                                           ListObjectsV2_start_after,
 		"ListObjectsV2_both_start_after_and_continuation_token":               ListObjectsV2_both_start_after_and_continuation_token,
 		"ListObjectsV2_start_after_not_in_list":                               ListObjectsV2_start_after_not_in_list,
@@ -667,7 +742,10 @@ func GetIntTests() IntTests {
 		"ListObjectsV2_single_dir_object_with_delim_and_prefix":               ListObjectsV2_single_dir_object_with_delim_and_prefix,
 		"ListObjectsV2_truncated_common_prefixes":                             ListObjectsV2_truncated_common_prefixes,
 		"ListObjectsV2_all_objs_max_keys":                                     ListObjectsV2_all_objs_max_keys,
+		"ListObjectsV2_list_all_objs":                                         ListObjectsV2_list_all_objs,
+		"ListObjectVersions_VD_success":                                       ListObjectVersions_VD_success,
 		"DeleteObject_non_existing_object":                                    DeleteObject_non_existing_object,
+		"DeleteObject_directory_object_noslash":                               DeleteObject_directory_object_noslash,
 		"DeleteObject_name_too_long":                                          DeleteObject_name_too_long,
 		"DeleteObject_non_existing_dir_object":                                DeleteObject_non_existing_dir_object,
 		"DeleteObject_success":                                                DeleteObject_success,
@@ -720,6 +798,8 @@ func GetIntTests() IntTests {
 		"UploadPartCopy_by_range_success":                                     UploadPartCopy_by_range_success,
 		"ListParts_incorrect_uploadId":                                        ListParts_incorrect_uploadId,
 		"ListParts_incorrect_object_key":                                      ListParts_incorrect_object_key,
+		"ListParts_invalid_max_parts":                                         ListParts_invalid_max_parts,
+		"ListParts_default_max_parts":                                         ListParts_default_max_parts,
 		"ListParts_truncated":                                                 ListParts_truncated,
 		"ListParts_success":                                                   ListParts_success,
 		"ListMultipartUploads_non_existing_bucket":                            ListMultipartUploads_non_existing_bucket,
@@ -738,6 +818,7 @@ func GetIntTests() IntTests {
 		"CompleteMultipartUpload_invalid_part_number":                         CompleteMultipartUpload_invalid_part_number,
 		"CompleteMultipartUpload_invalid_ETag":                                CompleteMultipartUpload_invalid_ETag,
 		"CompleteMultipartUpload_success":                                     CompleteMultipartUpload_success,
+		"CompleteMultipartUpload_racey_success":                               CompleteMultipartUpload_racey_success,
 		"PutBucketAcl_non_existing_bucket":                                    PutBucketAcl_non_existing_bucket,
 		"PutBucketAcl_disabled":                                               PutBucketAcl_disabled,
 		"PutBucketAcl_none_of_the_options_specified":                          PutBucketAcl_none_of_the_options_specified,
@@ -808,6 +889,7 @@ func GetIntTests() IntTests {
 		"PutObjectRetention_success":                                          PutObjectRetention_success,
 		"GetObjectRetention_non_existing_bucket":                              GetObjectRetention_non_existing_bucket,
 		"GetObjectRetention_non_existing_object":                              GetObjectRetention_non_existing_object,
+		"GetObjectRetention_disabled_lock":                                    GetObjectRetention_disabled_lock,
 		"GetObjectRetention_unset_config":                                     GetObjectRetention_unset_config,
 		"GetObjectRetention_success":                                          GetObjectRetention_success,
 		"PutObjectLegalHold_non_existing_bucket":                              PutObjectLegalHold_non_existing_bucket,
@@ -819,6 +901,7 @@ func GetIntTests() IntTests {
 		"PutObjectLegalHold_success":                                          PutObjectLegalHold_success,
 		"GetObjectLegalHold_non_existing_bucket":                              GetObjectLegalHold_non_existing_bucket,
 		"GetObjectLegalHold_non_existing_object":                              GetObjectLegalHold_non_existing_object,
+		"GetObjectLegalHold_disabled_lock":                                    GetObjectLegalHold_disabled_lock,
 		"GetObjectLegalHold_unset_config":                                     GetObjectLegalHold_unset_config,
 		"GetObjectLegalHold_success":                                          GetObjectLegalHold_success,
 		"WORMProtection_bucket_object_lock_configuration_compliance_mode":     WORMProtection_bucket_object_lock_configuration_compliance_mode,
@@ -834,6 +917,7 @@ func GetIntTests() IntTests {
 		"WORMProtection_root_bypass_governance_retention_delete_object":       WORMProtection_root_bypass_governance_retention_delete_object,
 		"PutObject_overwrite_dir_obj":                                         PutObject_overwrite_dir_obj,
 		"PutObject_overwrite_file_obj":                                        PutObject_overwrite_file_obj,
+		"PutObject_overwrite_file_obj_with_nested_obj":                        PutObject_overwrite_file_obj_with_nested_obj,
 		"PutObject_dir_obj_with_data":                                         PutObject_dir_obj_with_data,
 		"CreateMultipartUpload_dir_obj":                                       CreateMultipartUpload_dir_obj,
 		"IAM_user_access_denied":                                              IAM_user_access_denied,
@@ -854,21 +938,33 @@ func GetIntTests() IntTests {
 		"AccessControl_copy_object_with_starting_slash_for_user":              AccessControl_copy_object_with_starting_slash_for_user,
 		"PutBucketVersioning_non_existing_bucket":                             PutBucketVersioning_non_existing_bucket,
 		"PutBucketVersioning_invalid_status":                                  PutBucketVersioning_invalid_status,
-		"PutBucketVersioning_success":                                         PutBucketVersioning_success,
+		"PutBucketVersioning_success_enabled":                                 PutBucketVersioning_success_enabled,
+		"PutBucketVersioning_success_suspended":                               PutBucketVersioning_success_suspended,
 		"GetBucketVersioning_non_existing_bucket":                             GetBucketVersioning_non_existing_bucket,
+		"GetBucketVersioning_empty_response":                                  GetBucketVersioning_empty_response,
 		"GetBucketVersioning_success":                                         GetBucketVersioning_success,
+		"Versioning_DeleteBucket_not_empty":                                   Versioning_DeleteBucket_not_empty,
+		"Versioning_PutObject_suspended_null_versionId_obj":                   Versioning_PutObject_suspended_null_versionId_obj,
+		"Versioning_PutObject_null_versionId_obj":                             Versioning_PutObject_null_versionId_obj,
+		"Versioning_PutObject_overwrite_null_versionId_obj":                   Versioning_PutObject_overwrite_null_versionId_obj,
 		"Versioning_PutObject_success":                                        Versioning_PutObject_success,
 		"Versioning_CopyObject_success":                                       Versioning_CopyObject_success,
 		"Versioning_CopyObject_non_existing_version_id":                       Versioning_CopyObject_non_existing_version_id,
 		"Versioning_CopyObject_from_an_object_version":                        Versioning_CopyObject_from_an_object_version,
+		"Versioning_CopyObject_special_chars":                                 Versioning_CopyObject_special_chars,
 		"Versioning_HeadObject_invalid_versionId":                             Versioning_HeadObject_invalid_versionId,
 		"Versioning_HeadObject_success":                                       Versioning_HeadObject_success,
 		"Versioning_HeadObject_delete_marker":                                 Versioning_HeadObject_delete_marker,
 		"Versioning_GetObject_invalid_versionId":                              Versioning_GetObject_invalid_versionId,
 		"Versioning_GetObject_success":                                        Versioning_GetObject_success,
+		"Versioning_GetObject_delete_marker_without_versionId":                Versioning_GetObject_delete_marker_without_versionId,
 		"Versioning_GetObject_delete_marker":                                  Versioning_GetObject_delete_marker,
+		"Versioning_GetObject_null_versionId_obj":                             Versioning_GetObject_null_versionId_obj,
 		"Versioning_DeleteObject_delete_object_version":                       Versioning_DeleteObject_delete_object_version,
+		"Versioning_DeleteObject_non_existing_object":                         Versioning_DeleteObject_non_existing_object,
 		"Versioning_DeleteObject_delete_a_delete_marker":                      Versioning_DeleteObject_delete_a_delete_marker,
+		"Versioning_Delete_null_versionId_object":                             Versioning_Delete_null_versionId_object,
+		"Versioning_DeleteObject_suspended":                                   Versioning_DeleteObject_suspended,
 		"Versioning_DeleteObjects_success":                                    Versioning_DeleteObjects_success,
 		"Versioning_DeleteObjects_delete_deleteMarkers":                       Versioning_DeleteObjects_delete_deleteMarkers,
 		"ListObjectVersions_non_existing_bucket":                              ListObjectVersions_non_existing_bucket,
@@ -876,9 +972,23 @@ func GetIntTests() IntTests {
 		"ListObjectVersions_list_multiple_object_versions":                    ListObjectVersions_list_multiple_object_versions,
 		"ListObjectVersions_multiple_object_versions_truncated":               ListObjectVersions_multiple_object_versions_truncated,
 		"ListObjectVersions_with_delete_markers":                              ListObjectVersions_with_delete_markers,
+		"ListObjectVersions_containing_null_versionId_obj":                    ListObjectVersions_containing_null_versionId_obj,
+		"ListObjectVersions_single_null_versionId_object":                     ListObjectVersions_single_null_versionId_object,
 		"Versioning_Multipart_Upload_success":                                 Versioning_Multipart_Upload_success,
 		"Versioning_Multipart_Upload_overwrite_an_object":                     Versioning_Multipart_Upload_overwrite_an_object,
 		"Versioning_UploadPartCopy_non_existing_versionId":                    Versioning_UploadPartCopy_non_existing_versionId,
 		"Versioning_UploadPartCopy_from_an_object_version":                    Versioning_UploadPartCopy_from_an_object_version,
+		"Versioning_Enable_object_lock":                                       Versioning_Enable_object_lock,
+		"Versioning_status_switch_to_suspended_with_object_lock":              Versioning_status_switch_to_suspended_with_object_lock,
+		"Versioning_PutObjectRetention_invalid_versionId":                     Versioning_PutObjectRetention_invalid_versionId,
+		"Versioning_GetObjectRetention_invalid_versionId":                     Versioning_GetObjectRetention_invalid_versionId,
+		"Versioning_Put_GetObjectRetention_success":                           Versioning_Put_GetObjectRetention_success,
+		"Versioning_PutObjectLegalHold_invalid_versionId":                     Versioning_PutObjectLegalHold_invalid_versionId,
+		"Versioning_GetObjectLegalHold_invalid_versionId":                     Versioning_GetObjectLegalHold_invalid_versionId,
+		"Versioning_Put_GetObjectLegalHold_success":                           Versioning_Put_GetObjectLegalHold_success,
+		"Versioning_WORM_obj_version_locked_with_legal_hold":                  Versioning_WORM_obj_version_locked_with_legal_hold,
+		"Versioning_WORM_obj_version_locked_with_governance_retention":        Versioning_WORM_obj_version_locked_with_governance_retention,
+		"Versioning_WORM_obj_version_locked_with_compliance_retention":        Versioning_WORM_obj_version_locked_with_compliance_retention,
+		"Versioning_concurrent_upload_object":                                 Versioning_concurrent_upload_object,
 	}
 }