feat: Fixes all struct fields alignment issues, Adds betteralign check in the pipeline, to check the structs fields alignment

Test/GitHub actions speedup

.github/workflows/azurite.yml

@@ -0,0 +1,37 @@
+name: azurite functional tests
+
+on: pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+              
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Set up Docker Compose
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
+
+      - name: Wait for Azurite to be ready
+        run: sleep 40
+
+      - name: Get Dependencies
+        run: |
+          go mod download
+  
+      - name: Build and Run
+        run: |
+          make
+          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
+  
+      - name: Shut down services
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/betteralign.yml

@@ -0,0 +1,23 @@
+name: betteralign
+on: pull_request
+jobs:
+  build:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "stable"
+        id: go
+
+      - name: Install betteralign
+        run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest
+
+      - name: Run betteralign
+        run: betteralign -test_files ./...

.github/workflows/docker-bats.yaml

@@ -1,28 +0,0 @@
-name: docker bats tests
-
-on: pull_request
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build Docker Image
-        run: |
-          mv tests/.env.docker.default tests/.env.docker
-          mv tests/.secrets.default tests/.secrets
-          docker build --build-arg="GO_LIBRARY=go1.21.7.linux-amd64.tar.gz" \
-            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" --build-arg="MC_FOLDER=linux-amd64" \
-            --progress=plain -f Dockerfile_test_bats -t bats_test .
-
-      - name: Set up Docker Compose
-        run: sudo apt-get install -y docker-compose
-
-      - name: Run Docker Container
-        run: docker-compose -f docker-compose-bats.yml up posix_backend

.github/workflows/docker-bats.yml

@@ -0,0 +1,28 @@
+name: docker bats tests
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        run: |
+          cp tests/.env.docker.default tests/.env.docker
+          cp tests/.secrets.default tests/.secrets
+          docker build \
+            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
+            --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain \
+            -f tests/Dockerfile_test_bats \
+            -t bats_test .
+
+      - name: Run Docker Container
+        run: |
+          docker compose -f tests/docker-compose-bats.yml --project-directory . \
+          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/functional.yml

@@ -1,7 +1,8 @@
 name: functional tests
-on: pull_request
-jobs:
 
+on: pull_request
+
+jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
@@ -18,7 +19,7 @@ jobs:
 
     - name: Get Dependencies
       run: |
-        go get -v -t -d ./...
+        go mod download
 
     - name: Build and Run
       run: |

.github/workflows/system.yml

@@ -8,83 +8,96 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - set: 1
-            LOCAL_FOLDER: /tmp/gw1
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+          - set: "mc, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam1
-            AWS_ENDPOINT_URL: https://127.0.0.1:7070
-            RUN_SET: "s3cmd"
+            RUN_SET: "mc-non-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7070
             BACKEND: "posix"
-          - set: 2
-            LOCAL_FOLDER: /tmp/gw2
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
+          - set: "mc, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam2
-            AWS_ENDPOINT_URL: https://127.0.0.1:7071
-            RUN_SET: "s3"
+            RUN_SET: "mc-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7071
             BACKEND: "posix"
-          - set: 3
-            LOCAL_FOLDER: /tmp/gw3
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
+          - set: "REST, posix, non-static, all, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam3
-            AWS_ENDPOINT_URL: https://127.0.0.1:7072
-            RUN_SET: "s3api"
+            RUN_SET: "rest"
             RECREATE_BUCKETS: "true"
-            PORT: 7072
             BACKEND: "posix"
-          - set: 4
-            LOCAL_FOLDER: /tmp/gw4
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+          - set: "s3, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam4
-            AWS_ENDPOINT_URL: https://127.0.0.1:7073
-            RUN_SET: "mc"
+            RUN_SET: "s3-non-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7073
             BACKEND: "posix"
-          - set: 5
-            LOCAL_FOLDER: /tmp/gw5
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
+          - set: "s3, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, non-static, s3 IAM"
             IAM_TYPE: s3
-            USERS_BUCKET: versity-gwtest-iam
-            AWS_ENDPOINT_URL: https://127.0.0.1:7074
-            RUN_SET: "aws-user"
+            RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "true"
-            PORT: 7074
             BACKEND: "posix"
-          - set: 6
-            LOCAL_FOLDER: /tmp/gw6
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+          - set: "s3api, posix, bucket, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam6
-            AWS_ENDPOINT_URL: https://127.0.0.1:7075
-            RUN_SET: "aws"
+            RUN_SET: "s3api-bucket"
             RECREATE_BUCKETS: "false"
-            PORT: 7075
             BACKEND: "posix"
-          - set: 7
-            LOCAL_FOLDER: /tmp/gw7
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+          - set: "s3api, posix, multipart, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam7
-            AWS_ENDPOINT_URL: https://127.0.0.1:7076
-            RUN_SET: "aws"
+            RUN_SET: "s3api-multipart"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, object, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-object"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, s3, multipart|object, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
             RECREATE_BUCKETS: "true"
-            PORT: 7076
             BACKEND: "s3"
+          - set: "s3api, s3, policy|user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy,s3api-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "s3"
+          - set: "s3cmd, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, non-user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-non-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -115,17 +128,14 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Build and run, posix backend
+      - name: Install xmllint (for rest)
+        run: |
+          sudo apt-get install libxml2-utils
+
+      - name: Build and run
         env:
-          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
-          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
-          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
-          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
-          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
           IAM_TYPE: ${{ matrix.IAM_TYPE }}
-          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
           RUN_SET: ${{ matrix.RUN_SET }}
-          PORT: ${{ matrix.PORT }}
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
@@ -133,6 +143,13 @@ jobs:
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
+          LOCAL_FOLDER: /tmp/gw
+          BUCKET_ONE_NAME: versity-gwtest-bucket-one
+          BUCKET_TWO_NAME: versity-gwtest-bucket-two
+          USERS_FOLDER: /tmp/iam
+          USERS_BUCKET: versity-gwtest-iam
+          AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          PORT: 7070
           S3CMD_CONFIG: tests/s3cfg.local.default
           MC_ALIAS: versity
           LOG_LEVEL: 4
@@ -142,6 +159,10 @@ jobs:
           USERNAME_TWO: HIJKLMN
           PASSWORD_TWO: 8901234
           TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
+          REMOVE_TEST_FILE_FOLDER: true
+          VERSIONING_DIR: ${{ github.workspace }}/versioning
+          COMMAND_LOG: command.log
+          TIME_LOG: time.log
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -162,6 +183,9 @@ jobs:
           fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
+      - name: Time report
+        run: cat ${{ github.workspace }}/time.log
+
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

.gitignore

@@ -45,6 +45,7 @@ tests/.secrets*
 
 # IAM users files often created in testing
 users.json
+users.json.backup
 
 # env files for testing
 **/.env*
@@ -61,4 +62,8 @@ tests/!s3cfg.local.default
 *.patch
 
 # grafana's local database (kept on filesystem for survival between instantiations)
-metrics-exploration/grafana_data/**
+metrics-exploration/grafana_data/**
+
+# bats tools
+/tests/bats-assert
+/tests/bats-support

Dockerfile

@@ -32,4 +32,4 @@ RUN mkdir -p $SETUP_DIR
 
 COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
 
-ENTRYPOINT [ "/app/versitygw" ]
+ENTRYPOINT [ "/app/versitygw" ]

Makefile

@@ -18,6 +18,10 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 
+# docker-compose
+DCCMD=docker-compose
+DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
+
 BIN=versitygw
 
 VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -71,19 +75,19 @@ dist:
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
-	docker compose --env-file .env.dev up posix
+	$(DOCKERCOMPOSE) up posix
 
 # Creates and runs S3 gateway proxy instance in a docker container
 .PHONY: up-proxy
 up-proxy:
-	docker compose --env-file .env.dev up proxy
+	$(DOCKERCOMPOSE) up proxy
 
 # Creates and runs S3 gateway to azurite instance in a docker container
 .PHONY: up-azurite
 up-azurite:
-	docker compose --env-file .env.dev up azurite azuritegw
+	$(DOCKERCOMPOSE) up azurite azuritegw
 
 # Creates and runs both S3 gateway and proxy server instances in docker containers
 .PHONY: up-app
 up-app:
-	docker compose --env-file .env.dev up
+	$(DOCKERCOMPOSE) up

auth/acl.go

@@ -45,18 +45,18 @@ type GetBucketAclOutput struct {
 
 type PutBucketAclInput struct {
 	Bucket              *string
-	ACL                 types.BucketCannedACL
 	AccessControlPolicy *AccessControlPolicy
 	GrantFullControl    *string
 	GrantRead           *string
 	GrantReadACP        *string
 	GrantWrite          *string
 	GrantWriteACP       *string
+	ACL                 types.BucketCannedACL
 }
 
 type AccessControlPolicy struct {
-	AccessControlList AccessControlList `xml:"AccessControlList"`
 	Owner             *types.Owner
+	AccessControlList AccessControlList `xml:"AccessControlList"`
 }
 
 type AccessControlList struct {
@@ -352,13 +352,13 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 }
 
 type AccessOptions struct {
-	Acl           ACL
 	AclPermission types.Permission
-	IsRoot        bool
-	Acc           Account
 	Bucket        string
 	Object        string
 	Action        Action
+	Acl           ACL
+	Acc           Account
+	IsRoot        bool
 	Readonly      bool
 }
 

auth/bucket_policy.go

@@ -63,10 +63,10 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri
 }
 
 type BucketPolicyItem struct {
-	Effect     BucketPolicyAccessType `json:"Effect"`
 	Principals Principals             `json:"Principal"`
 	Actions    Actions                `json:"Action"`
 	Resources  Resources              `json:"Resource"`
+	Effect     BucketPolicyAccessType `json:"Effect"`
 }
 
 func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {

auth/bucket_policy_actions.go

@@ -36,6 +36,7 @@ const (
 	ListBucketMultipartUploadsAction       Action = "s3:ListBucketMultipartUploads"
 	PutObjectAction                        Action = "s3:PutObject"
 	GetObjectAction                        Action = "s3:GetObject"
+	GetObjectVersionAction                 Action = "s3:GetObjectVersion"
 	DeleteObjectAction                     Action = "s3:DeleteObject"
 	GetObjectAclAction                     Action = "s3:GetObjectAcl"
 	GetObjectAttributesAction              Action = "s3:GetObjectAttributes"
@@ -75,6 +76,7 @@ var supportedActionList = map[Action]struct{}{
 	ListBucketMultipartUploadsAction:       {},
 	PutObjectAction:                        {},
 	GetObjectAction:                        {},
+	GetObjectVersionAction:                 {},
 	DeleteObjectAction:                     {},
 	GetObjectAclAction:                     {},
 	GetObjectAttributesAction:              {},
@@ -103,6 +105,7 @@ var supportedObjectActionList = map[Action]struct{}{
 	ListMultipartUploadPartsAction:  {},
 	PutObjectAction:                 {},
 	GetObjectAction:                 {},
+	GetObjectVersionAction:          {},
 	DeleteObjectAction:              {},
 	GetObjectAclAction:              {},
 	GetObjectAttributesAction:       {},

auth/iam.go

@@ -28,6 +28,19 @@ const (
 	RoleUserPlus Role = "userplus"
 )
 
+func (r Role) IsValid() bool {
+	switch r {
+	case RoleAdmin:
+		return true
+	case RoleUser:
+		return true
+	case RoleUserPlus:
+		return true
+	default:
+		return false
+	}
+}
+
 // Account is a gateway IAM account
 type Account struct {
 	Access  string `json:"access"`
@@ -37,6 +50,10 @@ type Account struct {
 	GroupID int    `json:"groupID"`
 }
 
+type ListUserAccountsResult struct {
+	Accounts []Account
+}
+
 // Mutable props, which could be changed when updating an IAM account
 type MutableProps struct {
 	Secret  *string `json:"secret"`
@@ -76,7 +93,6 @@ var (
 )
 
 type Opts struct {
-	RootAccount            Account
 	Dir                    string
 	LDAPServerURL          string
 	LDAPBindDN             string
@@ -102,11 +118,12 @@ type Opts struct {
 	S3Region               string
 	S3Bucket               string
 	S3Endpoint             string
+	RootAccount            Account
+	CacheTTL               int
+	CachePrune             int
 	S3DisableSSlVerfiy     bool
 	S3Debug                bool
 	CacheDisable           bool
-	CacheTTL               int
-	CachePrune             int
 }
 
 func New(o *Opts) (IAMService, error) {

auth/iam_cache.go

@@ -36,14 +36,14 @@ type IAMCache struct {
 var _ IAMService = &IAMCache{}
 
 type item struct {
-	value Account
 	exp   time.Time
+	value Account
 }
 
 type icache struct {
-	sync.RWMutex
-	expire time.Duration
 	items  map[string]item
+	expire time.Duration
+	sync.RWMutex
 }
 
 func (i *icache) set(k string, v Account) {

auth/iam_internal.go

@@ -33,6 +33,8 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
+	dir     string
+	rootAcc Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -40,8 +42,6 @@ type IAMServiceInternal struct {
 	// IAM service. All account updates should be sent to a single
 	// gateway instance if possible.
 	sync.RWMutex
-	dir     string
-	rootAcc Account
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored

auth/iam_s3_object.go

@@ -42,6 +42,14 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
+	client *s3.Client
+
+	access   string
+	secret   string
+	region   string
+	bucket   string
+	endpoint string
+	rootAcc  Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -50,15 +58,8 @@ type IAMServiceS3 struct {
 	// gateway instance if possible.
 	sync.RWMutex
 
-	access        string
-	secret        string
-	region        string
-	bucket        string
-	endpoint      string
 	sslSkipVerify bool
 	debug         bool
-	rootAcc       Account
-	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}

auth/object_lock.go

@@ -25,12 +25,13 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
 type BucketLockConfig struct {
-	Enabled          bool
 	DefaultRetention *types.DefaultRetention
 	CreatedAt        *time.Time
+	Enabled          bool
 }
 
 func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
@@ -92,12 +93,12 @@ func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfigur
 }
 
 func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
-	var retention types.ObjectLockRetention
+	var retention s3response.PutObjectRetentionInput
 	if err := xml.Unmarshal(input, &retention); err != nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
-	if retention.RetainUntilDate == nil || retention.RetainUntilDate.Before(time.Now()) {
+	if retention.RetainUntilDate.Before(time.Now()) {
 		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
 	}
 	switch retention.Mode {
@@ -135,7 +136,7 @@ func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
 	}
 }
 
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, bypass bool, be backend.Backend) error {
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
 	data, err := be.GetObjectLockConfiguration(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -171,8 +172,15 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 	}
 
 	for _, obj := range objects {
+		var key, versionId string
+		if obj.Key != nil {
+			key = *obj.Key
+		}
+		if obj.VersionId != nil {
+			versionId = *obj.VersionId
+		}
 		checkRetention := true
-		retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
+		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
 			continue
 		}
@@ -203,7 +211,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 							if err != nil {
 								return err
 							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
+							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
 							if err != nil {
 								return s3err.GetAPIError(s3err.ErrObjectLocked)
 							}
@@ -217,8 +225,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 
 		checkLegalHold := true
 
-		status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
+		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
 		if err != nil {
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+				continue
+			}
 			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
 				checkLegalHold = false
 			} else {
@@ -243,7 +254,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 					if err != nil {
 						return err
 					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
+					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
 					if err != nil {
 						return s3err.GetAPIError(s3err.ErrObjectLocked)
 					}

aws/signer/v4/v4.go

@@ -85,6 +85,10 @@ type keyDerivator interface {
 
 // SignerOptions is the SigV4 Signer options.
 type SignerOptions struct {
+
+	// The logger to send log messages to.
+	Logger logging.Logger
+
 	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
 	// request header to the request's query string. This is most commonly used
 	// with pre-signed requests preventing headers from being added to the
@@ -100,9 +104,6 @@ type SignerOptions struct {
 	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
 	DisableURIPathEscaping bool
 
-	// The logger to send log messages to.
-	Logger logging.Logger
-
 	// Enable logging of signed requests.
 	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
 	// presigned URL.
@@ -117,8 +118,8 @@ type SignerOptions struct {
 // Signer applies AWS v4 signing to given request. Use this to sign requests
 // that need to be signed with AWS V4 Signatures.
 type Signer struct {
-	options      SignerOptions
 	keyDerivator keyDerivator
+	options      SignerOptions
 }
 
 // NewSigner returns a new SigV4 Signer
@@ -133,17 +134,19 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
 }
 
 type httpSigner struct {
+	KeyDerivator keyDerivator
 	Request      *http.Request
+	Credentials  aws.Credentials
+	Time         v4Internal.SigningTime
 	ServiceName  string
 	Region       string
-	Time         v4Internal.SigningTime
-	Credentials  aws.Credentials
-	KeyDerivator keyDerivator
-	IsPreSign    bool
-	SignedHdrs   []string
 
 	PayloadHash string
 
+	SignedHdrs []string
+
+	IsPreSign bool
+
 	DisableHeaderHoisting  bool
 	DisableURIPathEscaping bool
 	DisableSessionToken    bool

backend/backend.go

@@ -32,14 +32,14 @@ type Backend interface {
 	Shutdown()
 
 	// bucket operations
-	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
 	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
 	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
-	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
-	PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
-	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	DeleteBucket(_ context.Context, bucket string) error
+	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
+	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
@@ -57,18 +57,18 @@ type Backend interface {
 	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
 
 	// standard object operations
-	PutObject(context.Context, *s3.PutObjectInput) (string, error)
+	PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
 	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
 	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
-	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
 	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
-	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
+	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
 
 	// special case object operations
 	RestoreObject(context.Context, *s3.RestoreObjectInput) error
@@ -108,7 +108,7 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -123,14 +123,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
 func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
+func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -173,8 +173,8 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu
 	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -185,8 +185,8 @@ func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.Ge
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
-	return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -197,8 +197,8 @@ func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3
 func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 	return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -225,8 +225,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
 	}
 }
 
-func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {

backend/common.go

@@ -30,6 +30,12 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
+const (
+	// this is the media type for directories in AWS and Nextcloud
+	DirContentType     = "application/x-directory"
+	DefaultContentType = "binary/octet-stream"
+)
+
 func IsValidBucketName(name string) bool { return true }
 
 type ByBucketName []s3response.ListAllMyBucketsEntry
@@ -44,8 +50,18 @@ func (d ByObjectName) Len() int           { return len(d) }
 func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
 
-func GetStringPtr(s string) *string {
-	return &s
+func GetPtrFromString(str string) *string {
+	if str == "" {
+		return nil
+	}
+	return &str
+}
+
+func GetStringFromPtr(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
 }
 
 func GetTimePtr(t time.Time) *time.Time {
@@ -89,13 +105,37 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 		return 0, 0, errInvalidRange
 	}
 
-	if endOffset <= startOffset {
+	if endOffset < startOffset {
 		return 0, 0, errInvalidRange
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
+// ParseCopySource parses x-amz-copy-source header and returns source bucket,
+// source object, versionId, error respectively
+func ParseCopySource(copySourceHeader string) (string, string, string, error) {
+	if copySourceHeader[0] == '/' {
+		copySourceHeader = copySourceHeader[1:]
+	}
+
+	var copySource, versionId string
+	i := strings.LastIndex(copySourceHeader, "?versionId=")
+	if i == -1 {
+		copySource = copySourceHeader
+	} else {
+		copySource = copySourceHeader[:i]
+		versionId = copySourceHeader[i+11:]
+	}
+
+	srcBucket, srcObject, ok := strings.Cut(copySource, "/")
+	if !ok {
+		return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	return srcBucket, srcObject, versionId, nil
+}
+
 func CreateExceedingRangeErr(objSize int64) s3err.APIError {
 	return s3err.APIError{
 		Code:           "InvalidArgument",

backend/meta/meta.go

@@ -14,17 +14,19 @@
 
 package meta
 
+import "os"
+
 // MetadataStorer defines the interface for managing metadata.
 // When object == "", the operation is on the bucket.
 type MetadataStorer interface {
 	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
 	// Returns the value of the attribute, or an error if the attribute does not exist.
-	RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
+	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
 
 	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
 	// If attribute already exists, new attribute should replace existing.
 	// Returns an error if the operation fails.
-	StoreAttribute(bucket, object, attribute string, value []byte) error
+	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
 
 	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
 	// Returns an error if the operation fails.

backend/meta/xattr.go

@@ -17,6 +17,7 @@ package meta
 import (
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
@@ -36,7 +37,15 @@ var (
 type XattrMeta struct{}
 
 // RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
+func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
+	if f != nil {
+		b, err := xattr.FGet(f, xattrPrefix+attribute)
+		if errors.Is(err, xattr.ENOATTR) {
+			return nil, ErrNoSuchKey
+		}
+		return b, err
+	}
+
 	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
 	if errors.Is(err, xattr.ENOATTR) {
 		return nil, ErrNoSuchKey
@@ -45,7 +54,11 @@ func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte,
 }
 
 // StoreAttribute stores the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
+func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
+	if f != nil {
+		return xattr.FSet(f, xattrPrefix+attribute, value)
+	}
+
 	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
 }
 

backend/mkdir.go

@@ -15,11 +15,6 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
-var (
-	// TODO: make this configurable
-	defaultDirPerm fs.FileMode = 0755
-)
-
 // MkdirAll is similar to os.MkdirAll but it will return
 // ErrObjectParentIsFile when appropriate
 // MkdirAll creates a directory named path,
@@ -32,7 +27,7 @@ var (
 // and returns nil.
 // Any directory created will be set to provided uid/gid ownership
 // if doChown is true.
-func MkdirAll(path string, uid, gid int, doChown bool) error {
+func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
 	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
 	dir, err := os.Stat(path)
 	if err == nil {
@@ -55,14 +50,14 @@ func MkdirAll(path string, uid, gid int, doChown bool) error {
 
 	if j > 1 {
 		// Create parent.
-		err = MkdirAll(path[:j-1], uid, gid, doChown)
+		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
 		if err != nil {
 			return err
 		}
 	}
 
 	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, defaultDirPerm)
+	err = os.Mkdir(path, dirPerm)
 	if err != nil {
 		// Handle arguments like "foo/." by
 		// double-checking that directory doesn't exist.

backend/posix/with_otmpfile.go

@@ -38,11 +38,12 @@ type tmpfile struct {
 	f          *os.File
 	bucket     string
 	objname    string
-	isOTmp     bool
 	size       int64
-	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
+	isOTmp     bool
+	needsChown bool
 }
 
 var (
@@ -62,7 +63,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown)
+		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -108,6 +109,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
+		newDirPerm: p.newDirPerm,
 	}
 
 	// falloc is best effort, its fine if this fails
@@ -134,6 +136,9 @@ func (tmp *tmpfile) falloc() error {
 }
 
 func (tmp *tmpfile) link() error {
+	// make sure this is cleaned up in all error cases
+	defer tmp.f.Close()
+
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
 	// with any other simultaneous uploads. The final operation is to move the
@@ -148,7 +153,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -170,11 +175,21 @@ func (tmp *tmpfile) link() error {
 	}
 	defer dirf.Close()
 
-	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-	if err != nil {
-		return fmt.Errorf("link tmpfile (%q in %q): %w",
-			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
+	for {
+		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		if errors.Is(err, syscall.EEXIST) {
+			err := os.Remove(objPath)
+			if err != nil && !errors.Is(err, fs.ErrNotExist) {
+				return fmt.Errorf("remove stale path: %w", err)
+			}
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("link tmpfile (fd %q as %q): %w",
+				filepath.Base(tmp.f.Name()), objPath, err)
+		}
+		break
 	}
 
 	err = tmp.f.Close()

backend/posix/without_otmpfile.go

@@ -41,7 +41,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 
 	// Create a temp file for upload while in progress (see link comments below).
 	var err error
-	err = backend.MkdirAll(dir, uid, gid, doChown)
+	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}

backend/s3proxy/s3.go

@@ -75,8 +75,12 @@ func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify
 	return s, nil
 }
 
-func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
-	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
+func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{
+		ContinuationToken: &input.ContinuationToken,
+		MaxBuckets:        &input.MaxBuckets,
+		Prefix:            &input.Prefix,
+	})
 	if err != nil {
 		return s3response.ListAllMyBucketsResult{}, handleError(err)
 	}
@@ -96,6 +100,8 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
 		Buckets: s3response.ListAllMyBucketsList{
 			Bucket: buckets,
 		},
+		ContinuationToken: backend.GetStringFromPtr(output.ContinuationToken),
+		Prefix:            backend.GetStringFromPtr(output.Prefix),
 	}, nil
 }
 
@@ -112,8 +118,8 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 
 	var tagSet []types.Tag
 	tagSet = append(tagSet, types.Tag{
-		Key:   backend.GetStringPtr(aclKey),
-		Value: backend.GetStringPtr(base64Encode(acl)),
+		Key:   backend.GetPtrFromString(aclKey),
+		Value: backend.GetPtrFromString(base64Encode(acl)),
 	})
 
 	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
@@ -125,8 +131,10 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 	return handleError(err)
 }
 
-func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
-	_, err := s.client.DeleteBucket(ctx, input)
+func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: &bucket,
+	})
 	return handleError(err)
 }
 
@@ -161,6 +169,51 @@ func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket stri
 	return handleError(err)
 }
 
+func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
+	_, err := s.client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
+		Bucket: &bucket,
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: status,
+		},
+	})
+
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
+		Bucket: &bucket,
+	})
+
+	return s3response.GetBucketVersioningOutput{
+		Status:    &out.Status,
+		MFADelete: &out.MFADelete,
+	}, handleError(err)
+}
+
+func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	out, err := s.client.ListObjectVersions(ctx, input)
+	if err != nil {
+		return s3response.ListVersionsResult{}, handleError(err)
+	}
+
+	return s3response.ListVersionsResult{
+		CommonPrefixes:      out.CommonPrefixes,
+		DeleteMarkers:       out.DeleteMarkers,
+		Delimiter:           out.Delimiter,
+		EncodingType:        out.EncodingType,
+		IsTruncated:         out.IsTruncated,
+		KeyMarker:           out.KeyMarker,
+		MaxKeys:             out.MaxKeys,
+		Name:                out.Name,
+		NextKeyMarker:       out.NextKeyMarker,
+		NextVersionIdMarker: out.NextVersionIdMarker,
+		Prefix:              out.Prefix,
+		VersionIdMarker:     input.VersionIdMarker,
+		Versions:            out.Versions,
+	}, nil
+}
+
 func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	out, err := s.client.CreateMultipartUpload(ctx, input)
 	if err != nil {
@@ -304,17 +357,25 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
 	}, nil
 }
 
-func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
 	// streaming backend is not seekable,
 	// use unsigned payload for streaming ops
 	output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
 		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
 	))
 	if err != nil {
-		return "", handleError(err)
+		return s3response.PutObjectOutput{}, handleError(err)
 	}
 
-	return *output.ETag, nil
+	var versionID string
+	if output.VersionId != nil {
+		versionID = *output.VersionId
+	}
+
+	return s3response.PutObjectOutput{
+		ETag:      *output.ETag,
+		VersionID: versionID,
+	}, nil
 }
 
 func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
@@ -331,7 +392,7 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
 	return output, nil
 }
 
-func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	out, err := s.client.GetObjectAttributes(ctx, input)
 
 	parts := s3response.ObjectParts{}
@@ -358,12 +419,11 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
 		}
 	}
 
-	return s3response.GetObjectAttributesResult{
+	return s3response.GetObjectAttributesResponse{
 		ETag:         out.ETag,
 		LastModified: out.LastModified,
 		ObjectSize:   out.ObjectSize,
 		StorageClass: out.StorageClass,
-		VersionId:    out.VersionId,
 		ObjectParts:  &parts,
 	}, handleError(err)
 }
@@ -416,9 +476,9 @@ func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
 	}, nil
 }
 
-func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
-	_, err := s.client.DeleteObject(ctx, input)
-	return handleError(err)
+func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	res, err := s.client.DeleteObject(ctx, input)
+	return res, handleError(err)
 }
 
 func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
@@ -470,8 +530,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
 	for i, tag := range tagout.TagSet {
 		if *tag.Key == aclKey {
 			tagout.TagSet[i] = types.Tag{
-				Key:   backend.GetStringPtr(aclKey),
-				Value: backend.GetStringPtr(base64Encode(data)),
+				Key:   backend.GetPtrFromString(aclKey),
+				Value: backend.GetPtrFromString(base64Encode(data)),
 			}
 			found = true
 			break
@@ -479,8 +539,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
 	}
 	if !found {
 		tagout.TagSet = append(tagout.TagSet, types.Tag{
-			Key:   backend.GetStringPtr(aclKey),
-			Value: backend.GetStringPtr(base64Encode(data)),
+			Key:   backend.GetPtrFromString(aclKey),
+			Value: backend.GetPtrFromString(base64Encode(data)),
 		})
 	}
 
@@ -540,7 +600,7 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
 func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
 	_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
 		Bucket: &bucket,
-		Policy: backend.GetStringPtr(string(policy)),
+		Policy: backend.GetPtrFromString(string(policy)),
 	})
 	return handleError(err)
 }

backend/scoutfs/scoutfs.go

@@ -44,15 +44,25 @@ type ScoutfsOpts struct {
 	ChownGID    bool
 	GlacierMode bool
 	BucketLinks bool
+	NewDirPerm  fs.FileMode
 }
 
 type ScoutFS struct {
+
+	// bucket/object metadata storage facility
+	meta meta.MetadataStorer
+
 	*posix.Posix
 	rootfd  *os.File
 	rootdir string
 
-	// bucket/object metadata storage facility
-	meta meta.MetadataStorer
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
+
+	// newDirPerm is the permissions to use when creating new directories
+	newDirPerm fs.FileMode
 
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
@@ -69,11 +79,6 @@ type ScoutFS struct {
 	// when objects are uploaded
 	chownuid bool
 	chowngid bool
-
-	// euid/egid are the effective uid/gid of the running versitygw process
-	// used to determine if chowning is needed
-	euid int
-	egid int
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -226,7 +231,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
-		b, err := s.meta.RetrieveAttribute(bucket, partObjPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
@@ -262,7 +267,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = moveData(pf, f.f)
+		err = moveData(pf, f.File())
 		pf.Close()
 		if err != nil {
 			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
@@ -277,83 +282,76 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	dir := filepath.Dir(objname)
 	if dir != "" {
 		uid, gid, doChown := s.getChownIDs(acct)
-		err = backend.MkdirAll(dir, uid, gid, doChown)
+		err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
 		if err != nil {
 			return nil, err
 		}
 	}
-	err = f.link()
-	if err != nil {
-		return nil, fmt.Errorf("link object in namespace: %w", err)
-	}
 
 	for k, v := range userMetaData {
-		err = s.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
+		err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
-			// cleanup object if returning error
-			os.Remove(objname)
 			return nil, fmt.Errorf("set user attr %q: %w", k, err)
 		}
 	}
 
 	// load and set tagging
-	tagging, err := s.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
-	if err == nil {
-		if err := s.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
-			// cleanup object
-			os.Remove(objname)
-			return nil, fmt.Errorf("set object tagging: %w", err)
-		}
-	}
+	tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
 		return nil, fmt.Errorf("get object tagging: %w", err)
 	}
+	if err == nil {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
+		if err != nil {
+			return nil, fmt.Errorf("set object tagging: %w", err)
+		}
+	}
 
 	// set content-type
 	if cType != "" {
-		if err := s.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
-			// cleanup object
-			os.Remove(objname)
+		err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
+		if err != nil {
 			return nil, fmt.Errorf("set object content type: %w", err)
 		}
 	}
 
 	// load and set legal hold
-	lHold, err := s.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
-	if err == nil {
-		if err := s.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
-			// cleanup object
-			os.Remove(objname)
-			return nil, fmt.Errorf("set object legal hold: %w", err)
-		}
-	}
+	lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
 		return nil, fmt.Errorf("get object legal hold: %w", err)
 	}
-
-	// load and set retention
-	ret, err := s.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
 	if err == nil {
-		if err := s.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
-			// cleanup object
-			os.Remove(objname)
-			return nil, fmt.Errorf("set object retention: %w", err)
+		err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
+		if err != nil {
+			return nil, fmt.Errorf("set object legal hold: %w", err)
 		}
 	}
+
+	// load and set retention
+	ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
 		return nil, fmt.Errorf("get object retention: %w", err)
 	}
+	if err == nil {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
+		if err != nil {
+			return nil, fmt.Errorf("set object retention: %w", err)
+		}
+	}
 
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
-	err = s.meta.StoreAttribute(bucket, object, etagkey, []byte(s3MD5))
+	err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
 	if err != nil {
-		// cleanup object if returning error
-		os.Remove(objname)
 		return nil, fmt.Errorf("set etag attr: %w", err)
 	}
 
+	err = f.link()
+	if err != nil {
+		return nil, fmt.Errorf("link object in namespace: %w", err)
+	}
+
 	// cleanup tmp dirs
 	os.RemoveAll(upiddir)
 	// use Remove for objdir in case there are still other uploads
@@ -392,7 +390,7 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
 		if !isValidMeta(e) {
 			continue
 		}
-		b, err := s.meta.RetrieveAttribute(bucket, object, e)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, object, e)
 		if err != nil {
 			continue
 		}
@@ -404,13 +402,13 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
 	}
 
 	var contentType, contentEncoding string
-	b, _ := s.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
+	b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
 	contentType = string(b)
 	if contentType != "" {
 		m[contentTypeHdr] = contentType
 	}
 
-	b, _ = s.meta.RetrieveAttribute(bucket, object, contentEncHdr)
+	b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
 	contentEncoding = string(b)
 	if contentEncoding != "" {
 		m[contentEncHdr] = contentEncoding
@@ -466,7 +464,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 			return nil, fmt.Errorf("stat part: %w", err)
 		}
 
-		b, err := s.meta.RetrieveAttribute(bucket, partPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
@@ -514,7 +512,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 		contentType = "application/x-directory"
 	}
 
-	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
+	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -554,7 +552,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	contentLength := fi.Size()
 
 	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
-	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, "")
+	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
 	if err == nil {
 		if *status {
 			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
@@ -565,7 +563,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 
 	var objectLockMode types.ObjectLockMode
 	var objectLockRetainUntilDate *time.Time
-	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, "")
+	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
 	if err == nil {
 		var config types.ObjectLockRetention
 		if err := json.Unmarshal(retention, &config); err == nil {
@@ -685,7 +683,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 
 	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
+	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -840,7 +838,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			etagBytes, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
+			etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
 			if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
 				return s3response.Object{}, backend.ErrSkipObj
 			}
@@ -869,7 +867,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		}
 
 		// file object, get object info and fill out object data
-		b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
 			return s3response.Object{}, backend.ErrSkipObj
 		}

backend/scoutfs/scoutfs_compat.go

@@ -40,6 +40,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		ChownUID:    opts.ChownUID,
 		ChownGID:    opts.ChownGID,
 		BucketLinks: opts.BucketLinks,
+		NewDirPerm:  opts.NewDirPerm,
 	})
 	if err != nil {
 		return nil, err
@@ -58,6 +59,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		chownuid:    opts.ChownUID,
 		chowngid:    opts.ChownGID,
 		glaciermode: opts.GlacierMode,
+		newDirPerm:  opts.NewDirPerm,
 	}, nil
 }
 
@@ -68,13 +70,13 @@ type tmpfile struct {
 	bucket     string
 	objname    string
 	size       int64
-	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
+	needsChown bool
 }
 
 var (
-	// TODO: make this configurable
 	defaultFilePerm uint32 = 0644
 )
 
@@ -102,6 +104,7 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
+		newDirPerm: s.newDirPerm,
 	}
 
 	if doChown {
@@ -129,7 +132,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -174,6 +177,10 @@ func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}
+
 func moveData(from *os.File, to *os.File) error {
 	return scoutfs.MoveData(from, to)
 }

backend/scoutfs/scoutfs_incompat.go

@@ -28,9 +28,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
-type tmpfile struct {
-	f *os.File
-}
+type tmpfile struct{}
 
 var (
 	errNotSupported = errors.New("not supported")
@@ -56,6 +54,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return nil
+}
+
 func moveData(_, _ *os.File) error {
 	return errNotSupported
 }

backend/walk.go

@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
-	"os"
 	"sort"
 	"strings"
 
@@ -28,10 +27,10 @@ import (
 )
 
 type WalkResults struct {
+	NextMarker     string
 	CommonPrefixes []types.CommonPrefix
 	Objects        []s3response.Object
 	Truncated      bool
-	NextMarker     string
 }
 
 type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
@@ -53,7 +52,15 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 	var newMarker string
 	var truncated bool
 
-	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+	root := "."
+	if strings.Contains(prefix, "/") {
+		idx := strings.LastIndex(prefix, "/")
+		if idx > 0 {
+			root = prefix[:idx]
+		}
+	}
+
+	err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -76,6 +83,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipAll
 		}
 
+		// After this point, return skipflag instead of nil
+		// so we can skip a directory without an early return
+		var skipflag error
 		if d.IsDir() {
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
@@ -85,51 +95,57 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// building to match. So only skip if path isn't a prefix of prefix
 			// and prefix isn't a prefix of path.
 			if prefix != "" &&
-				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
-				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
+				!strings.HasPrefix(path+"/", prefix) &&
+				!strings.HasPrefix(prefix, path+"/") {
 				return fs.SkipDir
 			}
 
-			// TODO: can we do better here rather than a second readdir
-			// per directory?
-			ents, err := fs.ReadDir(fileSystem, path)
-			if err != nil {
-				return fmt.Errorf("readdir %q: %w", path, err)
-			}
-
-			path += string(os.PathSeparator)
-
-			if len(ents) == 0 && delimiter == "" {
-				dirobj, err := getObj(path, d)
-				if err == ErrSkipObj {
-					return nil
-				}
+			// Don't recurse into subdirectories which contain the delimiter
+			// after reaching the prefix
+			if delimiter != "" &&
+				strings.HasPrefix(path+"/", prefix) &&
+				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
+				skipflag = fs.SkipDir
+			} else {
+				// TODO: can we do better here rather than a second readdir
+				// per directory?
+				ents, err := fs.ReadDir(fileSystem, path)
 				if err != nil {
-					return fmt.Errorf("directory to object %q: %w", path, err)
+					return fmt.Errorf("readdir %q: %w", path, err)
 				}
-				objects = append(objects, dirobj)
+				if len(ents) == 0 && delimiter == "" {
+					dirobj, err := getObj(path+"/", d)
+					if err == ErrSkipObj {
+						return skipflag
+					}
+					if err != nil {
+						return fmt.Errorf("directory to object %q: %w", path, err)
+					}
+					objects = append(objects, dirobj)
 
-				return nil
-			}
+					return skipflag
+				}
 
-			if len(ents) != 0 {
-				return nil
+				if len(ents) != 0 {
+					return skipflag
+				}
 			}
+			path += "/"
 		}
 
 		if !pastMarker {
 			if path == marker {
 				pastMarker = true
-				return nil
+				return skipflag
 			}
 			if path < marker {
-				return nil
+				return skipflag
 			}
 		}
 
 		// If object doesn't have prefix, don't include in results.
 		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return nil
+			return skipflag
 		}
 
 		if delimiter == "" {
@@ -137,7 +153,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// prefix are included in results
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return nil
+				return skipflag
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -148,7 +164,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 				pastMax = true
 			}
 
-			return nil
+			return skipflag
 		}
 
 		// Since delimiter is specified, we only want results that
@@ -177,7 +193,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		if !found {
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return nil
+				return skipflag
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -186,17 +202,24 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			if (len(objects) + len(cpmap)) == int(max) {
 				pastMax = true
 			}
-			return nil
+			return skipflag
 		}
 
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
-		// delimiter at the end.
+		// delimiter at the end when we add to the map.
+		cprefNoDelim := prefix + before
 		cpref := prefix + before + delimiter
 		if cpref == marker {
 			pastMarker = true
-			return nil
+			return skipflag
 		}
+
+		if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
+			// skip common prefixes that are before the marker
+			return skipflag
+		}
+
 		cpmap[cpref] = struct{}{}
 		if (len(objects) + len(cpmap)) == int(max) {
 			newMarker = cpref
@@ -204,9 +227,13 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipAll
 		}
 
-		return nil
+		return skipflag
 	})
 	if err != nil {
+		// suppress file not found caused by user's prefix
+		if errors.Is(err, fs.ErrNotExist) {
+			return WalkResults{}, nil
+		}
 		return WalkResults{}, err
 	}
 
@@ -239,3 +266,213 @@ func contains(a string, strs []string) bool {
 	}
 	return false
 }
+
+type WalkVersioningResults struct {
+	NextMarker          string
+	NextVersionIdMarker string
+	CommonPrefixes      []types.CommonPrefix
+	ObjectVersions      []types.ObjectVersion
+	DelMarkers          []types.DeleteMarkerEntry
+	Truncated           bool
+}
+
+type ObjVersionFuncResult struct {
+	NextVersionIdMarker string
+	ObjectVersions      []types.ObjectVersion
+	DelMarkers          []types.DeleteMarkerEntry
+	Truncated           bool
+}
+
+type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
+
+// WalkVersions walks the supplied fs.FS and returns results compatible with
+// ListObjectVersions action response
+func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
+	cpmap := make(map[string]struct{})
+	var objects []types.ObjectVersion
+	var delMarkers []types.DeleteMarkerEntry
+
+	var pastMarker bool
+	if keyMarker == "" {
+		pastMarker = true
+	}
+	var nextMarker string
+	var nextVersionIdMarker string
+	var truncated bool
+
+	pastVersionIdMarker := versionIdMarker == ""
+
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
+
+		if !pastMarker {
+			if path == keyMarker {
+				pastMarker = true
+			}
+			if path < keyMarker {
+				return nil
+			}
+		}
+
+		if d.IsDir() {
+			// If prefix is defined and the directory does not match prefix,
+			// do not descend into the directory because nothing will
+			// match this prefix. Make sure to append the / at the end of
+			// directories since this is implied as a directory path name.
+			// If path is a prefix of prefix, then path could still be
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
+			if prefix != "" &&
+				!strings.HasPrefix(path+"/", prefix) &&
+				!strings.HasPrefix(prefix, path+"/") {
+				return fs.SkipDir
+			}
+
+			// Don't recurse into subdirectories when listing with delimiter.
+			if delimiter == "/" &&
+				prefix != path+"/" &&
+				strings.HasPrefix(path+"/", prefix) {
+				cpmap[path+"/"] = struct{}{}
+				return fs.SkipDir
+			}
+
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("directory to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+
+			return nil
+		}
+
+		// If object doesn't have prefix, don't include in results.
+		if prefix != "" && !strings.HasPrefix(path, prefix) {
+			return nil
+		}
+
+		if delimiter == "" {
+			// If no delimiter specified, then all files with matching
+			// prefix are included in results
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+
+			return nil
+		}
+
+		// Since delimiter is specified, we only want results that
+		// do not contain the delimiter beyond the prefix.  If the
+		// delimiter exists past the prefix, then the substring
+		// between the prefix and delimiter is part of common prefixes.
+		//
+		// For example:
+		// prefix = A/
+		// delimiter = /
+		// and objects:
+		// A/file
+		// A/B/file
+		// B/C
+		// would return:
+		// objects: A/file
+		// common prefix: A/B/
+		//
+		// Note: No objects are included past the common prefix since
+		// these are all rolled up into the common prefix.
+		// Note: The delimiter can be anything, so we have to operate on
+		// the full path without any assumptions on posix directory hierarchy
+		// here.  Usually the delimiter will be "/", but thats not required.
+		suffix := strings.TrimPrefix(path, prefix)
+		before, _, found := strings.Cut(suffix, delimiter)
+		if !found {
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+			return nil
+		}
+
+		// Common prefixes are a set, so should not have duplicates.
+		// These are abstractly a "directory", so need to include the
+		// delimiter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
+		if (len(objects) + len(cpmap)) == int(max) {
+			nextMarker = path
+			truncated = true
+
+			return fs.SkipAll
+		}
+
+		return nil
+	})
+	if err != nil {
+		return WalkVersioningResults{}, err
+	}
+
+	var commonPrefixStrings []string
+	for k := range cpmap {
+		commonPrefixStrings = append(commonPrefixStrings, k)
+	}
+	sort.Strings(commonPrefixStrings)
+	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
+	for _, cp := range commonPrefixStrings {
+		pfx := cp
+		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
+			Prefix: &pfx,
+		})
+	}
+
+	return WalkVersioningResults{
+		CommonPrefixes:      commonPrefixes,
+		ObjectVersions:      objects,
+		DelMarkers:          delMarkers,
+		Truncated:           truncated,
+		NextMarker:          nextMarker,
+		NextVersionIdMarker: nextVersionIdMarker,
+	}, nil
+}

backend/walk_test.go

@@ -31,9 +31,18 @@ import (
 )
 
 type walkTest struct {
-	fsys     fs.FS
-	expected backend.WalkResults
-	getobj   backend.GetObjFunc
+	fsys   fs.FS
+	getobj backend.GetObjFunc
+	cases  []testcase
+}
+
+type testcase struct {
+	name      string
+	prefix    string
+	delimiter string
+	marker    string
+	expected  backend.WalkResults
+	maxObjs   int32
 }
 
 func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
@@ -88,50 +97,154 @@ func TestWalk(t *testing.T) {
 				"photos/2006/February/sample3.jpg": {},
 				"photos/2006/February/sample4.jpg": {},
 			},
-			expected: backend.WalkResults{
-				CommonPrefixes: []types.CommonPrefix{{
-					Prefix: backend.GetStringPtr("photos/"),
-				}},
-				Objects: []s3response.Object{{
-					Key: backend.GetStringPtr("sample.jpg"),
-				}},
-			},
 			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "aws example",
+					delimiter: "/",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photos/"),
+						}},
+						Objects: []s3response.Object{{
+							Key: backend.GetPtrFromString("sample.jpg"),
+						}},
+					},
+				},
+			},
 		},
 		{
 			// test case single dir/single file
 			fsys: fstest.MapFS{
 				"test/file": {},
 			},
-			expected: backend.WalkResults{
-				CommonPrefixes: []types.CommonPrefix{{
-					Prefix: backend.GetStringPtr("test/"),
-				}},
-				Objects: []s3response.Object{},
+			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "single dir single file",
+					delimiter: "/",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("test/"),
+						}},
+						Objects: []s3response.Object{},
+					},
+				},
+			},
+		},
+		{
+			// non-standard delimiter
+			fsys: fstest.MapFS{
+				"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
+				"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
+				"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
 			},
 			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "different delimiter 1",
+					delimiter: "|",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 2",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 3",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 4",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 5",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|6/Januar|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 6",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|6/Januar|y/sampl|",
+					expected: backend.WalkResults{
+						Objects: []s3response.Object{
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
+							},
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
+							},
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
+							},
+						},
+					},
+				},
+			},
 		},
 	}
 
 	for _, tt := range tests {
-		res, err := backend.Walk(context.Background(), tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
-		if err != nil {
-			t.Fatalf("walk: %v", err)
-		}
+		for _, tc := range tt.cases {
+			res, err := backend.Walk(context.Background(),
+				tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
+				tt.getobj, []string{})
+			if err != nil {
+				t.Errorf("tc.name: walk: %v", err)
+			}
 
-		compareResults(res, tt.expected, t)
+			compareResults(tc.name, res, tc.expected, t)
+		}
 	}
 }
 
-func compareResults(got, wanted backend.WalkResults, t *testing.T) {
+func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
 	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
-		t.Errorf("unexpected common prefix, got %v wanted %v",
+		t.Errorf("%v: unexpected common prefix, got %v wanted %v",
+			name,
 			printCommonPrefixes(got.CommonPrefixes),
 			printCommonPrefixes(wanted.CommonPrefixes))
 	}
 
 	if !compareObjects(got.Objects, wanted.Objects) {
-		t.Errorf("unexpected object, got %v wanted %v",
+		t.Errorf("%v: unexpected object, got %v wanted %v",
+			name,
 			printObjects(got.Objects),
 			printObjects(wanted.Objects))
 	}
@@ -202,10 +315,16 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
 func printObjects(list []s3response.Object) string {
 	res := "["
 	for _, cp := range list {
-		if res == "[" {
-			res = res + *cp.Key
+		var key string
+		if cp.Key == nil {
+			key = "<nil>"
 		} else {
-			res = res + ", " + *cp.Key
+			key = *cp.Key
+		}
+		if res == "[" {
+			res = res + key
+		} else {
+			res = res + ", " + key
 		}
 	}
 	return res + "]"

cmd/versitygw/admin.go

@@ -19,7 +19,7 @@ import (
 	"crypto/sha256"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,6 +29,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3response"
@@ -37,6 +38,7 @@ import (
 var (
 	adminAccess   string
 	adminSecret   string
+	adminRegion   string
 	adminEndpoint string
 	allowInsecure bool
 )
@@ -171,6 +173,14 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminSecret,
 			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "admin s3 region string",
+				EnvVars:     []string{"ADMIN_REGION"},
+				Value:       "us-east-1",
+				Destination: &adminRegion,
+				Aliases:     []string{"r"},
+			},
 			&cli.StringFlag{
 				Name:        "endpoint-url",
 				Usage:       "admin apis endpoint url",
@@ -215,24 +225,24 @@ func createUser(ctx *cli.Context) error {
 		GroupID: groupID,
 	}
 
-	accJson, err := json.Marshal(acc)
+	accxml, err := xml.Marshal(acc)
 	if err != nil {
 		return fmt.Errorf("failed to parse user data: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(accJson)
+	hashedPayload := sha256.Sum256(accxml)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -251,11 +261,9 @@ func createUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -277,7 +285,7 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -296,11 +304,9 @@ func deleteUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -317,24 +323,24 @@ func updateUser(ctx *cli.Context) error {
 		props.GroupID = &groupId
 	}
 
-	propsJSON, err := json.Marshal(props)
+	propsxml, err := xml.Marshal(props)
 	if err != nil {
 		return fmt.Errorf("failed to parse user attributes: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(propsJSON)
+	hashedPayload := sha256.Sum256(propsxml)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -353,11 +359,9 @@ func updateUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -374,7 +378,7 @@ func listUsers(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -393,15 +397,15 @@ func listUsers(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var accs []auth.Account
-	if err := json.Unmarshal(body, &accs); err != nil {
+	var accs auth.ListUserAccountsResult
+	if err := xml.Unmarshal(body, &accs); err != nil {
 		return err
 	}
 
-	printAcctTable(accs)
+	printAcctTable(accs.Accounts)
 
 	return nil
 }
@@ -441,7 +445,7 @@ func changeBucketOwner(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -460,11 +464,9 @@ func changeBucketOwner(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Println(string(body))
-
 	return nil
 }
 
@@ -493,7 +495,7 @@ func listBuckets(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -512,15 +514,26 @@ func listBuckets(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var buckets []s3response.Bucket
-	if err := json.Unmarshal(body, &buckets); err != nil {
+	var result s3response.ListBucketsResult
+	if err := xml.Unmarshal(body, &result); err != nil {
 		return err
 	}
 
-	printBuckets(buckets)
+	printBuckets(result.Buckets)
 
 	return nil
 }
+
+func parseApiError(body []byte) error {
+	var apiErr smithy.GenericAPIError
+	err := xml.Unmarshal(body, &apiErr)
+	if err != nil {
+		apiErr.Code = "InternalServerError"
+		apiErr.Message = err.Error()
+	}
+
+	return &apiErr
+}

cmd/versitygw/gateway_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
@@ -57,7 +58,9 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
+	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
+		NewDirPerm: 0755,
+	})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}
@@ -75,6 +78,9 @@ func initPosix(ctx context.Context) {
 		}
 		wg.Done()
 	}()
+
+	// wait for server to start
+	time.Sleep(1 * time.Second)
 }
 
 func TestIntegration(t *testing.T) {

cmd/versitygw/posix.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/meta"
@@ -25,6 +27,8 @@ import (
 var (
 	chownuid, chowngid bool
 	bucketlinks        bool
+	versioningDir      string
+	dirPerms           uint
 )
 
 func posixCommand() *cli.Command {
@@ -61,6 +65,20 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_BUCKET_LINKS"},
 				Destination: &bucketlinks,
 			},
+			&cli.StringFlag{
+				Name:        "versioning-dir",
+				Usage:       "the directory path to enable bucket versioning",
+				EnvVars:     []string{"VGW_VERSIONING_DIR"},
+				Destination: &versioningDir,
+			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
 		},
 	}
 }
@@ -76,10 +94,16 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("posix xattr check: %v", err)
 	}
 
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
 	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
-		ChownUID:    chownuid,
-		ChownGID:    chowngid,
-		BucketLinks: bucketlinks,
+		ChownUID:      chownuid,
+		ChownGID:      chowngid,
+		BucketLinks:   bucketlinks,
+		VersioningDir: versioningDir,
+		NewDirPerm:    fs.FileMode(dirPerms),
 	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)

cmd/versitygw/scoutfs.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/scoutfs"
@@ -69,6 +71,14 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_BUCKET_LINKS"},
 				Destination: &bucketlinks,
 			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
 		},
 	}
 }
@@ -78,11 +88,16 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
 	var opts scoutfs.ScoutfsOpts
 	opts.GlacierMode = glacier
 	opts.ChownUID = chownuid
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
+	opts.NewDirPerm = fs.FileMode(dirPerms)
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

cmd/versitygw/test.go

@@ -22,20 +22,22 @@ import (
 )
 
 var (
-	awsID           string
-	awsSecret       string
-	endpoint        string
-	prefix          string
-	dstBucket       string
-	partSize        int64
-	objSize         int64
-	concurrency     int
-	files           int
-	totalReqs       int
-	upload          bool
-	download        bool
-	pathStyle       bool
-	checksumDisable bool
+	awsID             string
+	awsSecret         string
+	endpoint          string
+	prefix            string
+	dstBucket         string
+	partSize          int64
+	objSize           int64
+	concurrency       int
+	files             int
+	totalReqs         int
+	upload            bool
+	download          bool
+	pathStyle         bool
+	checksumDisable   bool
+	versioningEnabled bool
+	azureTests        bool
 )
 
 func testCommand() *cli.Command {
@@ -87,11 +89,33 @@ func initTestCommands() []*cli.Command {
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+				&cli.BoolFlag{
+					Name:        "azure-test-mode",
+					Usage:       "Skips tests that are not supported by Azure",
+					Destination: &azureTests,
+					Aliases:     []string{"azure"},
+				},
+			},
 		},
 		{
 			Name:   "posix",
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test posix when versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		},
 		{
 			Name:   "iam",
@@ -276,6 +300,12 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if debug {
 			opts = append(opts, integration.WithDebug())
 		}
+		if versioningEnabled {
+			opts = append(opts, integration.WithVersioningEnabled())
+		}
+		if azureTests {
+			opts = append(opts, integration.WithAzureMode())
+		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -307,11 +337,22 @@ func extractIntTests() (commands []*cli.Command) {
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
+				if versioningEnabled {
+					opts = append(opts, integration.WithVersioningEnabled())
+				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)
 				return err
 			},
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		})
 	}
 	return

docker-compose-bats.yml

@@ -1,36 +0,0 @@
-version: '3'
-
-services:
-  no_certs:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.nocerts
-  static_buckets:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.static
-  posix_backend:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.default
-    image: bats_test
-  s3_backend:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.s3
-        - SECRETS_FILE=tests/.secrets.s3
-  direct:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.direct
-        - SECRETS_FILE=tests/.secrets.direct

extra/example.conf

@@ -311,6 +311,12 @@ ROOT_SECRET_ACCESS_KEY=
 # to directories at the top level gateway directory as buckets.
 #VGW_BUCKET_LINKS=false
 
+# The default permissions mode when creating new directories is 0755. Use
+# VGW_DIR_PERMS option to set a different mode for any new directory that the
+# gateway creates. This applies to buckets created through the gateway as well
+# as any parent directories automatically created with object uploads.
+#VGW_DIR_PERMS=0755
+
 ###########
 # scoutfs #
 ###########
@@ -346,6 +352,12 @@ ROOT_SECRET_ACCESS_KEY=
 # to directories at the top level gateway directory as buckets.
 #VGW_BUCKET_LINKS=false
 
+# The default permissions mode when creating new directories is 0755. Use
+# VGW_DIR_PERMS option to set a different mode for any new directory that the
+# gateway creates. This applies to buckets created through the gateway as well
+# as any parent directories automatically created with object uploads.
+#VGW_DIR_PERMS=0755
+
 ######
 # s3 #
 ######

go.mod

@@ -3,38 +3,40 @@ module github.com/versity/versitygw
 go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.30.4
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1
-	github.com/aws/smithy-go v1.20.4
+	github.com/aws/aws-sdk-go-v2 v1.32.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
+	github.com/aws/smithy-go v1.22.0
 	github.com/go-ldap/ldap/v3 v3.4.8
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
 	github.com/nats-io/nats.go v1.37.0
+	github.com/oklog/ulid/v2 v2.1.0
 	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
-	github.com/smira/go-statsd v1.3.3
-	github.com/urfave/cli/v2 v2.27.4
-	github.com/valyala/fasthttp v1.55.0
+	github.com/smira/go-statsd v1.3.4
+	github.com/urfave/cli/v2 v2.27.5
+	github.com/valyala/fasthttp v1.57.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sys v0.24.0
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.26.0
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -48,27 +50,27 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.26.0 // indirect
-	golang.org/x/net v0.28.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
-	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.31
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.30
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.15
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.28.1
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.42
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect

go.sum

@@ -1,17 +1,21 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
 github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -19,51 +23,55 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
-github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
-github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
-github.com/aws/aws-sdk-go-v2/config v1.27.31 h1:kxBoRsjhT3pq0cKthgj6RU6bXTm/2SgdoUMyrVw0rAI=
-github.com/aws/aws-sdk-go-v2/config v1.27.31/go.mod h1:z04nZdSWFPaDwK3DdJOG2r+scLQzMYuJeW0CujEm9FM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.30 h1:aau/oYFtibVovr2rDt8FHlU17BTicFEMAi29V1U+L5Q=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.30/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.15 h1:ijB7hr56MngOiELJe0C5aQRaBQ11LveNgWFyG02AUto=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.15/go.mod h1:0QEmQSSWMVfiAk93l1/ayR9DQ9+jwni7gHS2NARZXB0=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
+github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
+github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw=
+github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35 h1:ihPPdcCVSN0IvBByXwqVp28/l4VosBZ6sDulcvU2J7w=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35/go.mod h1:JkgEhs3SVF51Dj3m1Bj+yL8IznpxzkwlA3jLg3x7Kls=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1 h1:mx2ucgtv+MWzJesJY9Ig/8AFHgoE5FwLXwUVgW/FGdI=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c=
-github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac=
-github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8=
-github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0=
-github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
-github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE=
+github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
+github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
@@ -109,9 +117,11 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -129,6 +139,9 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
+github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
+github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -139,6 +152,8 @@ github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
 github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
+github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -149,8 +164,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
-github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
+github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
+github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
@@ -162,12 +177,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
-github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
-github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
+github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
+github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -180,6 +195,8 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -189,8 +206,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
-golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -206,12 +223,14 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
-golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -231,8 +250,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -248,10 +267,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
-golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

metrics/actions.go

@@ -72,6 +72,14 @@ var (
 	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
 	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
 	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
+
+	// Admin actions
+	ActionAdminCreateUser        = "admin_CreateUser"
+	ActionAdminUpdateUser        = "admin_UpdateUser"
+	ActionAdminDeleteUser        = "admin_DeleteUser"
+	ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
+	ActionAdminListUsers         = "admin_ListUsers"
+	ActionAdminListBuckets       = "admin_ListBuckets"
 )
 
 func init() {

metrics/metrics.go

@@ -43,13 +43,14 @@ type Tag struct {
 
 // Manager is a manager of metrics plugins
 type Manager struct {
-	wg  sync.WaitGroup
 	ctx context.Context
 
+	addDataChan chan datapoint
+
 	config Config
 
-	publishers  []publisher
-	addDataChan chan datapoint
+	publishers []publisher
+	wg         sync.WaitGroup
 }
 
 type Config struct {
@@ -220,6 +221,6 @@ func (m *Manager) addForwarder(addChan <-chan datapoint) {
 
 type datapoint struct {
 	key   string
-	value int64
 	tags  []Tag
+	value int64
 }

runtests.sh

@@ -5,15 +5,19 @@ rm -rf /tmp/gw
 mkdir /tmp/gw
 rm -rf /tmp/covdata
 mkdir /tmp/covdata
+rm -rf /tmp/versioing.covdata
+mkdir /tmp/versioning.covdata
+rm -rf /tmp/versioningdir
+mkdir /tmp/versioningdir
 
-# run server in background
+# run server in background not versioning-enabled
+# port: 7070(default)
 GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
 GW_PID=$!
 
-# wait a second for server to start up
 sleep 1
 
-# check if server is still running
+# check if versioning-enabled gateway process is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -39,8 +43,39 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
 	exit 1
 fi
 
-# kill off server
 kill $GW_PID
+
+# run server in background versioning-enabled
+# port: 7071
+GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7071 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+GW_VS_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if versioning-enabled gateway process is still running
+if ! kill -0 $GW_VS_PID; then
+	echo "versioning-enabled server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 full-flow -vs; then
+	echo "versioning-enabled full-flow tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 posix -vs; then
+	echo "versiongin-enabled posix tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_VS_PID
+
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-server.go

@@ -26,11 +26,11 @@ import (
 )
 
 type S3AdminServer struct {
-	app     *fiber.App
 	backend backend.Backend
+	app     *fiber.App
 	router  *S3AdminRouter
-	port    string
 	cert    *tls.Certificate
+	port    string
 }
 
 func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
@@ -53,6 +53,9 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
 	app.Use(middlewares.VerifyMD5Body(l))
 
+	// Admin role checker
+	app.Use(middlewares.IsAdmin(l))
+
 	server.router.Init(app, be, iam, l)
 
 	return server

s3api/controllers/admin.go

@@ -16,15 +16,19 @@ package controllers
 
 import (
 	"encoding/json"
-	"errors"
+	"encoding/xml"
 	"fmt"
+	"net/http"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
+	"github.com/versity/versitygw/s3response"
 )
 
 type AdminController struct {
@@ -38,187 +42,124 @@ func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLo
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:CreateUser",
-			})
-	}
 	var usr auth.Account
-	err := json.Unmarshal(ctx.Body(), &usr)
+	err := xml.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return sendResponse(ctx, fmt.Errorf("failed to parse request body: %w", err), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusBadRequest,
-				action: "admin:CreateUser",
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
 			})
 	}
 
-	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return sendResponse(ctx, errors.New("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusBadRequest,
-				action: "admin:CreateUser",
+	if !usr.Role.IsValid() {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
 			})
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		status := fiber.StatusInternalServerError
-		err = fmt.Errorf("failed to create user: %w", err)
-
 		if strings.Contains(err.Error(), "user already exists") {
-			status = fiber.StatusConflict
+			err = s3err.GetAPIError(s3err.ErrAdminUserExists)
 		}
 
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				status: status,
-				logger: c.l,
-				action: "admin:CreateUser",
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
 			})
 	}
 
-	return sendResponse(ctx, nil, "The user has been created successfully", &metaOptions{
-		status: fiber.StatusCreated,
-		logger: c.l,
-		action: "admin:CreateUser",
-	})
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminCreateUser,
+			Status: http.StatusCreated,
+		})
 }
 
 func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:UpdateUser",
-			})
-	}
-
 	access := ctx.Query("access")
 	if access == "" {
-		return sendResponse(ctx, errors.New("missing user access parameter"), nil,
-			&metaOptions{
-				status: fiber.StatusBadRequest,
-				logger: c.l,
-				action: "admin:UpdateUser",
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
 			})
 	}
 
 	var props auth.MutableProps
-	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
-		return sendResponse(ctx, fmt.Errorf("invalid request body %w", err), nil,
-			&metaOptions{
-				status: fiber.StatusBadRequest,
-				logger: c.l,
-				action: "admin:UpdateUser",
+	if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
 			})
 	}
 
 	err := c.iam.UpdateUserAccount(access, props)
 	if err != nil {
-		status := fiber.StatusInternalServerError
-		err = fmt.Errorf("failed to update user account: %w", err)
-
 		if strings.Contains(err.Error(), "user not found") {
-			status = fiber.StatusNotFound
+			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
 		}
 
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				status: status,
-				logger: c.l,
-				action: "admin:UpdateUser",
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
 			})
 	}
 
-	return sendResponse(ctx, nil, "the user has been updated successfully",
-		&metaOptions{
-			logger: c.l,
-			action: "admin:UpdateUser",
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminUpdateUser,
 		})
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:DeleteUser",
-			})
-	}
 
 	err := c.iam.DeleteUserAccount(access)
-	if err != nil {
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:DeleteUser",
-			})
-	}
-
-	return sendResponse(ctx, nil, "The user has been deleted successfully",
-		&metaOptions{
-			logger: c.l,
-			action: "admin:DeleteUser",
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminDeleteUser,
 		})
 }
 
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:ListUsers",
-			})
-	}
 	accs, err := c.iam.ListUserAccounts()
-	return sendResponse(ctx, err, accs,
-		&metaOptions{
-			logger: c.l,
-			action: "admin:ListUsers",
+	return SendXMLResponse(ctx,
+		auth.ListUserAccountsResult{
+			Accounts: accs,
+		}, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListUsers,
 		})
 }
 
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:ChangeBucketOwner",
-			})
-	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
 
 	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
 	if err != nil {
-		return sendResponse(ctx, err, nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:ChangeBucketOwner",
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
 			})
 	}
 	if len(accs) > 0 {
-		return sendResponse(ctx, errors.New("user specified as the new bucket owner does not exist"), nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:ChangeBucketOwner",
-				status: fiber.StatusNotFound,
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
 			})
 	}
 
@@ -235,91 +176,28 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 
 	aclParsed, err := json.Marshal(acl)
 	if err != nil {
-		return sendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err), nil,
-			&metaOptions{
-				logger: c.l,
-				action: "admin:ChangeBucketOwner",
+		return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
 			})
 	}
 
 	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
-	return sendResponse(ctx, err, "Bucket owner has been updated successfully",
-		&metaOptions{
-			logger: c.l,
-			action: "admin:ChangeBucketOwner",
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminChangeBucketOwner,
 		})
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
-			&metaOptions{
-				logger: c.l,
-				status: fiber.StatusForbidden,
-				action: "admin:ListBuckets",
-			})
-	}
-
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
-	return sendResponse(ctx, err, buckets,
-		&metaOptions{
-			logger: c.l,
-			action: "admin:ListBuckets",
+	return SendXMLResponse(ctx,
+		s3response.ListBucketsResult{
+			Buckets: buckets,
+		}, err, &MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListBuckets,
 		})
 }
-
-type metaOptions struct {
-	action string
-	status int
-	logger s3log.AuditLogger
-}
-
-func sendResponse(ctx *fiber.Ctx, err error, data any, m *metaOptions) error {
-	status := m.status
-	if err != nil {
-		if status == 0 {
-			status = fiber.StatusInternalServerError
-		}
-		if m.logger != nil {
-			m.logger.Log(ctx, err, []byte(err.Error()), s3log.LogMeta{
-				Action:     m.action,
-				HttpStatus: status,
-			})
-		}
-
-		return ctx.Status(status).SendString(err.Error())
-	}
-
-	if status == 0 {
-		status = fiber.StatusOK
-	}
-
-	msg, ok := data.(string)
-	if ok {
-		if m.logger != nil {
-			m.logger.Log(ctx, nil, []byte(msg), s3log.LogMeta{
-				Action:     m.action,
-				HttpStatus: status,
-			})
-		}
-
-		return ctx.Status(status).SendString(msg)
-	}
-
-	dataJSON, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-
-	if m.logger != nil {
-		m.logger.Log(ctx, nil, dataJSON, s3log.LogMeta{
-			HttpStatus: status,
-			Action:     m.action,
-		})
-	}
-
-	ctx.Set(fiber.HeaderContentType, fiber.MIMEApplicationJSON)
-
-	return ctx.Status(status).Send(dataJSON)
-}

s3api/controllers/admin_test.go

@@ -15,12 +15,11 @@
 package controllers
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
@@ -43,67 +42,60 @@ func TestAdminController_CreateUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/create-user", adminController.CreateUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	usr := auth.Account{
-		Access: "access",
-		Secret: "secret",
-		Role:   "invalid role",
-	}
-
-	user, _ := json.Marshal(&usr)
-
-	usr.Role = "admin"
-
-	succUsr, _ := json.Marshal(&usr)
-
-	appErr.Patch("/create-user", adminController.CreateUser)
+	succUser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>admin</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+	invuser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>invalid_role</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
-			name: "Admin-create-user-success",
+			name: "Admin-create-user-malformed-body",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
-			},
-			wantErr:    false,
-			statusCode: 201,
-		},
-		{
-			name: "Admin-create-user-invalid-user-role",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
 			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
-			app:  appErr,
+			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
 			},
 			wantErr:    false,
-			statusCode: 403,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-create-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
+			},
+			wantErr:    false,
+			statusCode: 201,
 		},
 	}
 	for _, tt := range tests {
@@ -134,24 +126,8 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/update-user", adminController.UpdateUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appErr.Patch("/update-user", adminController.UpdateUser)
-
-	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
-
 	adminControllerErr := AdminController{
 		iam: &IAMServiceMock{
 			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
@@ -162,25 +138,28 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	appNotFound := fiber.New()
 
-	appNotFound.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
 
+	succUser := `
+		<Account>
+			<Secret>secret</Secret>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Admin-update-user-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
 			statusCode: 200,
@@ -189,10 +168,10 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			name: "Admin-update-user-missing-access",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
-			statusCode: 400,
+			statusCode: 404,
 		},
 		{
 			name: "Admin-update-user-invalid-request-body",
@@ -203,20 +182,11 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "Admin-update-user-invalid-requester-role",
-			app:  appErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Admin-update-user-not-found",
 			app:  appNotFound,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
 			statusCode: 404,
@@ -250,28 +220,14 @@ func TestAdminController_DeleteUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/delete-user", adminController.DeleteUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appErr.Patch("/delete-user", adminController.DeleteUser)
-
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Admin-delete-user-success",
@@ -282,15 +238,6 @@ func TestAdminController_DeleteUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
-		{
-			name: "Admin-delete-user-invalid-requester-role",
-			app:  appErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -327,48 +274,18 @@ func TestAdminController_ListUsers(t *testing.T) {
 	}
 
 	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appErr.Patch("/list-users", adminControllerErr.ListUsers)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-users", adminController.ListUsers)
-
 	appSucc := fiber.New()
-
-	appSucc.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
-		{
-			name: "Admin-list-users-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Admin-list-users-iam-error",
 			app:  appErr,
@@ -435,57 +352,21 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
-
 	appIamErr := fiber.New()
-
-	appIamErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
 
 	appIamNoSuchUser := fiber.New()
-
-	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
-		{
-			name: "Change-bucket-owner-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
 			app:  appIamErr,
@@ -540,39 +421,15 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
-
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
-		{
-			name: "List-buckets-incorrect-role",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "List-buckets-success",
 			app:  app,

s3api/controllers/backend_moq_test.go

@@ -41,7 +41,7 @@ var _ backend.Backend = &BackendMock{}
 //			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 //				panic("mock out the CreateMultipartUpload method")
 //			},
-//			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
+//			DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucket method")
 //			},
 //			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
@@ -53,7 +53,7 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketTagging method")
 //			},
-//			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
+//			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
 //				panic("mock out the DeleteObject method")
 //			},
 //			DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error {
@@ -74,7 +74,7 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 //				panic("mock out the GetBucketTagging method")
 //			},
-//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
 //				panic("mock out the GetBucketVersioning method")
 //			},
 //			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
@@ -83,7 +83,7 @@ var _ backend.Backend = &BackendMock{}
 //			GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 //				panic("mock out the GetObjectAcl method")
 //			},
-//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 //				panic("mock out the GetObjectAttributes method")
 //			},
 //			GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error) {
@@ -104,7 +104,7 @@ var _ backend.Backend = &BackendMock{}
 //			HeadObjectFunc: func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 //				panic("mock out the HeadObject method")
 //			},
-//			ListBucketsFunc: func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+//			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 //				panic("mock out the ListBuckets method")
 //			},
 //			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
@@ -113,7 +113,7 @@ var _ backend.Backend = &BackendMock{}
 //			ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 //				panic("mock out the ListMultipartUploads method")
 //			},
-//			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+//			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
 //				panic("mock out the ListObjectVersions method")
 //			},
 //			ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
@@ -137,10 +137,10 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
 //				panic("mock out the PutBucketTagging method")
 //			},
-//			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+//			PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 //				panic("mock out the PutBucketVersioning method")
 //			},
-//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
+//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
 //				panic("mock out the PutObject method")
 //			},
 //			PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error {
@@ -202,7 +202,7 @@ type BackendMock struct {
 	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 
 	// DeleteBucketFunc mocks the DeleteBucket method.
-	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
+	DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error
 
 	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
 	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
@@ -214,7 +214,7 @@ type BackendMock struct {
 	DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
 
 	// DeleteObjectFunc mocks the DeleteObject method.
-	DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error
+	DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
 
 	// DeleteObjectTaggingFunc mocks the DeleteObjectTagging method.
 	DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error
@@ -235,7 +235,7 @@ type BackendMock struct {
 	GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
 
 	// GetBucketVersioningFunc mocks the GetBucketVersioning method.
-	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
 
 	// GetObjectFunc mocks the GetObject method.
 	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
@@ -244,7 +244,7 @@ type BackendMock struct {
 	GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 
 	// GetObjectAttributesFunc mocks the GetObjectAttributes method.
-	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
+	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
 
 	// GetObjectLegalHoldFunc mocks the GetObjectLegalHold method.
 	GetObjectLegalHoldFunc func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error)
@@ -265,7 +265,7 @@ type BackendMock struct {
 	HeadObjectFunc func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 
 	// ListBucketsFunc mocks the ListBuckets method.
-	ListBucketsFunc func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	ListBucketsFunc func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
 
 	// ListBucketsAndOwnersFunc mocks the ListBucketsAndOwners method.
 	ListBucketsAndOwnersFunc func(contextMoqParam context.Context) ([]s3response.Bucket, error)
@@ -274,7 +274,7 @@ type BackendMock struct {
 	ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 
 	// ListObjectVersionsFunc mocks the ListObjectVersions method.
-	ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
+	ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
 
 	// ListObjectsFunc mocks the ListObjects method.
 	ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
@@ -298,10 +298,10 @@ type BackendMock struct {
 	PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error
 
 	// PutBucketVersioningFunc mocks the PutBucketVersioning method.
-	PutBucketVersioningFunc func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error
+	PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error
 
 	// PutObjectFunc mocks the PutObject method.
-	PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error)
+	PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error)
 
 	// PutObjectAclFunc mocks the PutObjectAcl method.
 	PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error
@@ -388,8 +388,8 @@ type BackendMock struct {
 		DeleteBucket []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// DeleteBucketInput is the deleteBucketInput argument value.
-			DeleteBucketInput *s3.DeleteBucketInput
+			// Bucket is the bucket argument value.
+			Bucket string
 		}
 		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
 		DeleteBucketOwnershipControls []struct {
@@ -547,10 +547,8 @@ type BackendMock struct {
 		ListBuckets []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// Owner is the owner argument value.
-			Owner string
-			// IsAdmin is the isAdmin argument value.
-			IsAdmin bool
+			// ListBucketsInput is the listBucketsInput argument value.
+			ListBucketsInput s3response.ListBucketsInput
 		}
 		// ListBucketsAndOwners holds details about calls to the ListBucketsAndOwners method.
 		ListBucketsAndOwners []struct {
@@ -632,8 +630,10 @@ type BackendMock struct {
 		PutBucketVersioning []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// PutBucketVersioningInput is the putBucketVersioningInput argument value.
-			PutBucketVersioningInput *s3.PutBucketVersioningInput
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Status is the status argument value.
+			Status types.BucketVersioningStatus
 		}
 		// PutObject holds details about calls to the PutObject method.
 		PutObject []struct {
@@ -1010,21 +1010,21 @@ func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
 }
 
 // DeleteBucket calls DeleteBucketFunc.
-func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
+func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketFunc == nil {
 		panic("BackendMock.DeleteBucketFunc: method is nil but Backend.DeleteBucket was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam   context.Context
-		DeleteBucketInput *s3.DeleteBucketInput
+		ContextMoqParam context.Context
+		Bucket          string
 	}{
-		ContextMoqParam:   contextMoqParam,
-		DeleteBucketInput: deleteBucketInput,
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
 	}
 	mock.lockDeleteBucket.Lock()
 	mock.calls.DeleteBucket = append(mock.calls.DeleteBucket, callInfo)
 	mock.lockDeleteBucket.Unlock()
-	return mock.DeleteBucketFunc(contextMoqParam, deleteBucketInput)
+	return mock.DeleteBucketFunc(contextMoqParam, bucket)
 }
 
 // DeleteBucketCalls gets all the calls that were made to DeleteBucket.
@@ -1032,12 +1032,12 @@ func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBuc
 //
 //	len(mockedBackend.DeleteBucketCalls())
 func (mock *BackendMock) DeleteBucketCalls() []struct {
-	ContextMoqParam   context.Context
-	DeleteBucketInput *s3.DeleteBucketInput
+	ContextMoqParam context.Context
+	Bucket          string
 } {
 	var calls []struct {
-		ContextMoqParam   context.Context
-		DeleteBucketInput *s3.DeleteBucketInput
+		ContextMoqParam context.Context
+		Bucket          string
 	}
 	mock.lockDeleteBucket.RLock()
 	calls = mock.calls.DeleteBucket
@@ -1154,7 +1154,7 @@ func (mock *BackendMock) DeleteBucketTaggingCalls() []struct {
 }
 
 // DeleteObject calls DeleteObjectFunc.
-func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
+func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
 	if mock.DeleteObjectFunc == nil {
 		panic("BackendMock.DeleteObjectFunc: method is nil but Backend.DeleteObject was just called")
 	}
@@ -1410,7 +1410,7 @@ func (mock *BackendMock) GetBucketTaggingCalls() []struct {
 }
 
 // GetBucketVersioning calls GetBucketVersioningFunc.
-func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
 	if mock.GetBucketVersioningFunc == nil {
 		panic("BackendMock.GetBucketVersioningFunc: method is nil but Backend.GetBucketVersioning was just called")
 	}
@@ -1518,7 +1518,7 @@ func (mock *BackendMock) GetObjectAclCalls() []struct {
 }
 
 // GetObjectAttributes calls GetObjectAttributesFunc.
-func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	if mock.GetObjectAttributesFunc == nil {
 		panic("BackendMock.GetObjectAttributesFunc: method is nil but Backend.GetObjectAttributes was just called")
 	}
@@ -1790,23 +1790,21 @@ func (mock *BackendMock) HeadObjectCalls() []struct {
 }
 
 // ListBuckets calls ListBucketsFunc.
-func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 	if mock.ListBucketsFunc == nil {
 		panic("BackendMock.ListBucketsFunc: method is nil but Backend.ListBuckets was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam context.Context
-		Owner           string
-		IsAdmin         bool
+		ContextMoqParam  context.Context
+		ListBucketsInput s3response.ListBucketsInput
 	}{
-		ContextMoqParam: contextMoqParam,
-		Owner:           owner,
-		IsAdmin:         isAdmin,
+		ContextMoqParam:  contextMoqParam,
+		ListBucketsInput: listBucketsInput,
 	}
 	mock.lockListBuckets.Lock()
 	mock.calls.ListBuckets = append(mock.calls.ListBuckets, callInfo)
 	mock.lockListBuckets.Unlock()
-	return mock.ListBucketsFunc(contextMoqParam, owner, isAdmin)
+	return mock.ListBucketsFunc(contextMoqParam, listBucketsInput)
 }
 
 // ListBucketsCalls gets all the calls that were made to ListBuckets.
@@ -1814,14 +1812,12 @@ func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner stri
 //
 //	len(mockedBackend.ListBucketsCalls())
 func (mock *BackendMock) ListBucketsCalls() []struct {
-	ContextMoqParam context.Context
-	Owner           string
-	IsAdmin         bool
+	ContextMoqParam  context.Context
+	ListBucketsInput s3response.ListBucketsInput
 } {
 	var calls []struct {
-		ContextMoqParam context.Context
-		Owner           string
-		IsAdmin         bool
+		ContextMoqParam  context.Context
+		ListBucketsInput s3response.ListBucketsInput
 	}
 	mock.lockListBuckets.RLock()
 	calls = mock.calls.ListBuckets
@@ -1898,7 +1894,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct {
 }
 
 // ListObjectVersions calls ListObjectVersionsFunc.
-func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
 	if mock.ListObjectVersionsFunc == nil {
 		panic("BackendMock.ListObjectVersionsFunc: method is nil but Backend.ListObjectVersions was just called")
 	}
@@ -2202,21 +2198,23 @@ func (mock *BackendMock) PutBucketTaggingCalls() []struct {
 }
 
 // PutBucketVersioning calls PutBucketVersioningFunc.
-func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 	if mock.PutBucketVersioningFunc == nil {
 		panic("BackendMock.PutBucketVersioningFunc: method is nil but Backend.PutBucketVersioning was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam          context.Context
-		PutBucketVersioningInput *s3.PutBucketVersioningInput
+		ContextMoqParam context.Context
+		Bucket          string
+		Status          types.BucketVersioningStatus
 	}{
-		ContextMoqParam:          contextMoqParam,
-		PutBucketVersioningInput: putBucketVersioningInput,
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Status:          status,
 	}
 	mock.lockPutBucketVersioning.Lock()
 	mock.calls.PutBucketVersioning = append(mock.calls.PutBucketVersioning, callInfo)
 	mock.lockPutBucketVersioning.Unlock()
-	return mock.PutBucketVersioningFunc(contextMoqParam, putBucketVersioningInput)
+	return mock.PutBucketVersioningFunc(contextMoqParam, bucket, status)
 }
 
 // PutBucketVersioningCalls gets all the calls that were made to PutBucketVersioning.
@@ -2224,12 +2222,14 @@ func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, pu
 //
 //	len(mockedBackend.PutBucketVersioningCalls())
 func (mock *BackendMock) PutBucketVersioningCalls() []struct {
-	ContextMoqParam          context.Context
-	PutBucketVersioningInput *s3.PutBucketVersioningInput
+	ContextMoqParam context.Context
+	Bucket          string
+	Status          types.BucketVersioningStatus
 } {
 	var calls []struct {
-		ContextMoqParam          context.Context
-		PutBucketVersioningInput *s3.PutBucketVersioningInput
+		ContextMoqParam context.Context
+		Bucket          string
+		Status          types.BucketVersioningStatus
 	}
 	mock.lockPutBucketVersioning.RLock()
 	calls = mock.calls.PutBucketVersioning
@@ -2238,7 +2238,7 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct {
 }
 
 // PutObject calls PutObjectFunc.
-func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
+func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
 	if mock.PutObjectFunc == nil {
 		panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called")
 	}

s3api/controllers/base.go

@@ -51,8 +51,9 @@ type S3ApiController struct {
 }
 
 const (
-	iso8601Format      = "20060102T150405Z"
-	defaultContentType = "binary/octet-stream"
+	iso8601Format             = "20060102T150405Z"
+	iso8601TimeFormatExtended = "Mon Jan _2 15:04:05 2006"
+	defaultContentType        = "binary/octet-stream"
 )
 
 func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) S3ApiController {
@@ -68,8 +69,32 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs
 }
 
 func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
+	cToken := ctx.Query("continuation-token")
+	prefix := ctx.Query("prefix")
+	maxBucketsStr := ctx.Query("max-buckets")
 	acct := ctx.Locals("account").(auth.Account)
-	res, err := c.be.ListBuckets(ctx.Context(), acct.Access, acct.Role == "admin")
+
+	maxBuckets, err := utils.ParseUint(maxBucketsStr)
+	if err != nil || maxBuckets > 10000 {
+		if c.debug {
+			log.Printf("error parsing max-buckets %q: %v\n", maxBucketsStr, err)
+		}
+		return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
+			&MetaOpts{
+				Logger:     c.logger,
+				MetricsMng: c.mm,
+				Action:     metrics.ActionListAllMyBuckets,
+			})
+	}
+
+	res, err := c.be.ListBuckets(ctx.Context(),
+		s3response.ListBucketsInput{
+			Owner:             acct.Access,
+			IsAdmin:           acct.Role == auth.RoleAdmin,
+			MaxBuckets:        maxBuckets,
+			ContinuationToken: cToken,
+			Prefix:            prefix,
+		})
 	return SendXMLResponse(ctx, res, err,
 		&MetaOpts{
 			Logger:     c.logger,
@@ -83,7 +108,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	maxParts := int32(ctx.QueryInt("max-parts", -1))
 	partNumberMarker := ctx.Query("part-number-marker")
 	acceptRange := ctx.Get("Range")
 	acct := ctx.Locals("account").(auth.Account)
@@ -221,16 +245,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if uploadId != "" {
-		if maxParts < 0 && ctx.Request().URI().QueryArgs().Has("max-parts") {
-			return SendResponse(ctx,
-				s3err.GetAPIError(s3err.ErrInvalidMaxParts),
-				&MetaOpts{
-					Logger:      c.logger,
-					MetricsMng:  c.mm,
-					Action:      metrics.ActionListParts,
-					BucketOwner: parsedAcl.Owner,
-				})
-		}
 		if partNumberMarker != "" {
 			n, err := strconv.Atoi(partNumberMarker)
 			if err != nil || n < 0 {
@@ -248,8 +262,24 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					})
 			}
 		}
+		mxParts := ctx.Query("max-parts")
+		maxParts, err := utils.ParseUint(mxParts)
+		if err != nil {
+			if c.debug {
+				log.Printf("error parsing max parts %q: %v",
+					mxParts, err)
+			}
+			return SendResponse(ctx,
+				s3err.GetAPIError(s3err.ErrInvalidMaxParts),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionListParts,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
 
-		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
 			AclPermission: types.PermissionRead,
@@ -268,17 +298,13 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
-		var mxParts *int32
-		if ctx.Request().URI().QueryArgs().Has("max-parts") {
-			mxParts = &maxParts
-		}
 
 		res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
 			Bucket:           &bucket,
 			Key:              &key,
 			UploadId:         &uploadId,
 			PartNumberMarker: &partNumberMarker,
-			MaxParts:         mxParts,
+			MaxParts:         &maxParts,
 		})
 		return SendXMLResponse(ctx, res, err,
 			&MetaOpts{
@@ -346,6 +372,19 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker")
 		maxPartsParsed, err := utils.ParseUint(maxParts)
 		if err != nil {
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxParts),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionGetObjectAttributes,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+		attrs, err := utils.ParseObjectAttributes(ctx)
+		if err != nil {
+			if c.debug {
+				log.Printf("error parsing object attributes: %v", err)
+			}
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
 					Logger:      c.logger,
@@ -354,7 +393,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
-		attrs := utils.ParseObjectAttributes(ctx)
 
 		res, err := c.be.GetObjectAttributes(ctx.Context(),
 			&s3.GetObjectAttributesInput{
@@ -365,6 +403,22 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 				VersionId:        &versionId,
 			})
 		if err != nil {
+			hdrs := []utils.CustomHeader{}
+
+			if res.DeleteMarker != nil {
+				hdrs = append(hdrs, utils.CustomHeader{
+					Key:   "x-amz-delete-marker",
+					Value: "true",
+				})
+			}
+			if getstring(res.VersionId) != "" {
+				hdrs = append(hdrs, utils.CustomHeader{
+					Key:   "x-amz-version-id",
+					Value: getstring(res.VersionId),
+				})
+			}
+
+			utils.SetResponseHeaders(ctx, hdrs)
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
 					Logger:      c.logger,
@@ -373,7 +427,31 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
-		return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), err,
+
+		hdrs := []utils.CustomHeader{}
+
+		if getstring(res.VersionId) != "" {
+			hdrs = append(hdrs, utils.CustomHeader{
+				Key:   "x-amz-version-id",
+				Value: getstring(res.VersionId),
+			})
+		}
+		if res.DeleteMarker != nil && *res.DeleteMarker {
+			hdrs = append(hdrs, utils.CustomHeader{
+				Key:   "x-amz-delete-marker",
+				Value: "true",
+			})
+		}
+		if res.LastModified != nil {
+			hdrs = append(hdrs, utils.CustomHeader{
+				Key:   "Last-Modified",
+				Value: res.LastModified.UTC().Format(iso8601TimeFormatExtended),
+			})
+		}
+
+		utils.SetResponseHeaders(ctx, hdrs)
+
+		return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), nil,
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
@@ -382,6 +460,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	action := auth.GetObjectAction
+	if versionId != "" {
+		action = auth.GetObjectVersionAction
+	}
+
 	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 		Readonly:      c.readonly,
 		Acl:           parsedAcl,
@@ -390,7 +473,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		Acc:           acct,
 		Bucket:        bucket,
 		Object:        key,
-		Action:        auth.GetObjectAction,
+		Action:        action,
 	})
 	if err != nil {
 		return SendResponse(ctx, err,
@@ -410,11 +493,23 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		VersionId: &versionId,
 	})
 	if err != nil {
+		if res != nil {
+			utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+				{
+					Key:   "x-amz-delete-marker",
+					Value: "true",
+				},
+				{
+					Key:   "Last-Modified",
+					Value: res.LastModified.Format(timefmt),
+				},
+			})
+		}
 		return SendResponse(ctx, err,
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
-				Action:      metrics.ActionGetObject,
+				Action:      metrics.ActionHeadObject,
 				BucketOwner: parsedAcl.Owner,
 			})
 	}
@@ -478,6 +573,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	utils.SetMetaHeaders(ctx, res.Metadata)
 	// Set other response headers
 	utils.SetResponseHeaders(ctx, hdrs)
+	// Set version id header
+	if getstring(res.VersionId) != "" {
+		utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+			{
+				Key:   "x-amz-version-id",
+				Value: getstring(res.VersionId),
+			},
+		})
+	}
 
 	status := http.StatusOK
 	if acceptRange != "" {
@@ -710,7 +814,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max keys %q: %v",
 					maxkeysStr, err)
 			}
-			return SendXMLResponse(ctx, nil, err,
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -843,12 +947,13 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max uploads %q: %v",
 					maxUploadsStr, err)
 			}
-			return SendXMLResponse(ctx, nil, err, &MetaOpts{
-				Logger:      c.logger,
-				MetricsMng:  c.mm,
-				Action:      metrics.ActionListMultipartUploads,
-				BucketOwner: parsedAcl.Owner,
-			})
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxUploads),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionListMultipartUploads,
+					BucketOwner: parsedAcl.Owner,
+				})
 		}
 		res, err := c.be.ListMultipartUploads(ctx.Context(),
 			&s3.ListMultipartUploadsInput{
@@ -893,7 +998,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				log.Printf("error parsing max keys %q: %v",
 					maxkeysStr, err)
 			}
-			return SendXMLResponse(ctx, nil, err,
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -944,7 +1049,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			log.Printf("error parsing max keys %q: %v",
 				maxkeysStr, err)
 		}
-		return SendXMLResponse(ctx, nil, err,
+		return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys),
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
@@ -981,8 +1086,8 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	objectOwnership := types.ObjectOwnership(
 		ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)),
 	)
-	mfa := ctx.Get("X-Amz-Mfa")
-	contentMD5 := ctx.Get("Content-MD5")
+	// mfa := ctx.Get("X-Amz-Mfa")
+	// contentMD5 := ctx.Get("Content-MD5")
 	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 
@@ -1045,6 +1150,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				MetricsMng:  c.mm,
 				Action:      metrics.ActionPutBucketTagging,
 				BucketOwner: parsedAcl.Owner,
+				Status:      http.StatusNoContent,
 			})
 	}
 
@@ -1136,13 +1242,21 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err = c.be.PutBucketVersioning(ctx.Context(),
-			&s3.PutBucketVersioningInput{
-				Bucket:                  &bucket,
-				MFA:                     &mfa,
-				VersioningConfiguration: &versioningConf,
-				ContentMD5:              &contentMD5,
-			})
+		if versioningConf.Status != types.BucketVersioningStatusEnabled &&
+			versioningConf.Status != types.BucketVersioningStatusSuspended {
+			if c.debug {
+				log.Printf("invalid versioning configuration status: %v\n", versioningConf.Status)
+			}
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketVersioning,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status)
 		return SendResponse(ctx, err,
 			&MetaOpts{
 				Logger:      c.logger,
@@ -1530,6 +1644,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	versionId := ctx.Query("versionId")
 	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
+	contentType := ctx.Get("Content-Type")
+	contentEncoding := ctx.Get("Content-Encoding")
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	tagging := ctx.Get("x-amz-tagging")
 
@@ -1814,6 +1930,14 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				ExpectedBucketOwner: &bucketOwner,
 				CopySourceRange:     &copySrcRange,
 			})
+		if err == nil && resp.CopySourceVersionId != "" {
+			utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+				{
+					Key:   "x-amz-copy-source-version-id",
+					Value: resp.CopySourceVersionId,
+				},
+			})
+		}
 		return SendXMLResponse(ctx, resp, err,
 			&MetaOpts{
 				Logger:      c.logger,
@@ -2139,6 +2263,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				StorageClass:                types.StorageClass(storageClass),
 			})
 		if err == nil {
+			hdrs := []utils.CustomHeader{}
+			if getstring(res.VersionId) != "" {
+				hdrs = append(hdrs, utils.CustomHeader{
+					Key:   "x-amz-version-id",
+					Value: getstring(res.VersionId),
+				})
+			}
+			if getstring(res.CopySourceVersionId) != "" {
+				hdrs = append(hdrs, utils.CustomHeader{
+					Key:   "x-amz-copy-source-version-id",
+					Value: getstring(res.CopySourceVersionId),
+				})
+			}
+			utils.SetResponseHeaders(ctx, hdrs)
+
 			return SendXMLResponse(ctx, res.CopyObjectResult, err,
 				&MetaOpts{
 					Logger:      c.logger,
@@ -2184,7 +2323,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{keyStart}, true, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &keyStart}}, true, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2230,11 +2369,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	ctx.Locals("logReqBody", false)
-	etag, err := c.be.PutObject(ctx.Context(),
+	res, err := c.be.PutObject(ctx.Context(),
 		&s3.PutObjectInput{
 			Bucket:                    &bucket,
 			Key:                       &keyStart,
 			ContentLength:             &contentLength,
+			ContentType:               &contentType,
+			ContentEncoding:           &contentEncoding,
 			Metadata:                  metadata,
 			Body:                      body,
 			Tagging:                   &tagging,
@@ -2242,8 +2383,36 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			ObjectLockMode:            objLock.ObjectLockMode,
 			ObjectLockLegalHoldStatus: objLock.LegalHoldStatus,
 		})
-	ctx.Response().Header.Set("ETag", etag)
-	return SendResponse(ctx, err,
+	if err != nil {
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:        c.logger,
+				MetricsMng:    c.mm,
+				ContentLength: contentLength,
+				EvSender:      c.evSender,
+				Action:        metrics.ActionPutObject,
+				BucketOwner:   parsedAcl.Owner,
+				ObjectSize:    contentLength,
+				EventName:     s3event.EventObjectCreatedPut,
+			})
+	}
+	hdrs := []utils.CustomHeader{
+		{
+			Key:   "ETag",
+			Value: res.ETag,
+		},
+	}
+
+	if res.VersionID != "" {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-version-id",
+			Value: res.VersionID,
+		})
+	}
+
+	utils.SetResponseHeaders(ctx, hdrs)
+
+	return SendResponse(ctx, nil,
 		&MetaOpts{
 			Logger:        c.logger,
 			MetricsMng:    c.mm,
@@ -2251,7 +2420,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			EvSender:      c.evSender,
 			Action:        metrics.ActionPutObject,
 			BucketOwner:   parsedAcl.Owner,
-			ObjectETag:    &etag,
+			ObjectETag:    &res.ETag,
 			ObjectSize:    contentLength,
 			EventName:     s3event.EventObjectCreatedPut,
 		})
@@ -2379,10 +2548,7 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = c.be.DeleteBucket(ctx.Context(),
-		&s3.DeleteBucketInput{
-			Bucket: &bucket,
-		})
+	err = c.be.DeleteBucket(ctx.Context(), bucket)
 	return SendResponse(ctx, err,
 		&MetaOpts{
 			Logger:      c.logger,
@@ -2438,7 +2604,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 	// The AWS CLI sends 'True', while Go SDK sends 'true'
 	bypass := strings.EqualFold(bypassHdr, "true")
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, utils.ParseDeleteObjects(dObj.Objects), bypass, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2565,6 +2731,8 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	//TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object
+
 	err := auth.VerifyAccess(ctx.Context(), c.be,
 		auth.AccessOptions{
 			Readonly:      c.readonly,
@@ -2589,7 +2757,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	// The AWS CLI sends 'True', while Go SDK sends 'true'
 	bypass := strings.EqualFold(bypassHdr, "true")
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{key}, bypass, c.be)
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key, VersionId: &versionId}}, bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2600,13 +2768,42 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = c.be.DeleteObject(ctx.Context(),
+	res, err := c.be.DeleteObject(ctx.Context(),
 		&s3.DeleteObjectInput{
 			Bucket:    &bucket,
 			Key:       &key,
 			VersionId: &versionId,
 		})
-	return SendResponse(ctx, err,
+	if err != nil {
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				EvSender:    c.evSender,
+				Action:      metrics.ActionDeleteObject,
+				BucketOwner: parsedAcl.Owner,
+				EventName:   s3event.EventObjectRemovedDelete,
+				Status:      http.StatusNoContent,
+			})
+	}
+
+	hdrs := []utils.CustomHeader{}
+	if res.VersionId != nil && *res.VersionId != "" {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-version-id",
+			Value: *res.VersionId,
+		})
+	}
+	if res.DeleteMarker != nil && *res.DeleteMarker {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-delete-marker",
+			Value: "true",
+		})
+	}
+
+	utils.SetResponseHeaders(ctx, hdrs)
+
+	return SendResponse(ctx, nil,
 		&MetaOpts{
 			Logger:      c.logger,
 			MetricsMng:  c.mm,
@@ -2679,6 +2876,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	partNumberQuery := int32(ctx.QueryInt("partNumber", -1))
+	versionId := ctx.Query("versionId")
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	if keyEnd != "" {
@@ -2733,18 +2931,22 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 			Bucket:     &bucket,
 			Key:        &key,
 			PartNumber: partNumber,
+			VersionId:  &versionId,
 		})
 	if err != nil {
-		return SendResponse(ctx, err,
-			&MetaOpts{
-				Logger:      c.logger,
-				MetricsMng:  c.mm,
-				Action:      metrics.ActionHeadObject,
-				BucketOwner: parsedAcl.Owner,
+		if res != nil {
+			utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+				{
+					Key:   "x-amz-delete-marker",
+					Value: "true",
+				},
+				{
+					Key:   "Last-Modified",
+					Value: res.LastModified.Format(timefmt),
+				},
 			})
-	}
-	if res == nil {
-		return SendResponse(ctx, fmt.Errorf("head object nil response"),
+		}
+		return SendResponse(ctx, err,
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
@@ -2822,6 +3024,13 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 		Value: contentType,
 	})
 
+	if getstring(res.VersionId) != "" {
+		headers = append(headers, utils.CustomHeader{
+			Key:   "x-amz-version-id",
+			Value: getstring(res.VersionId),
+		})
+	}
+
 	utils.SetResponseHeaders(ctx, headers)
 
 	return SendResponse(ctx, nil,
@@ -2842,6 +3051,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	contentType := ctx.Get("Content-Type")
+	contentEncoding := ctx.Get("Content-Encoding")
 	tagging := ctx.Get("X-Amz-Tagging")
 
 	if keyEnd != "" {
@@ -3010,6 +3220,14 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				},
 			})
 		if err == nil {
+			if getstring(res.VersionId) != "" {
+				utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+					{
+						Key:   "x-amz-version-id",
+						Value: getstring(res.VersionId),
+					},
+				})
+			}
 			return SendXMLResponse(ctx, res, err,
 				&MetaOpts{
 					Logger:      c.logger,
@@ -3071,6 +3289,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			Key:                       &key,
 			Tagging:                   &tagging,
 			ContentType:               &contentType,
+			ContentEncoding:           &contentEncoding,
 			ObjectLockRetainUntilDate: &objLockState.RetainUntilDate,
 			ObjectLockMode:            objLockState.ObjectLockMode,
 			ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,
@@ -3089,14 +3308,14 @@ type MetaOpts struct {
 	Logger        s3log.AuditLogger
 	EvSender      s3event.S3EventSender
 	MetricsMng    *metrics.Manager
-	ContentLength int64
-	Action        string
-	BucketOwner   string
-	ObjectSize    int64
-	ObjectCount   int64
-	EventName     s3event.EventType
 	ObjectETag    *string
 	VersionId     *string
+	Action        string
+	BucketOwner   string
+	EventName     s3event.EventType
+	ContentLength int64
+	ObjectSize    int64
+	ObjectCount   int64
 	Status        int
 }
 

s3api/controllers/base_test.go

@@ -92,7 +92,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, nil
 			},
 		},
@@ -109,7 +109,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr := fiber.New()
 	s3ApiControllerErr := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
 			},
 		},
@@ -123,11 +123,11 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
 
 	tests := []struct {
-		name       string
 		args       args
 		app        *fiber.App
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "List-bucket-method-not-allowed",
@@ -187,8 +187,8 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 				return &s3.GetObjectAclOutput{}, nil
 			},
-			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
-				return s3response.GetObjectAttributesResult{}, nil
+			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+				return s3response.GetObjectAttributesResponse{}, nil
 			},
 			GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
@@ -233,11 +233,11 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Get-actions-get-tags-success",
@@ -382,11 +382,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return map[string]string{}, nil
 			},
-			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-				return &s3.GetBucketVersioningOutput{}, nil
+			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+				return s3response.GetBucketVersioningOutput{}, nil
 			},
-			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
-				return &s3.ListObjectVersionsOutput{}, nil
+			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+				return s3response.ListVersionsResult{}, nil
 			},
 			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 				return []byte{}, nil
@@ -435,11 +435,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Get-bucket-tagging-non-existing-bucket",
@@ -677,7 +677,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
 				return nil
 			},
-			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+			PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 				return nil
 			},
 			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
@@ -728,11 +728,11 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Put-bucket-tagging-invalid-body",
@@ -750,7 +750,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 		{
 			name: "Put-bucket-ownership-controls-invalid-ownership",
@@ -968,8 +968,8 @@ func TestS3ApiController_PutActions(t *testing.T) {
 					CopyObjectResult: &types.CopyObjectResult{},
 				}, nil
 			},
-			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
-				return "ETag", nil
+			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+				return s3response.PutObjectOutput{}, nil
 			},
 			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
 				return "hello", nil
@@ -1034,11 +1034,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Put-object-part-error-case",
@@ -1217,7 +1217,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
+			DeleteBucketFunc: func(_ context.Context, bucket string) error {
 				return nil
 			},
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
@@ -1243,11 +1243,11 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app.Delete("/:bucket", s3ApiController.DeleteBucket)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Delete-bucket-success",
@@ -1334,11 +1334,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	request.Header.Set("Content-Type", "application/xml")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Delete-Objects-success",
@@ -1383,8 +1383,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
-				return nil
+			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+				return &s3.DeleteObjectOutput{}, nil
 			},
 			AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
 				return nil
@@ -1414,8 +1414,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 		GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 			return acldata, nil
 		},
-		DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
-			return s3err.GetAPIError(7)
+		DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		},
 		GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 			return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
@@ -1432,11 +1432,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Abort-multipart-upload-success",
@@ -1541,11 +1541,11 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Head-bucket-success",
@@ -1628,7 +1628,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 				return acldata, nil
 			},
 			HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-				return nil, s3err.GetAPIError(42)
+				return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 			},
 		},
 	}
@@ -1643,11 +1643,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Head-object-success",
@@ -1723,11 +1723,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Restore-object-success",
@@ -1808,10 +1808,10 @@ func Test_XMLresponse(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		name       string
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Internal-server-error",
@@ -1883,10 +1883,10 @@ func Test_response(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		name       string
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Internal-server-error",

s3api/middlewares/admin.go

@@ -0,0 +1,59 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		acct := ctx.Locals("account").(auth.Account)
+		if acct.Role != auth.RoleAdmin {
+			path := ctx.Path()
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
+				&controllers.MetaOpts{
+					Logger: logger,
+					Action: detectAction(path),
+				})
+		}
+
+		return ctx.Next()
+	}
+}
+
+func detectAction(path string) (action string) {
+	if strings.Contains(path, "create-user") {
+		action = metrics.ActionAdminCreateUser
+	} else if strings.Contains(path, "update-user") {
+		action = metrics.ActionAdminUpdateUser
+	} else if strings.Contains(path, "delete-user") {
+		action = metrics.ActionAdminDeleteUser
+	} else if strings.Contains(path, "list-user") {
+		action = metrics.ActionAdminListUsers
+	} else if strings.Contains(path, "list-buckets") {
+		action = metrics.ActionAdminListBuckets
+	} else if strings.Contains(path, "change-bucket-owner") {
+		action = metrics.ActionAdminChangeBucketOwner
+	}
+	return action
+}

s3api/middlewares/authentication.go

@@ -149,8 +149,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 }
 
 type accounts struct {
-	root RootUserConfig
 	iam  auth.IAMService
+	root RootUserConfig
 }
 
 func (a accounts) getAccount(access string) (auth.Account, error) {

s3api/middlewares/url-decoder.go

@@ -26,12 +26,11 @@ import (
 
 func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		reqURL := ctx.Request().URI().String()
-		decoded, err := url.Parse(reqURL)
+		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
 		if err != nil {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}
-		ctx.Path(decoded.Path)
+		ctx.Path(unescp)
 		return ctx.Next()
 	}
 }

s3api/router.go

@@ -20,6 +20,7 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
 )
@@ -35,22 +36,22 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		adminController := controllers.NewAdminController(iam, be, aLogger)
 
 		// CreateUser admin api
-		app.Patch("/create-user", adminController.CreateUser)
+		app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
 
 		// DeleteUsers admin api
-		app.Patch("/delete-user", adminController.DeleteUser)
+		app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
 
 		// UpdateUser admin api
-		app.Patch("update-user", adminController.UpdateUser)
+		app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
 
 		// ListUsers admin api
-		app.Patch("/list-users", adminController.ListUsers)
+		app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
 
 		// ChangeBucketOwner admin api
-		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+		app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
 
 		// ListBucketsAndOwners admin api
-		app.Patch("/list-buckets", adminController.ListBuckets)
+		app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
 	}
 
 	// ListBuckets action

s3api/router_test.go

@@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) {
 		iam auth.IAMService
 	}
 	tests := []struct {
-		name string
-		sa   *S3ApiRouter
 		args args
+		sa   *S3ApiRouter
+		name string
 	}{
 		{
 			name: "Initialize S3 api router",

s3api/server.go

@@ -29,15 +29,15 @@ import (
 )
 
 type S3ApiServer struct {
-	app      *fiber.App
 	backend  backend.Backend
+	app      *fiber.App
 	router   *S3ApiRouter
-	port     string
 	cert     *tls.Certificate
+	port     string
+	health   string
 	quiet    bool
 	debug    bool
 	readonly bool
-	health   string
 }
 
 func New(

s3api/server_test.go

@@ -39,9 +39,9 @@ func TestNew(t *testing.T) {
 	port := ":7070"
 
 	tests := []struct {
-		name            string
-		args            args
 		wantS3ApiServer *S3ApiServer
+		args            args
+		name            string
 		wantErr         bool
 	}{
 		{
@@ -78,8 +78,8 @@ func TestNew(t *testing.T) {
 
 func TestS3ApiServer_Serve(t *testing.T) {
 	tests := []struct {
-		name    string
 		sa      *S3ApiServer
+		name    string
 		wantErr bool
 	}{
 		{

s3api/utils/auth-reader.go

@@ -41,10 +41,10 @@ const (
 // the data is completely read.
 type AuthReader struct {
 	ctx    *fiber.Ctx
+	r      *HashReader
 	auth   AuthData
 	secret string
 	size   int
-	r      *HashReader
 	debug  bool
 }
 

s3api/utils/chunk-reader.go

@@ -48,15 +48,15 @@ const (
 // object data stream
 type ChunkReader struct {
 	r                io.Reader
-	signingKey       []byte
+	chunkHash        hash.Hash
 	prevSig          string
 	parsedSig        string
+	strToSignPrefix  string
+	signingKey       []byte
+	stash            []byte
 	currentChunkSize int64
 	chunkDataLeft    int64
 	trailerExpected  int
-	stash            []byte
-	chunkHash        hash.Hash
-	strToSignPrefix  string
 	skipcheck        bool
 }
 

s3api/utils/presign-auth-reader.go

@@ -41,10 +41,10 @@ const (
 // data requests where the data size is not known until
 // the data is completely read.
 type PresignedAuthReader struct {
+	r      io.Reader
 	ctx    *fiber.Ctx
 	auth   AuthData
 	secret string
-	r      io.Reader
 	debug  bool
 }
 

s3api/utils/presign-auth-reader_test.go

@@ -23,13 +23,13 @@ import (
 
 func Test_validateExpiration(t *testing.T) {
 	type args struct {
-		str  string
 		date time.Time
+		str  string
 	}
 	tests := []struct {
-		name string
 		args args
 		err  error
+		name string
 	}{
 		{
 			name: "empty-expiration",

s3api/utils/utils.go

@@ -39,6 +39,10 @@ var (
 	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
+const (
+	upperhex = "0123456789ABCDEF"
+)
+
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
 	headers.DisableNormalizing()
@@ -64,7 +68,9 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		body = bytes.NewReader(req.Body())
 	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
+	escapedURI := escapeOriginalURI(ctx)
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -174,7 +180,7 @@ func ParseUint(str string) (int32, error) {
 	}
 	num, err := strconv.ParseUint(str, 10, 16)
 	if err != nil {
-		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+		return 1000, fmt.Errorf("invalid uint: %w", err)
 	}
 	return int32(num), nil
 }
@@ -248,35 +254,51 @@ func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
 	return
 }
 
-func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResult) s3response.GetObjectAttributesResult {
-	if _, ok := attrs[types.ObjectAttributesEtag]; !ok {
+func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResponse) s3response.GetObjectAttributesResponse {
+	// These properties shouldn't appear in the final response body
+	output.LastModified = nil
+	output.VersionId = nil
+	output.DeleteMarker = nil
+
+	if _, ok := attrs[s3response.ObjectAttributesEtag]; !ok {
 		output.ETag = nil
 	}
-	if _, ok := attrs[types.ObjectAttributesObjectParts]; !ok {
+	if _, ok := attrs[s3response.ObjectAttributesObjectParts]; !ok {
 		output.ObjectParts = nil
 	}
-	if _, ok := attrs[types.ObjectAttributesObjectSize]; !ok {
+	if _, ok := attrs[s3response.ObjectAttributesObjectSize]; !ok {
 		output.ObjectSize = nil
 	}
-	if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
+	if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
 		output.StorageClass = ""
 	}
+	fmt.Printf("%+v\n", output)
 
 	return output
 }
 
-func ParseObjectAttributes(ctx *fiber.Ctx) map[types.ObjectAttributes]struct{} {
-	attrs := map[types.ObjectAttributes]struct{}{}
+func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
+	attrs := map[s3response.ObjectAttributes]struct{}{}
+	var err error
 	ctx.Request().Header.VisitAll(func(key, value []byte) {
 		if string(key) == "X-Amz-Object-Attributes" {
 			oattrs := strings.Split(string(value), ",")
 			for _, a := range oattrs {
-				attrs[types.ObjectAttributes(a)] = struct{}{}
+				attr := s3response.ObjectAttributes(a)
+				if !attr.IsValid() {
+					err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
+					break
+				}
+				attrs[attr] = struct{}{}
 			}
 		}
 	})
 
-	return attrs
+	if len(attrs) == 0 {
+		return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
+	}
+
+	return attrs, err
 }
 
 type objLockCfg struct {
@@ -339,3 +361,74 @@ func IsValidOwnership(val types.ObjectOwnership) bool {
 		return false
 	}
 }
+
+func escapeOriginalURI(ctx *fiber.Ctx) string {
+	path := ctx.Path()
+
+	// Escape the URI original path
+	escapedURI := escapePath(path)
+
+	// Add the URI query params
+	query := string(ctx.Request().URI().QueryArgs().QueryString())
+	if query != "" {
+		escapedURI = escapedURI + "?" + query
+	}
+
+	return escapedURI
+}
+
+// Escapes the path string
+// Most of the parts copied from std url
+func escapePath(s string) string {
+	hexCount := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			hexCount++
+		}
+	}
+
+	if hexCount == 0 {
+		return s
+	}
+
+	var buf [64]byte
+	var t []byte
+
+	required := len(s) + 2*hexCount
+	if required <= len(buf) {
+		t = buf[:required]
+	} else {
+		t = make([]byte, required)
+	}
+
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case shouldEscape(c):
+			t[j] = '%'
+			t[j+1] = upperhex[c>>4]
+			t[j+2] = upperhex[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+
+	return string(t)
+}
+
+// Checks if the character needs to be escaped
+func shouldEscape(c byte) bool {
+	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '_', '.', '~', '/':
+		return false
+	}
+
+	return true
+}

s3api/utils/utils_test.go

@@ -19,10 +19,12 @@ import (
 	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3response"
 )
 
@@ -47,11 +49,11 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 	request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
 
 	tests := []struct {
-		name    string
 		args    args
 		want    *http.Request
-		wantErr bool
+		name    string
 		hdrs    []string
+		wantErr bool
 	}{
 		{
 			name: "Success-response",
@@ -99,9 +101,9 @@ func TestGetUserMetaData(t *testing.T) {
 	req := ctx.Request()
 
 	tests := []struct {
-		name         string
 		args         args
 		wantMetadata map[string]string
+		name         string
 	}{
 		{
 			name: "Success-empty-response",
@@ -283,47 +285,68 @@ func TestParseUint(t *testing.T) {
 
 func TestFilterObjectAttributes(t *testing.T) {
 	type args struct {
-		attrs  map[types.ObjectAttributes]struct{}
-		output s3response.GetObjectAttributesResult
+		attrs  map[s3response.ObjectAttributes]struct{}
+		output s3response.GetObjectAttributesResponse
 	}
+
 	etag, objSize := "etag", int64(3222)
+	delMarker := true
+
 	tests := []struct {
 		name string
 		args args
-		want s3response.GetObjectAttributesResult
+		want s3response.GetObjectAttributesResponse
 	}{
 		{
 			name: "keep only ETag",
 			args: args{
-				attrs: map[types.ObjectAttributes]struct{}{
-					types.ObjectAttributesEtag: {},
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag: {},
 				},
-				output: s3response.GetObjectAttributesResult{
+				output: s3response.GetObjectAttributesResponse{
 					ObjectSize: &objSize,
 					ETag:       &etag,
 				},
 			},
-			want: s3response.GetObjectAttributesResult{ETag: &etag},
+			want: s3response.GetObjectAttributesResponse{ETag: &etag},
 		},
 		{
 			name: "keep multiple props",
 			args: args{
-				attrs: map[types.ObjectAttributes]struct{}{
-					types.ObjectAttributesEtag:         {},
-					types.ObjectAttributesObjectSize:   {},
-					types.ObjectAttributesStorageClass: {},
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag:         {},
+					s3response.ObjectAttributesObjectSize:   {},
+					s3response.ObjectAttributesStorageClass: {},
 				},
-				output: s3response.GetObjectAttributesResult{
+				output: s3response.GetObjectAttributesResponse{
 					ObjectSize:  &objSize,
 					ETag:        &etag,
 					ObjectParts: &s3response.ObjectParts{},
 					VersionId:   &etag,
 				},
 			},
-			want: s3response.GetObjectAttributesResult{
+			want: s3response.GetObjectAttributesResponse{
 				ETag:       &etag,
 				ObjectSize: &objSize,
-				VersionId:  &etag,
+			},
+		},
+		{
+			name: "make sure LastModified, DeleteMarker and VersionId are removed",
+			args: args{
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag: {},
+				},
+				output: s3response.GetObjectAttributesResponse{
+					ObjectSize:   &objSize,
+					ETag:         &etag,
+					ObjectParts:  &s3response.ObjectParts{},
+					VersionId:    &etag,
+					LastModified: backend.GetTimePtr(time.Now()),
+					DeleteMarker: &delMarker,
+				},
+			},
+			want: s3response.GetObjectAttributesResponse{
+				ETag: &etag,
 			},
 		},
 	}
@@ -382,3 +405,125 @@ func TestIsValidOwnership(t *testing.T) {
 		})
 	}
 }
+
+func Test_shouldEscape(t *testing.T) {
+	type args struct {
+		c byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "shouldn't-escape-alphanum",
+			args: args{
+				c: 'h',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-char",
+			args: args{
+				c: '_',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-number",
+			args: args{
+				c: '0',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-path-separator",
+			args: args{
+				c: '/',
+			},
+			want: false,
+		},
+		{
+			name: "should-escape-special-char-1",
+			args: args{
+				c: '&',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-2",
+			args: args{
+				c: '*',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-3",
+			args: args{
+				c: '(',
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldEscape(tt.args.c); got != tt.want {
+				t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_escapePath(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "empty-string",
+			args: args{
+				s: "",
+			},
+			want: "",
+		},
+		{
+			name: "alphanum-path",
+			args: args{
+				s: "/test-bucket/test-key",
+			},
+			want: "/test-bucket/test-key",
+		},
+		{
+			name: "path-with-unescapable-chars",
+			args: args{
+				s: "/test~bucket/test.key",
+			},
+			want: "/test~bucket/test.key",
+		},
+		{
+			name: "path-with-escapable-chars",
+			args: args{
+				s: "/bucket-*(/test=key&",
+			},
+			want: "/bucket-%2A%28/test%3Dkey%26",
+		},
+		{
+			name: "path-with-space",
+			args: args{
+				s: "/test-bucket/my key",
+			},
+			want: "/test-bucket/my%20key",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := escapePath(tt.args.s); got != tt.want {
+				t.Errorf("escapePath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -57,6 +57,7 @@ const (
 	ErrAccessDenied
 	ErrMethodNotAllowed
 	ErrBucketNotEmpty
+	ErrVersionedBucketNotEmpty
 	ErrBucketAlreadyExists
 	ErrBucketAlreadyOwnedByYou
 	ErrNoSuchBucket
@@ -65,9 +66,11 @@ const (
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidMaxKeys
+	ErrInvalidMaxBuckets
 	ErrInvalidMaxUploads
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
+	ErrInvalidObjectAttributes
 	ErrInvalidPart
 	ErrInvalidPartNumber
 	ErrInternalError
@@ -122,6 +125,7 @@ const (
 	ErrNoSuchBucketPolicy
 	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
+	ErrObjectAttributesInvalidHeader
 	ErrRequestTimeTooSkewed
 	ErrInvalidBucketAclWithObjectOwnership
 	ErrBothCannedAndHeaderGrants
@@ -132,12 +136,23 @@ const (
 	ErrMissingSecurityHeader
 	ErrInvalidMetadataDirective
 	ErrKeyTooLong
+	ErrInvalidVersionId
+	ErrNoSuchVersion
+	ErrSuspendedVersioningNotAllowed
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
 	ErrQuotaExceeded
+	ErrVersioningNotConfigured
+
+	// Admin api errors
+	ErrAdminAccessDenied
+	ErrAdminUserNotFound
+	ErrAdminUserExists
+	ErrAdminInvalidUserRole
+	ErrAdminMissingUserAcess
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -156,6 +171,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrVersionedBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty. You must delete all versions in the bucket.",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrBucketAlreadyExists: {
 		Code:           "BucketAlreadyExists",
 		Description:    "The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again.",
@@ -176,6 +196,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The Content-Md5 you specified is not valid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidMaxBuckets: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-buckets must be an integer between 1 and 10000.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidMaxUploads: {
 		Code:           "InvalidArgument",
 		Description:    "Argument max-uploads must be an integer between 0 and 2147483647.",
@@ -196,6 +221,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Argument partNumberMarker must be an integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidObjectAttributes: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid attribute name specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchBucket: {
 		Code:           "NoSuchBucket",
 		Description:    "The specified bucket does not exist.",
@@ -353,7 +383,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
-		Description:    "The access key ID you provided does not exist in our records.",
+		Description:    "The AWS Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrRequestNotReadyYet: {
@@ -433,12 +463,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
-		Description:    "The specified object does not have an ObjectLock configuration.",
+		Description:    "The specified object does not have a ObjectLock configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidBucketObjectLockConfiguration: {
 		Code:           "InvalidRequest",
-		Description:    "Bucket is missing ObjectLockConfiguration.",
+		Description:    "Bucket is missing Object Lock Configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectLockConfigurationNotAllowed: {
@@ -476,6 +506,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectAttributesInvalidHeader: {
+		Code:           "InvalidRequest",
+		Description:    "The x-amz-object-attributes header specifying the attributes to be retrieved is either missing or empty",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrRequestTimeTooSkewed: {
 		Code:           "RequestTimeTooSkewed",
 		Description:    "The difference between the request time and the server's time is too large.",
@@ -521,11 +556,26 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Unknown metadata directive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidVersionId: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid version id specified",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrKeyTooLong: {
 		Code:           "KeyTooLongError",
 		Description:    "Your key is too long.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrNoSuchVersion: {
+		Code:           "NoSuchVersion",
+		Description:    "The specified version does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrSuspendedVersioningNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {
@@ -548,6 +598,38 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Your request was denied due to quota exceeded.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrVersioningNotConfigured: {
+		Code:           "VersioningNotConfigured",
+		Description:    "Versioning has not been configured for the gateway.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+
+	// Admin api errors
+	ErrAdminAccessDenied: {
+		Code:           "XAdminAccessDenied",
+		Description:    "Only admin users have access to this resource.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAdminUserNotFound: {
+		Code:           "XAdminUserNotFound",
+		Description:    "No user exists with the provided access key ID.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminUserExists: {
+		Code:           "XAdminUserExists",
+		Description:    "A user with the provided access key ID already exists.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAdminInvalidUserRole: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User role has to be one of the following: 'user', 'admin', 'userplus'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminMissingUserAcess: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User access key ID is missing.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3event/event.go

@@ -30,11 +30,11 @@ type S3EventSender interface {
 }
 
 type EventMeta struct {
+	ObjectETag  *string
+	VersionId   *string
 	BucketOwner string
 	EventName   EventType
 	ObjectSize  int64
-	ObjectETag  *string
-	VersionId   *string
 }
 
 type EventSchema struct {
@@ -42,6 +42,8 @@ type EventSchema struct {
 }
 
 type EventRecord struct {
+	ResponseElements  EventResponseElements `json:"responseElements"`
+	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
 	AwsRegion         string                `json:"awsRegion"`
@@ -49,9 +51,7 @@ type EventRecord struct {
 	EventName         EventType             `json:"eventName"`
 	UserIdentity      EventUserIdentity     `json:"userIdentity"`
 	RequestParameters EventRequestParams    `json:"requestParameters"`
-	ResponseElements  EventResponseElements `json:"responseElements"`
 	S3                EventS3Data           `json:"s3"`
-	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 }
 
 type EventUserIdentity struct {
@@ -99,11 +99,11 @@ type EventS3BucketData struct {
 }
 
 type EventObjectData struct {
-	Key       string  `json:"key"`
-	Size      int64   `json:"size"`
 	ETag      *string `json:"eTag"`
 	VersionId *string `json:"versionId"`
+	Key       string  `json:"key"`
 	Sequencer string  `json:"sequencer"`
+	Size      int64   `json:"size"`
 }
 
 type EventConfig struct {

s3event/kafka.go

@@ -31,9 +31,9 @@ import (
 var sequencer = 0
 
 type Kafka struct {
-	key    string
 	writer *kafka.Writer
 	filter EventFilter
+	key    string
 	mu     sync.Mutex
 }
 

s3event/nats.go

@@ -27,10 +27,10 @@ import (
 )
 
 type NatsEventSender struct {
-	topic  string
 	client *nats.Conn
-	mu     sync.Mutex
 	filter EventFilter
+	topic  string
+	mu     sync.Mutex
 }
 
 func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {

s3event/webhook.go

@@ -30,9 +30,9 @@ import (
 )
 
 type Webhook struct {
-	url    string
 	client *http.Client
 	filter EventFilter
+	url    string
 	mu     sync.Mutex
 }
 

s3log/audit-logger.go

@@ -33,8 +33,8 @@ type AuditLogger interface {
 
 type LogMeta struct {
 	BucketOwner string
-	ObjectSize  int64
 	Action      string
+	ObjectSize  int64
 	HttpStatus  int
 }
 
@@ -45,21 +45,16 @@ type LogConfig struct {
 }
 
 type LogFields struct {
+	Time               time.Time
 	BucketOwner        string
 	Bucket             string
-	Time               time.Time
 	RemoteIP           string
 	Requester          string
 	RequestID          string
 	Operation          string
 	Key                string
 	RequestURI         string
-	HttpStatus         int
 	ErrorCode          string
-	BytesSent          int
-	ObjectSize         int64
-	TotalTime          int64
-	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	VersionID          string
@@ -71,6 +66,11 @@ type LogFields struct {
 	TLSVersion         string
 	AccessPointARN     string
 	AclRequired        string
+	HttpStatus         int
+	BytesSent          int
+	ObjectSize         int64
+	TotalTime          int64
+	TurnAroundTime     int64
 }
 
 type AdminLogFields struct {
@@ -80,17 +80,17 @@ type AdminLogFields struct {
 	RequestID          string
 	Operation          string
 	RequestURI         string
-	HttpStatus         int
 	ErrorCode          string
-	BytesSent          int
-	TotalTime          int64
-	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	SignatureVersion   string
 	CipherSuite        string
 	AuthenticationType string
 	TLSVersion         string
+	HttpStatus         int
+	BytesSent          int
+	TotalTime          int64
+	TurnAroundTime     int64
 }
 
 type Loggers struct {

s3log/file.go

@@ -34,10 +34,10 @@ const (
 
 // FileLogger is a local file audit log
 type FileLogger struct {
-	logfile string
 	f       *os.File
-	gotErr  bool
+	logfile string
 	mu      sync.Mutex
+	gotErr  bool
 }
 
 var _ AuditLogger = &FileLogger{}

s3log/webhook.go

@@ -33,8 +33,8 @@ import (
 
 // WebhookLogger is a webhook URL audit log
 type WebhookLogger struct {
-	mu  sync.Mutex
 	url string
+	mu  sync.Mutex
 }
 
 var _ AuditLogger = &WebhookLogger{}

s3response/s3response.go

@@ -19,28 +19,38 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
 )
 
-const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+const (
+	iso8601TimeFormat         = "2006-01-02T15:04:05.000Z"
+	iso8601TimeFormatExtended = "2006-01-02T15:04:05.000000Z"
+	iso8601TimeFormatWithTZ   = "2006-01-02T15:04:05-0700"
+)
+
+type PutObjectOutput struct {
+	ETag      string
+	VersionID string
+}
 
 // Part describes part metadata.
 type Part struct {
-	PartNumber   int
 	LastModified time.Time
 	ETag         string
+	PartNumber   int
 	Size         int64
 }
 
 func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Part
 	aux := &struct {
-		LastModified string `xml:"LastModified"`
 		*Alias
+		LastModified string `xml:"LastModified"`
 	}{
 		Alias: (*Alias)(&p),
 	}
 
-	aux.LastModified = p.LastModified.Format(RFC3339TimeFormat)
+	aux.LastModified = p.LastModified.UTC().Format(iso8601TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -49,40 +59,61 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
+	Initiator Initiator
+	Owner     Owner
+
 	Bucket   string
 	Key      string
 	UploadID string `xml:"UploadId"`
 
-	Initiator Initiator
-	Owner     Owner
-
 	// The class of storage used to store the object.
 	StorageClass types.StorageClass
 
+	// List of parts.
+	Parts []Part `xml:"Part"`
+
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
-
-	// List of parts.
-	Parts []Part `xml:"Part"`
 }
 
-type GetObjectAttributesResult struct {
-	ETag         *string
-	LastModified *time.Time
-	ObjectSize   *int64
-	StorageClass types.StorageClass
+type ObjectAttributes string
+
+const (
+	ObjectAttributesEtag         ObjectAttributes = "ETag"
+	ObjectAttributesChecksum     ObjectAttributes = "Checksum"
+	ObjectAttributesObjectParts  ObjectAttributes = "ObjectParts"
+	ObjectAttributesStorageClass ObjectAttributes = "StorageClass"
+	ObjectAttributesObjectSize   ObjectAttributes = "ObjectSize"
+)
+
+func (o ObjectAttributes) IsValid() bool {
+	return o == ObjectAttributesChecksum ||
+		o == ObjectAttributesEtag ||
+		o == ObjectAttributesObjectParts ||
+		o == ObjectAttributesObjectSize ||
+		o == ObjectAttributesStorageClass
+}
+
+type GetObjectAttributesResponse struct {
+	ETag        *string
+	ObjectSize  *int64
+	ObjectParts *ObjectParts
+
+	// Not included in the response body
 	VersionId    *string
-	ObjectParts  *ObjectParts
+	LastModified *time.Time
+	DeleteMarker *bool
+	StorageClass types.StorageClass `xml:",omitempty"`
 }
 
 type ObjectParts struct {
+	Parts                []types.ObjectPart `xml:"Part"`
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
-	Parts                []types.ObjectPart `xml:"Part"`
 }
 
 // ListMultipartUploadsResponse - s3 api list multipart uploads response.
@@ -97,18 +128,17 @@ type ListMultipartUploadsResult struct {
 	Delimiter          string
 	Prefix             string
 	EncodingType       string `xml:"EncodingType,omitempty"`
-	MaxUploads         int
-	IsTruncated        bool
 
 	// List of pending uploads.
 	Uploads []Upload `xml:"Upload"`
 
 	// Delimed common prefixes.
 	CommonPrefixes []CommonPrefix
+	MaxUploads     int
+	IsTruncated    bool
 }
 
 type ListObjectsResult struct {
-	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name           *string
 	Prefix         *string
 	Marker         *string
@@ -116,13 +146,13 @@ type ListObjectsResult struct {
 	MaxKeys        *int32
 	Delimiter      *string
 	IsTruncated    *bool
+	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	EncodingType   types.EncodingType
 	Contents       []Object
 	CommonPrefixes []types.CommonPrefix
-	EncodingType   types.EncodingType
 }
 
 type ListObjectsV2Result struct {
-	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name                  *string
 	Prefix                *string
 	StartAfter            *string
@@ -132,9 +162,10 @@ type ListObjectsV2Result struct {
 	MaxKeys               *int32
 	Delimiter             *string
 	IsTruncated           *bool
+	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	EncodingType          types.EncodingType
 	Contents              []Object
 	CommonPrefixes        []types.CommonPrefix
-	EncodingType          types.EncodingType
 }
 
 type Object struct {
@@ -157,7 +188,7 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	}
 
 	if o.LastModified != nil {
-		formattedTime := o.LastModified.Format(RFC3339TimeFormat)
+		formattedTime := o.LastModified.UTC().Format(iso8601TimeFormat)
 		aux.LastModified = &formattedTime
 	}
 
@@ -166,24 +197,24 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 // Upload describes in progress multipart upload
 type Upload struct {
-	Key          string
-	UploadID     string `xml:"UploadId"`
+	Initiated    time.Time
 	Initiator    Initiator
 	Owner        Owner
+	Key          string
+	UploadID     string `xml:"UploadId"`
 	StorageClass types.StorageClass
-	Initiated    time.Time
 }
 
 func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Upload
 	aux := &struct {
-		Initiated string `xml:"Initiated"`
 		*Alias
+		Initiated string `xml:"Initiated"`
 	}{
 		Alias: (*Alias)(&u),
 	}
 
-	aux.Initiated = u.Initiated.Format(RFC3339TimeFormat)
+	aux.Initiated = u.Initiated.UTC().Format(iso8601TimeFormat)
 
 	return e.EncodeElement(aux, start)
 }
@@ -230,11 +261,11 @@ type DeleteResult struct {
 }
 type SelectObjectContentPayload struct {
 	Expression          *string
-	ExpressionType      types.ExpressionType
 	RequestProgress     *types.RequestProgress
 	InputSerialization  *types.InputSerialization
 	OutputSerialization *types.OutputSerialization
 	ScanRange           *types.ScanRange
+	ExpressionType      types.ExpressionType
 }
 
 type SelectObjectContentResult struct {
@@ -250,15 +281,39 @@ type Bucket struct {
 	Owner string `json:"owner"`
 }
 
+type ListBucketsInput struct {
+	Owner             string
+	ContinuationToken string
+	Prefix            string
+	MaxBuckets        int32
+	IsAdmin           bool
+}
+
 type ListAllMyBucketsResult struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
-	Owner   CanonicalUser
-	Buckets ListAllMyBucketsList
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	Owner             CanonicalUser
+	ContinuationToken string `xml:"ContinuationToken,omitempty"`
+	Prefix            string `xml:"Prefix,omitempty"`
+	Buckets           ListAllMyBucketsList
 }
 
 type ListAllMyBucketsEntry struct {
-	Name         string
 	CreationDate time.Time
+	Name         string
+}
+
+func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias ListAllMyBucketsEntry
+	aux := &struct {
+		*Alias
+		CreationDate string `xml:"CreationDate"`
+	}{
+		Alias: (*Alias)(&r),
+	}
+
+	aux.CreationDate = r.CreationDate.UTC().Format(iso8601TimeFormat)
+
+	return e.EncodeElement(aux, start)
 }
 
 type ListAllMyBucketsList struct {
@@ -271,9 +326,24 @@ type CanonicalUser struct {
 }
 
 type CopyObjectResult struct {
-	XMLName      xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
-	LastModified time.Time
-	ETag         string
+	XMLName             xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	LastModified        time.Time
+	ETag                string
+	CopySourceVersionId string `xml:"-"`
+}
+
+func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias CopyObjectResult
+	aux := &struct {
+		*Alias
+		LastModified string `xml:"LastModified"`
+	}{
+		Alias: (*Alias)(&r),
+	}
+
+	aux.LastModified = r.LastModified.UTC().Format(iso8601TimeFormat)
+
+	return e.EncodeElement(aux, start)
 }
 
 type AccessControlPolicy struct {
@@ -315,3 +385,85 @@ type InitiateMultipartUploadResult struct {
 	Key      string
 	UploadId string
 }
+
+type ListVersionsResult struct {
+	XMLName             xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"`
+	CommonPrefixes      []types.CommonPrefix
+	DeleteMarkers       []types.DeleteMarkerEntry `xml:"DeleteMarker"`
+	Delimiter           *string
+	EncodingType        types.EncodingType
+	IsTruncated         *bool
+	KeyMarker           *string
+	MaxKeys             *int32
+	Name                *string
+	NextKeyMarker       *string
+	NextVersionIdMarker *string
+	Prefix              *string
+	VersionIdMarker     *string
+	Versions            []types.ObjectVersion `xml:"Version"`
+}
+
+type GetBucketVersioningOutput struct {
+	MFADelete *types.MFADeleteStatus
+	Status    *types.BucketVersioningStatus
+	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
+}
+
+type PutObjectRetentionInput struct {
+	RetainUntilDate AmzDate
+	XMLName         xml.Name `xml:"Retention"`
+	Mode            types.ObjectLockRetentionMode
+}
+
+type AmzDate struct {
+	time.Time
+}
+
+// Parses the date from xml string and validates for predefined date formats
+func (d *AmzDate) UnmarshalXML(e *xml.Decoder, startElement xml.StartElement) error {
+	var dateStr string
+	err := e.DecodeElement(&dateStr, &startElement)
+	if err != nil {
+		return err
+	}
+
+	retDate, err := d.ISO8601Parse(dateStr)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	*d = AmzDate{retDate}
+	return nil
+}
+
+// Encodes expiration date if it is non-zero
+// Encodes empty string if it's zero
+func (d AmzDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if d.IsZero() {
+		return nil
+	}
+	return e.EncodeElement(d.UTC().Format(iso8601TimeFormat), startElement)
+}
+
+// Parses ISO8601 date string to time.Time by
+// validating different time layouts
+func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
+	for _, layout := range []string{
+		iso8601TimeFormat,
+		iso8601TimeFormatExtended,
+		iso8601TimeFormatWithTZ,
+		time.RFC3339,
+	} {
+		t, err = time.Parse(layout, date)
+		if err == nil {
+			return t, nil
+		}
+	}
+
+	return t, err
+}
+
+// Admin api response types
+type ListBucketsResult struct {
+	Buckets []Bucket
+}

s3select/message-handler.go

@@ -204,7 +204,6 @@ func genErrorMessage(errorCode, errorMessage string) []byte {
 type GetProgress func() (bytesScanned int64, bytesProcessed int64)
 
 type MessageHandler struct {
-	sync.Mutex
 	ctx           context.Context
 	cancel        context.CancelFunc
 	writer        *bufio.Writer
@@ -213,6 +212,7 @@ type MessageHandler struct {
 	stopCh        chan bool
 	resetCh       chan bool
 	bytesReturned int64
+	sync.Mutex
 }
 
 // NewMessageHandler creates a new MessageHandler instance and starts the event streaming

tests/.env.default

@@ -26,3 +26,7 @@ PASSWORD_ONE=HIJKLMN
 USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
+REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning
+COMMAND_LOG=command.log
+TIME_LOG=time.log

tests/.env.docker.default

@@ -24,4 +24,5 @@ PASSWORD_ONE=HIJKLMN
 USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
-RECREATE_BUCKETS=true
+RECREATE_BUCKETS=true
+REMOVE_TEST_FILE_FOLDER=true

tests/Dockerfile_direct

@@ -0,0 +1,73 @@
+FROM ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG SECRETS_FILE=tests/.secrets.direct
+ARG CONFIG_FILE=tests/.env.direct
+ARG AWS_CLI=awscli-exe-linux-aarch64.zip
+ARG MC_FOLDER=linux-arm64
+
+ENV TZ=Etc/UTC
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    git \
+    wget \
+    curl \
+    unzip \
+    tzdata \
+    s3cmd \
+    jq \
+    bc \
+    libxml2-utils \
+    ca-certificates && \
+    update-ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /tmp
+
+# Install AWS cli
+RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+
+# Install mc
+RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
+  --create-dirs \
+  -o /usr/local/minio-binaries/mc && \
+chmod -R 755 /usr/local/minio-binaries
+ENV PATH=/usr/local/minio-binaries:${PATH}
+
+# Create tester user
+RUN groupadd -r tester && useradd -r -g tester tester
+RUN mkdir /home/tester && chown tester:tester /home/tester
+ENV HOME=/home/tester
+
+# install bats
+RUN git clone https://github.com/bats-core/bats-core.git && \
+    cd bats-core && \
+    ./install.sh /home/tester
+
+USER tester
+RUN mkdir -p /home/tester/tests
+COPY --chown=tester:tester . /home/tester
+
+# add bats support libraries
+RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
+RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
+
+WORKDIR /home/tester
+
+RUN . $SECRETS_FILE && \
+    export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
+    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
+    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
+    aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
+
+RUN mkdir /tmp/gw
+
+RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen_bits:2048 && \
+    openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
+        -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+ENV WORKSPACE=.
+ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
+
+CMD ["tests/run_all.sh"]

tests/Dockerfile_test_bats

@@ -3,7 +3,7 @@ FROM ubuntu:latest
 ARG DEBIAN_FRONTEND=noninteractive
 ARG SECRETS_FILE=tests/.secrets
 ARG CONFIG_FILE=tests/.env.docker
-ARG GO_LIBRARY=go1.21.7.linux-arm64.tar.gz
+ARG GO_LIBRARY=go1.23.1.linux-arm64.tar.gz
 ARG AWS_CLI=awscli-exe-linux-aarch64.zip
 ARG MC_FOLDER=linux-arm64
 
@@ -19,6 +19,7 @@ RUN apt-get update && \
     s3cmd \
     jq \
     bc \
+    libxml2-utils \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*
@@ -34,7 +35,7 @@ RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
   --create-dirs \
   -o /usr/local/minio-binaries/mc && \
 chmod -R 755 /usr/local/minio-binaries
-ENV PATH="/usr/local/minio-binaries":${PATH}
+ENV PATH=/usr/local/minio-binaries:${PATH}
 
 # Download Go 1.21 (adjust the version and platform as needed)
 RUN wget https://golang.org/dl/${GO_LIBRARY}
@@ -85,4 +86,5 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
 ENV WORKSPACE=.
 ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
 
-CMD ["tests/run_all.sh"]
+ENTRYPOINT ["tests/run.sh"]
+CMD ["s3api,s3,s3cmd,mc,rest"]

tests/README.md

@@ -81,4 +81,56 @@ For the s3 backend, see the **S3 Backend** instructions above.
 
 If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries.
 
-A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`
+A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`
+
+## Environment Parameters
+
+**AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**:  identical to the same parameters in **s3**.
+
+**VERSITY_EXE**:  location of the versity executable relative to test folder.
+
+**RUN_VERSITYGW**:  whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
+
+**BACKEND**:  the storage backend type for the gateway, e.g. **posix** or **s3**.
+
+**LOCAL_FOLDER**:  if running with a **posix** backend, the backend storage folder.
+
+**BUCKET_ONE_NAME**, **BUCKET_TWO_NAME**:  test bucket names.
+
+**RECREATE_BUCKETS**:  whether to delete buckets between tests.  If set to false, the bucket will be restored to an original state for the purpose of ensuring consistent tests, but not deleted.
+
+**CERT**, **KEY**:  certificate and key locations if using SSL.
+
+**S3CMD_CONFIG**:  location of **s3cmd** config file if running **s3cmd** tests.
+
+**SECRETS_FILE**:  file where sensitive values, such as **AWS_SECRET_ACCESS_KEY**, should be stored.
+
+**MC_ALIAS**:  Minio MC alias if running MC tests.
+
+**LOG_LEVEL**:  level for test logger (1 - only critical, 2 - errors, 3 - warnings, 4 - info, 5 - debug info, 6 - tracing)
+
+**GOCOVERDIR**:  folder to put golang coverage info in, if checking coverage info.
+
+**USERS_FOLDER**:  folder to use if storing IAM data in a folder.
+
+**IAM_TYPE**:  how to store IAM data (**s3** or **folder**).
+
+**TEST_LOG_FILE**:  log file location for these bats tests.
+
+**VERSITY_LOG_FILE**:  log file for versity application as it is tested by bats tests.
+
+**DIRECT**:  if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes).
+
+**DIRECT_DISPLAY_NAME**:  username if **DIRECT** is set to **true**.
+
+**COVERAGE_DB**:  database to store client command coverage info and usage counts, if using.
+
+**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  credentials for users created and tested for non-root user **versitygw** operations.
+
+**TEST_FILE_FOLDER**:  where to put temporary test files.
+
+**REMOVE_TEST_FILE_FOLDER**:  whether to delete the test file folder between tests, should be set to **true** unless checking the files after a single test, or not yet sure that the test folder is in a safe location to avoid deleting other files.
+
+**VERSIONING_DIR**:  where to put gateway file versioning info.
+
+**COMMAND_LOG**:  where to store list of client commands, which if using will be reported during test failures.

tests/commands/abort_multipart_upload.sh

@@ -20,7 +20,7 @@ abort_multipart_upload() {
     log 2 "'abort multipart upload' command requires bucket, key, upload ID"
     return 1
   fi
-  if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! error=$(send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error aborting upload: $error"
     return 1
   fi
@@ -33,10 +33,10 @@ abort_multipart_upload_with_user() {
     return 1
   fi
   record_command "abort-multipart-upload" "client:s3api"
-  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
     log 2 "Error aborting upload: $abort_multipart_upload_error"
     export abort_multipart_upload_error
     return 1
   fi
   return 0
-}
+}

tests/commands/command.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/logger.sh
+
+send_command() {
+  if [ $# -eq 0 ]; then
+    return 1
+  fi
+  if [ -n "$COMMAND_LOG" ]; then
+    args=(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" "$@")
+    if ! mask_arg_array "${args[@]}"; then
+      return 1
+    fi
+    # shellcheck disable=SC2154
+    echo "${masked_args[*]}" >> "$COMMAND_LOG"
+    "$@"
+    return $?
+  fi
+  "$@"
+}

tests/commands/complete_multipart_upload.sh

@@ -21,7 +21,7 @@ complete_multipart_upload() {
   fi
   log 5 "complete multipart upload id: $3, parts: $4"
   record_command "complete-multipart-upload" "client:s3api"
-  error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
+  error=$(send_command aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
   if [[ $completed -ne 0 ]]; then
     log 2 "error completing multipart upload: $error"
     return 1

tests/commands/copy_object.sh

@@ -23,14 +23,14 @@ copy_object() {
   local error
   record_command "copy-object" "client:$1"
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4"
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
   else
     echo "'copy-object' not implemented for '$1'"
     return 1
@@ -45,7 +45,7 @@ copy_object() {
 
 copy_object_empty() {
   record-command "copy-object" "client:s3api"
-  error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
+  error=$(send_command aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
   if [[ $result -eq 0 ]]; then
     log 2 "copy object with empty parameters returned no error"
     return 1

tests/commands/create_bucket.sh

@@ -30,14 +30,14 @@ create_bucket() {
   local error
   log 6 "create bucket"
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -56,11 +56,11 @@ create_bucket_with_user() {
   fi
   local exit_code=0
   if [[ $1 == "aws" ]] || [[ $1 == "s3api" ]]; then
-    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -80,7 +80,7 @@ create_bucket_object_lock_enabled() {
   fi
 
   local exit_code=0
-  error=$(aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
+  error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
   if [ $exit_code -ne 0 ]; then
     log 2 "error creating bucket: $error"
     return 1

tests/commands/create_multipart_upload.sh

@@ -24,7 +24,7 @@ create_multipart_upload() {
     return 1
   fi
 
-  if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+  if ! multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
     log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
@@ -44,7 +44,7 @@ create_multipart_upload_with_user() {
     return 1
   fi
 
-  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
     log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
@@ -65,7 +65,7 @@ create_multipart_upload_params() {
     return 1
   fi
   local multipart_data
-  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload \
+  multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload \
     --bucket "$1" \
     --key "$2" \
     --content-type "$3" \
@@ -96,7 +96,7 @@ create_multipart_upload_custom() {
   done
   log 5 "${*:3}"
   log 5 "aws --no-verify-ssl s3api create-multipart-upload --bucket $1 --key $2 ${*:3}"
-  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
+  multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
   if [[ $result -ne 0 ]]; then
     log 2 "error creating custom multipart data command: $multipart_data"
     return 1
@@ -107,3 +107,19 @@ create_multipart_upload_custom() {
   log 5 "upload id: $upload_id"
   return 0
 }
+
+create_multipart_upload_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'create_multipart_upload_rest' requires bucket name, key"
+    return 1
+  fi
+  if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then
+    log 2 "error creating multipart upload: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")"
+    return 1
+  fi
+  return 0
+}

tests/commands/create_presigned_url.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+create_presigned_url() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "create presigned url function requires command type, bucket, and filename"
+    return 1
+  fi
+
+  local presign_result=0
+  if [[ $1 == 'aws' ]]; then
+    presigned_url=$(send_command aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    presigned_url=$(send_command s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    presigned_url_data=$(send_command mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
+    presigned_url="${presigned_url_data#*Share: }"
+  else
+    log 2 "unrecognized command type $1"
+    return 1
+  fi
+  if [[ $presign_result -ne 0 ]]; then
+    log 2 "error generating presigned url: $presigned_url"
+    return 1
+  fi
+  export presigned_url
+  return 0
+}

tests/commands/delete_bucket.sh

@@ -31,13 +31,13 @@ delete_bucket() {
 
   exit_code=0
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
   elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
+    error=$(send_command aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
   else
     log 2 "Invalid command type $1"
     return 1

tests/commands/delete_bucket_policy.sh

@@ -21,12 +21,12 @@ delete_bucket_policy() {
     return 1
   fi
   local delete_result=0
-  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+    error=$(send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
+    error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
+    error=$(send_command mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
   else
     log 2 "command 'delete bucket policy' not implemented for '$1'"
     return 1
@@ -44,7 +44,7 @@ delete_bucket_policy_with_user() {
     log 2 "'delete bucket policy with user' command requires bucket, username, password"
     return 1
   fi
-  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
+  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
     log 2 "error deleting bucket policy: $delete_bucket_policy_error"
     export delete_bucket_policy_error
     return 1

tests/commands/delete_bucket_tagging.sh

@@ -22,9 +22,9 @@ delete_bucket_tagging() {
   fi
   local result
   if [[ $1 == 'aws' ]]; then
-    tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
+    tags=$(send_command aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
-    tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
+    tags=$(send_command mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -43,7 +43,7 @@ delete_bucket_tagging_with_user() {
     log 2 "delete bucket tagging command missing username, password, bucket name"
     return 1
   fi
-  if ! error=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
+  if ! error=$(send_command AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
     log 2 "error deleting bucket tagging with user: $error"
     return 1
   fi