Merge pull request #1348 from versity/ben/crc-mp-complete

feat: calculate full object crc for multi-part uploads for compatible checksums

.github/SECURITY.md

@@ -0,0 +1,25 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+If you discover a security vulnerability in `versitygw`, we strongly encourage you to report it privately and responsibly.
+
+Please do **not** create public issues or pull requests that contain details about the vulnerability.
+
+Instead, report the issue using GitHub's private **Security Advisories** feature:
+
+- Go to [versitygw's Security Advisories page](https://github.com/versity/versitygw/security/advisories)
+- Click on **"Report a vulnerability"**
+
+We aim to respond within **2 business days** and work with you to quickly resolve the issue.
+
+## Supported Versions
+
+| Version         | Supported |
+| --------------- | --------- |
+| Latest (v1.x.x) | ✅        |
+| Older versions  | ❌        |
+
+## Responsible Disclosure
+
+We appreciate responsible disclosures and are committed to fixing vulnerabilities in a timely manner. Thank you for helping keep `versitygw` secure.

.github/workflows/azurite.yml

@@ -1,5 +1,5 @@
 name: azurite functional tests
-
+permissions: {}
 on: pull_request
 
 jobs:

.github/workflows/docker-bats.yml

@@ -1,5 +1,5 @@
 name: docker bats tests
-
+permissions: {}
 on: pull_request
 
 jobs:

.github/workflows/docker.yml

@@ -1,5 +1,4 @@
 name: Publish Docker image
-
 on:
   release:
     types: [published]

.github/workflows/functional.yml

@@ -1,5 +1,5 @@
 name: functional tests
-
+permissions: {}
 on: pull_request
 
 jobs:

.github/workflows/go.yml

@@ -1,9 +1,10 @@
 name: general
+permissions: {}
 on: pull_request
 jobs:
 
   build:
-    name: Build
+    name: Go Basic Checks
     runs-on: ubuntu-latest
     steps:
 
@@ -23,9 +24,6 @@ jobs:
       run: |
         go get -v -t -d ./...
 
-    - name: Build
-      run: make
-
     - name: Test
       run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
 
@@ -35,4 +33,26 @@ jobs:
 
     - name: Run govulncheck
       run: govulncheck ./...
-      shell: bash
+      shell: bash
+
+  verify-build:
+    name: Verify Build Targets
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [darwin, freebsd, linux]
+        arch: [amd64, arm64]
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+
+      - name: Build for ${{ matrix.os }}/${{ matrix.arch }}
+        run: |
+          GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o versitygw-${{ matrix.os }}-${{ matrix.arch }} cmd/versitygw/*.go

.github/workflows/goreleaser.yml

@@ -1,16 +1,12 @@
 name: goreleaser
-
+permissions:
+  contents: write
 on:
   push:
     # run only against tags
     tags:
       - '*'
 
-permissions:
-  contents: write
-  # packages: write
-  # issues: write
-
 jobs:
   goreleaser:
     runs-on: ubuntu-latest
@@ -29,10 +25,10 @@ jobs:
           go-version: stable
 
       - name: Run Releaser
-        uses: goreleaser/goreleaser-action@v5
+        uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser
-          version: latest
+          version: '~> v2'
           args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/host-style-tests.yml

@@ -0,0 +1,13 @@
+name: host style tests
+permissions: {}
+on: pull_request
+
+jobs:
+  build-and-run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: run host-style tests
+        run: make test-host-style

.github/workflows/shellcheck.yml

@@ -1,4 +1,5 @@
 name: shellcheck
+permissions: {}
 on: pull_request
 jobs:
 

.github/workflows/static.yml

@@ -1,4 +1,5 @@
 name: staticcheck
+permissions: {}
 on: pull_request
 jobs: 
 

.github/workflows/system.yml

@@ -1,4 +1,5 @@
 name: system tests
+permissions: {}
 on: pull_request
 jobs:
   build:
@@ -12,66 +13,85 @@ jobs:
             IAM_TYPE: folder
             RUN_SET: "mc-non-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "mc, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "mc-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
-          - set: "REST, posix, non-static, all, folder IAM"
+          - set: "REST, posix, non-static, base|acl|multipart, folder IAM"
             IAM_TYPE: folder
-            RUN_SET: "rest"
+            RUN_SET: "rest-base,rest-acl,rest-multipart"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "REST, posix, non-static, chunked|checksum|versioning|bucket, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3-non-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, policy, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, user, non-static, s3 IAM"
             IAM_TYPE: s3
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, bucket, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-bucket"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, multipart, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-multipart"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, object, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-object"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, policy, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, user, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           # TODO fix/debug s3 gateway
           #- set: "s3api, s3, multipart|object, non-static, folder IAM"
@@ -88,16 +108,19 @@ jobs:
             IAM_TYPE: folder
             RUN_SET: "s3cmd-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3cmd, posix, non-user, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-non-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3cmd, posix, user, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
@@ -106,7 +129,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 'stable'
+          go-version: "stable"
         id: go
 
       - name: Get Dependencies
@@ -122,6 +145,7 @@ jobs:
 
       - name: Install s3cmd
         run: |
+          sudo apt-get update
           sudo apt-get install s3cmd
 
       - name: Install mc
@@ -129,9 +153,10 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Install xmllint (for rest)
+      - name: Install xml libraries (for rest)
         run: |
-          sudo apt-get install libxml2-utils
+          sudo apt-get update
+          sudo apt-get install libxml2-utils xmlstarlet
 
       # see https://github.com/versity/versitygw/issues/1034
       - name: Install AWS cli
@@ -150,6 +175,7 @@ jobs:
           RUN_VERSITYGW: true
           BACKEND: ${{ matrix.BACKEND }}
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
+          DELETE_BUCKETS_AFTER_TEST: ${{ matrix.DELETE_BUCKETS_AFTER_TEST }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
           LOCAL_FOLDER: /tmp/gw
@@ -173,6 +199,8 @@ jobs:
           COMMAND_LOG: command.log
           TIME_LOG: time.log
           PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
+          AUTOGENERATE_USERS: true
+          USER_AUTOGENERATION_PREFIX: github-actions-test-
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST

.goreleaser.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 before:
   hooks:
     - go mod tidy
@@ -23,7 +25,7 @@ builds:
       - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
 
 archives:
-  - format: tar.gz
+  - formats: [ 'tar.gz' ]
     # this name template makes the OS and Arch compatible with the results of uname.
     name_template: >-
       {{ .ProjectName }}_v{{ .Version }}_
@@ -43,7 +45,7 @@ archives:
     # use zip for windows archives
     format_overrides:
     - goos: windows
-      format: zip
+      formats: [ 'zip' ]
 
     # Additional files/globs you want to add to the archive.
     #
@@ -58,7 +60,7 @@ checksum:
   name_template: 'checksums.txt'
 
 snapshot:
-  name_template: "{{ incpatch .Version }}-next"
+  version_template: "{{ incpatch .Version }}-{{.ShortCommit}}"
 
 changelog:
   sort: asc
@@ -86,7 +88,7 @@ nfpms:
 
     license: Apache 2.0
 
-    builds:
+    ids:
       - versitygw
 
     formats:

Makefile

@@ -72,6 +72,11 @@ dist:
 	rm -f VERSION
 	gzip -f $(TARFILE)
 
+.PHONY: snapshot
+snapshot:
+# brew install goreleaser/tap/goreleaser
+	goreleaser release --snapshot --skip publish --clean
+
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
@@ -91,3 +96,9 @@ up-azurite:
 .PHONY: up-app
 up-app:
 	$(DOCKERCOMPOSE) up
+
+# Run the host-style tests in docker containers
+.PHONY: test-host-style
+test-host-style:
+	docker compose -f tests/host-style-tests/docker-compose.yml up --build --abort-on-container-exit --exit-code-from test
+

auth/access-control.go

@@ -0,0 +1,201 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+)
+
+func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	// Verify destination bucket access
+	if err := VerifyAccess(ctx, be, opts); err != nil {
+		return err
+	}
+	// Verify source bucket access
+	srcBucket, srcObject, found := strings.Cut(copySource, "/")
+	if !found {
+		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	// Get source bucket ACL
+	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+	if err != nil {
+		return err
+	}
+
+	var srcBucketAcl ACL
+	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+		return err
+	}
+
+	if err := VerifyAccess(ctx, be, AccessOptions{
+		Acl:           srcBucketAcl,
+		AclPermission: PermissionRead,
+		IsRoot:        opts.IsRoot,
+		Acc:           opts.Acc,
+		Bucket:        srcBucket,
+		Object:        srcObject,
+		Action:        GetObjectAction,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type AccessOptions struct {
+	Acl            ACL
+	AclPermission  Permission
+	IsRoot         bool
+	Acc            Account
+	Bucket         string
+	Object         string
+	Action         Action
+	Readonly       bool
+	IsBucketPublic bool
+}
+
+func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	// Skip the access check for public buckets
+	if opts.IsBucketPublic {
+		return nil
+	}
+	if opts.Readonly {
+		if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+	}
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
+	}
+
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Detects if the action is policy related
+// e.g.
+// 'GetBucketPolicy', 'PutBucketPolicy'
+func isPolicyAction(action Action) bool {
+	return action == GetBucketPolicyAction || action == PutBucketPolicyAction
+}
+
+// VerifyPublicAccess checks if the bucket is publically accessible by ACL or Policy
+func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, permission Permission, bucket, object string) error {
+	// ACL disabled
+	policy, err := be.GetBucketPolicy(ctx, bucket)
+	if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+		return err
+	}
+	if err == nil {
+		err = VerifyPublicBucketPolicy(policy, bucket, object, action)
+		if err == nil {
+			// if ACLs are disabled, and the bucket grants public access,
+			// policy actions should return 'MethodNotAllowed'
+			if isPolicyAction(action) {
+				return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+			}
+
+			return nil
+		}
+	}
+
+	// if the action is not in the ACL whitelist the access is denied
+	_, ok := publicACLAllowedActions[action]
+	if !ok {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	err = VerifyPublicBucketACL(ctx, be, bucket, action, permission)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+func MayCreateBucket(acct Account, isRoot bool) error {
+	if isRoot {
+		return nil
+	}
+
+	if acct.Role == RoleUser {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
+	// Owner check
+	if acct.Access == acl.Owner {
+		return nil
+	}
+
+	// Root user has access over almost everything
+	if isRoot {
+		return nil
+	}
+
+	// Admin user case
+	if acct.Role == RoleAdmin {
+		return nil
+	}
+
+	// Return access denied in all other cases
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}
+
+type PublicACLAllowedActions map[Action]struct{}
+
+var publicACLAllowedActions PublicACLAllowedActions = PublicACLAllowedActions{
+	ListBucketAction:                 struct{}{},
+	PutObjectAction:                  struct{}{},
+	ListBucketMultipartUploadsAction: struct{}{},
+	DeleteObjectAction:               struct{}{},
+	ListBucketVersionsAction:         struct{}{},
+	GetObjectAction:                  struct{}{},
+	GetObjectAttributesAction:        struct{}{},
+	GetObjectAclAction:               struct{}{},
+}

auth/acl.go

@@ -33,6 +33,17 @@ type ACL struct {
 	Grantees []Grantee
 }
 
+// IsPublic specifies if the acl grants public read access
+func (acl *ACL) IsPublic(permission Permission) bool {
+	for _, grt := range acl.Grantees {
+		if grt.Permission == permission && grt.Type == types.TypeGroup && grt.Access == "all-users" {
+			return true
+		}
+	}
+
+	return false
+}
+
 type Grantee struct {
 	Permission Permission
 	Access     string
@@ -435,117 +446,22 @@ func verifyACL(acl ACL, access string, permission Permission) error {
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
-func MayCreateBucket(acct Account, isRoot bool) error {
-	if isRoot {
-		return nil
-	}
-
-	if acct.Role == RoleUser {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	return nil
-}
-
-func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
-	// Owner check
-	if acct.Access == acl.Owner {
-		return nil
-	}
-
-	// Root user has access over almost everything
-	if isRoot {
-		return nil
-	}
-
-	// Admin user case
-	if acct.Role == RoleAdmin {
-		return nil
-	}
-
-	// Return access denied in all other cases
-	return s3err.GetAPIError(s3err.ErrAccessDenied)
-}
-
-type AccessOptions struct {
-	Acl           ACL
-	AclPermission Permission
-	IsRoot        bool
-	Acc           Account
-	Bucket        string
-	Object        string
-	Action        Action
-	Readonly      bool
-}
-
-func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
-	if opts.Readonly {
-		if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-	}
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
-		return nil
-	}
-
-	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
-	if policyErr != nil {
-		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-			return policyErr
-		}
-	} else {
-		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
-	}
-
-	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
-		return nil
-	}
-
-	// Verify destination bucket access
-	if err := VerifyAccess(ctx, be, opts); err != nil {
-		return err
-	}
-	// Verify source bucket access
-	srcBucket, srcObject, found := strings.Cut(copySource, "/")
-	if !found {
-		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
-	}
-
-	// Get source bucket ACL
-	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+// Verifies if the bucket acl grants public access
+func VerifyPublicBucketACL(ctx context.Context, be backend.Backend, bucket string, action Action, permission Permission) error {
+	aclBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{
+		Bucket: &bucket,
+	})
 	if err != nil {
 		return err
 	}
 
-	var srcBucketAcl ACL
-	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+	acl, err := ParseACL(aclBytes)
+	if err != nil {
 		return err
 	}
 
-	if err := VerifyAccess(ctx, be, AccessOptions{
-		Acl:           srcBucketAcl,
-		AclPermission: PermissionRead,
-		IsRoot:        opts.IsRoot,
-		Acc:           opts.Acc,
-		Bucket:        srcBucket,
-		Object:        srcObject,
-		Action:        GetObjectAction,
-	}); err != nil {
-		return err
+	if !acl.IsPublic(permission) {
+		return ErrAccessDenied
 	}
 
 	return nil

auth/bucket_policy.go

@@ -22,6 +22,8 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
+var ErrAccessDenied = errors.New("access denied")
+
 type policyErr string
 
 func (p policyErr) Error() string {
@@ -89,6 +91,24 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri
 	return isAllowed
 }
 
+// isPublic checks if the bucket policy statements contain
+// an entity granting public access
+func (bp *BucketPolicy) isPublic(resource string, action Action) bool {
+	var isAllowed bool
+	for _, statement := range bp.Statement {
+		if statement.isPublic(resource, action) {
+			switch statement.Effect {
+			case BucketPolicyAccessTypeAllow:
+				isAllowed = true
+			case BucketPolicyAccessTypeDeny:
+				return false
+			}
+		}
+	}
+
+	return isAllowed
+}
+
 type BucketPolicyItem struct {
 	Effect     BucketPolicyAccessType `json:"Effect"`
 	Principals Principals             `json:"Principal"`
@@ -134,6 +154,11 @@ func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource
 	return false
 }
 
+// isPublic checks if the bucket policy statemant grants public access
+func (bpi *BucketPolicyItem) isPublic(resource string, action Action) bool {
+	return bpi.Principals.IsPublic() && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource)
+}
+
 func getMalformedPolicyError(err error) error {
 	return s3err.APIError{
 		Code:           "MalformedPolicy",
@@ -183,3 +208,22 @@ func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Act
 
 	return nil
 }
+
+// Checks if the bucket policy grants public access
+func VerifyPublicBucketPolicy(policy []byte, bucket, object string, action Action) error {
+	var bucketPolicy BucketPolicy
+	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
+		return err
+	}
+
+	resource := bucket
+	if object != "" {
+		resource += "/" + object
+	}
+
+	if !bucketPolicy.isPublic(resource, action) {
+		return ErrAccessDenied
+	}
+
+	return nil
+}

auth/bucket_policy_actions.go

@@ -91,6 +91,7 @@ var supportedActionList = map[Action]struct{}{
 	DeleteObjectTaggingAction:              {},
 	ListBucketVersionsAction:               {},
 	ListBucketAction:                       {},
+	GetBucketObjectLockConfigurationAction: {},
 	PutBucketObjectLockConfigurationAction: {},
 	GetObjectLegalHoldAction:               {},
 	PutObjectLegalHoldAction:               {},

auth/bucket_policy_principals.go

@@ -121,3 +121,10 @@ func (p Principals) Contains(userAccess string) bool {
 	_, found := p[userAccess]
 	return found
 }
+
+// Bucket policy grants public access, if it contains
+// a wildcard match to all the users
+func (p Principals) IsPublic() bool {
+	_, ok := p["*"]
+	return ok
+}

auth/iam.go

@@ -18,6 +18,8 @@ import (
 	"errors"
 	"fmt"
 	"time"
+
+	"github.com/versity/versitygw/s3err"
 )
 
 type Role string
@@ -57,10 +59,19 @@ type ListUserAccountsResult struct {
 // Mutable props, which could be changed when updating an IAM account
 type MutableProps struct {
 	Secret  *string `json:"secret"`
+	Role    Role    `json:"role"`
 	UserID  *int    `json:"userID"`
 	GroupID *int    `json:"groupID"`
 }
 
+func (m MutableProps) Validate() error {
+	if m.Role != "" && !m.Role.IsValid() {
+		return s3err.GetAPIError(s3err.ErrAdminInvalidUserRole)
+	}
+
+	return nil
+}
+
 func updateAcc(acc *Account, props MutableProps) {
 	if props.Secret != nil {
 		acc.Secret = *props.Secret
@@ -71,6 +82,9 @@ func updateAcc(acc *Account, props MutableProps) {
 	if props.UserID != nil {
 		acc.UserID = *props.UserID
 	}
+	if props.Role != "" {
+		acc.Role = props.Role
+	}
 }
 
 // IAMService is the interface for all IAM service implementations

auth/iam_internal.go

@@ -290,93 +290,49 @@ func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
 
 func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
 	// We are going to be racing with other running gateways without any
-	// coordination. So the strategy here is to read the current file data.
-	// If the file doesn't exist, then we assume someone else is currently
-	// updating the file. So we just need to keep retrying. We also need
-	// to make sure the data is consistent within a single update. So racing
-	// writes to a file would possibly leave this in some invalid state.
-	// We can get atomic updates with rename. If we read the data, update
-	// the data, write to a temp file, then rename the tempfile back to the
-	// data file. This should always result in a complete data image.
+	// coordination. So the strategy here is to read the current file data,
+	// update the data, write back out to a temp file, then rename the
+	// temp file to the original file. This rename will replace the
+	// original file with the new file. This is atomic and should always
+	// allow for a consistent view of the data. There is a small
+	// window where the file could be read and then updated by
+	// another process. In this case any updates the other process did
+	// will be lost. This is a limitation of the internal IAM service.
+	// This should be rare, and even when it does happen should result
+	// in a valid IAM file, just without the other process's updates.
 
-	// There is at least one unsolved failure mode here.
-	// If a gateway removes the data file and then crashes, all other
-	// gateways will retry forever thinking that the original will eventually
-	// write the file.
+	iamFname := filepath.Join(s.dir, iamFile)
+	backupFname := filepath.Join(s.dir, iamBackupFile)
 
-	retries := 0
-	fname := filepath.Join(s.dir, iamFile)
+	b, err := os.ReadFile(iamFname)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("read iam file: %w", err)
+	}
 
-	for {
-		b, err := os.ReadFile(fname)
-		if errors.Is(err, fs.ErrNotExist) {
-			// racing with someone else updating
-			// keep retrying after backoff
-			retries++
-			if retries < maxretry {
-				time.Sleep(backoff)
-				continue
-			}
+	// save copy of data
+	datacopy := make([]byte, len(b))
+	copy(datacopy, b)
 
-			// we have been unsuccessful trying to read the iam file
-			// so this must be the case where something happened and
-			// the file did not get updated successfully, and probably
-			// isn't going to be. The recovery procedure would be to
-			// copy the backup file into place of the original.
-			return fmt.Errorf("no iam file, needs backup recovery")
-		}
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("read iam file: %w", err)
-		}
+	// make a backup copy in case something happens
+	err = s.writeUsingTempFile(b, backupFname)
+	if err != nil {
+		return fmt.Errorf("write backup iam file: %w", err)
+	}
 
-		// reset retries on successful read
-		retries = 0
+	b, err = update(b)
+	if err != nil {
+		return fmt.Errorf("update iam data: %w", err)
+	}
 
-		err = os.Remove(fname)
-		if errors.Is(err, fs.ErrNotExist) {
-			// racing with someone else updating
-			// keep retrying after backoff
-			time.Sleep(backoff)
-			continue
-		}
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("remove old iam file: %w", err)
-		}
-
-		// save copy of data
-		datacopy := make([]byte, len(b))
-		copy(datacopy, b)
-
-		// make a backup copy in case we crash before update
-		// this is after remove, so there is a small window something
-		// can go wrong, but the remove should barrier other gateways
-		// from trying to write backup at the same time. Only one
-		// gateway will successfully remove the file.
-		os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
-
-		b, err = update(b)
-		if err != nil {
-			// update failed, try to write old data back out
-			os.WriteFile(fname, datacopy, iamMode)
-			return fmt.Errorf("update iam data: %w", err)
-		}
-
-		err = s.writeTempFile(b)
-		if err != nil {
-			// update failed, try to write old data back out
-			os.WriteFile(fname, datacopy, iamMode)
-			return err
-		}
-
-		break
+	err = s.writeUsingTempFile(b, iamFname)
+	if err != nil {
+		return fmt.Errorf("write iam file: %w", err)
 	}
 
 	return nil
 }
 
-func (s *IAMServiceInternal) writeTempFile(b []byte) error {
-	fname := filepath.Join(s.dir, iamFile)
-
+func (s *IAMServiceInternal) writeUsingTempFile(b []byte, fname string) error {
 	f, err := os.CreateTemp(s.dir, iamFile)
 	if err != nil {
 		return fmt.Errorf("create temp file: %w", err)
@@ -384,6 +340,7 @@ func (s *IAMServiceInternal) writeTempFile(b []byte) error {
 	defer os.Remove(f.Name())
 
 	_, err = f.Write(b)
+	f.Close()
 	if err != nil {
 		return fmt.Errorf("write temp file: %w", err)
 	}

auth/iam_ipa.go

@@ -30,6 +30,7 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
+	"slices"
 	"strconv"
 	"strings"
 )
@@ -52,7 +53,6 @@ type IpaIAMService struct {
 var _ IAMService = &IpaIAMService{}
 
 func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
-
 	ipa := IpaIAMService{
 		id:        0,
 		version:   IpaVersion,
@@ -72,6 +72,7 @@ func NewIpaIAMService(rootAcc Account, host, vaultName, username, password strin
 	mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
 	tr := &http.Transport{
 		TLSClientConfig: mTLSConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
 	ipa.client = http.Client{Jar: jar, Transport: tr}
 
@@ -102,13 +103,7 @@ func NewIpaIAMService(rootAcc Account, host, vaultName, username, password strin
 
 	ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
 
-	isSupported := false
-	for _, algo := range vaultConfig.Wrapping_supported_algorithms {
-		if algo == "aes-128-cbc" {
-			isSupported = true
-			break
-		}
-	}
+	isSupported := slices.Contains(vaultConfig.Wrapping_supported_algorithms, "aes-128-cbc")
 
 	if !isSupported {
 		return nil,

auth/iam_ldap.go

@@ -139,6 +139,9 @@ func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) e
 	if props.UserID != nil {
 		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
 	}
+	if props.Role != "" {
+		req.Replace(ld.roleAtr, []string{string(props.Role)})
+	}
 
 	err := ld.conn.Modify(req)
 	//TODO: Handle non existing user case

auth/object_lock.go

@@ -136,7 +136,7 @@ func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResu
 	}
 }
 
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass, isBucketPublic bool, be backend.Backend) error {
 	data, err := be.GetObjectLockConfiguration(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -211,7 +211,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 							if err != nil {
 								return err
 							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+							if isBucketPublic {
+								err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
+							} else {
+								err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+							}
 							if err != nil {
 								return s3err.GetAPIError(s3err.ErrObjectLocked)
 							}
@@ -254,7 +258,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 					if err != nil {
 						return err
 					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+					if isBucketPublic {
+						err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
+					} else {
+						err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+					}
 					if err != nil {
 						return s3err.GetAPIError(s3err.ErrObjectLocked)
 					}

backend/azure/azure.go

@@ -181,11 +181,9 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 			return err
 		}
 
-		var acl auth.ACL
-		if len(aclBytes) > 0 {
-			if err := json.Unmarshal(aclBytes, &acl); err != nil {
-				return fmt.Errorf("unmarshal acl: %w", err)
-			}
+		acl, err := auth.ParseACL(aclBytes)
+		if err != nil {
+			return err
 		}
 
 		if acl.Owner == acct.Access {
@@ -295,7 +293,7 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin
 }
 
 func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
-	tags, err := parseTags(po.Tagging)
+	tags, err := backend.ParseObjectTags(getString(po.Tagging))
 	if err != nil {
 		return s3response.PutObjectOutput{}, err
 	}
@@ -418,7 +416,7 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G
 
 	var opts *azblob.DownloadStreamOptions
 	if *input.Range != "" {
-		offset, count, isValid, err := backend.ParseGetObjectRange(*resp.ContentLength, *input.Range)
+		offset, count, isValid, err := backend.ParseObjectRange(*resp.ContentLength, *input.Range)
 		if err != nil {
 			return nil, err
 		}
@@ -507,10 +505,26 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 	if err != nil {
 		return nil, azureErrToS3Err(err)
 	}
+	var size int64
+	if resp.ContentLength != nil {
+		size = *resp.ContentLength
+	}
+
+	startOffset, length, isValid, err := backend.ParseObjectRange(size, getString(input.Range))
+	if err != nil {
+		return nil, err
+	}
+
+	var contentRange string
+	if isValid {
+		contentRange = fmt.Sprintf("bytes %v-%v/%v",
+			startOffset, startOffset+length-1, size)
+	}
 
 	result := &s3.HeadObjectOutput{
-		AcceptRanges:       resp.AcceptRanges,
-		ContentLength:      resp.ContentLength,
+		ContentRange:       &contentRange,
+		AcceptRanges:       backend.GetPtrFromString("bytes"),
+		ContentLength:      &length,
 		ContentType:        resp.ContentType,
 		ContentEncoding:    resp.ContentEncoding,
 		ContentLanguage:    resp.ContentLanguage,
@@ -591,9 +605,9 @@ func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s
 		return s3response.ListObjectsResult{}, azureErrToS3Err(err)
 	}
 
-	var acl auth.ACL
-	if err := json.Unmarshal(aclBytes, &acl); err != nil {
-		return s3response.ListObjectsResult{}, fmt.Errorf("unmarshal acl: %w", err)
+	acl, err := auth.ParseACL(aclBytes)
+	if err != nil {
+		return s3response.ListObjectsResult{}, err
 	}
 
 Pager:
@@ -694,8 +708,9 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input
 			return s3response.ListObjectsV2Result{}, azureErrToS3Err(err)
 		}
 
-		if err := json.Unmarshal(aclBytes, &acl); err != nil {
-			return s3response.ListObjectsV2Result{}, fmt.Errorf("unmarshal acl: %w", err)
+		acl, err = auth.ParseACL(aclBytes)
+		if err != nil {
+			return s3response.ListObjectsV2Result{}, err
 		}
 	}
 
@@ -807,14 +822,14 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
 	}, nil
 }
 
-func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
 	dstClient, err := az.getBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return nil, err
+		return s3response.CopyObjectOutput{}, err
 	}
 	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource {
 		if input.MetadataDirective != types.MetadataDirectiveReplace {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+			return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
 		}
 
 		// Set object meta http headers
@@ -826,7 +841,7 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
 			BlobContentType:        input.ContentType,
 		}, nil)
 		if err != nil {
-			return nil, azureErrToS3Err(err)
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 		}
 
 		meta := input.Metadata
@@ -841,14 +856,14 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
 		// Set object metadata
 		_, err = dstClient.SetMetadata(ctx, parseMetadata(meta), nil)
 		if err != nil {
-			return nil, azureErrToS3Err(err)
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 		}
 
 		// Set object legal hold
 		if input.ObjectLockLegalHoldStatus != "" {
 			err = az.PutObjectLegalHold(ctx, *input.Bucket, *input.Key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn)
 			if err != nil {
-				return nil, azureErrToS3Err(err)
+				return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 			}
 		}
 		// Set object retention
@@ -862,28 +877,28 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
 
 			retParsed, err := json.Marshal(retention)
 			if err != nil {
-				return nil, fmt.Errorf("parse object retention: %w", err)
+				return s3response.CopyObjectOutput{}, fmt.Errorf("parse object retention: %w", err)
 			}
 			err = az.PutObjectRetention(ctx, *input.Bucket, *input.Key, "", true, retParsed)
 			if err != nil {
-				return nil, azureErrToS3Err(err)
+				return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 			}
 		}
 
 		// Set object Tagging, if tagging directive is "REPLACE"
 		if input.TaggingDirective == types.TaggingDirectiveReplace {
-			tags, err := parseTags(input.Tagging)
+			tags, err := backend.ParseObjectTags(getString(input.Tagging))
 			if err != nil {
-				return nil, err
+				return s3response.CopyObjectOutput{}, err
 			}
 			_, err = dstClient.SetTags(ctx, tags, nil)
 			if err != nil {
-				return nil, azureErrToS3Err(err)
+				return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 			}
 		}
 
-		return &s3.CopyObjectOutput{
-			CopyObjectResult: &types.CopyObjectResult{
+		return s3response.CopyObjectOutput{
+			CopyObjectResult: &s3response.CopyObjectResult{
 				LastModified: res.LastModified,
 				ETag:         (*string)(res.ETag),
 			},
@@ -892,13 +907,13 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
 
 	srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
 	if err != nil {
-		return nil, err
+		return s3response.CopyObjectOutput{}, err
 	}
 
 	// Get the source object
 	downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
 	if err != nil {
-		return nil, azureErrToS3Err(err)
+		return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 	}
 
 	pInput := s3response.PutObjectInput{
@@ -936,28 +951,28 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
 	// Create the destination object
 	resp, err := az.PutObject(ctx, pInput)
 	if err != nil {
-		return nil, err
+		return s3response.CopyObjectOutput{}, err
 	}
 
 	// Copy the object tagging, if tagging directive is "COPY"
 	if input.TaggingDirective == types.TaggingDirectiveCopy {
 		srcClient, err := az.getBlobClient(srcBucket, srcObj)
 		if err != nil {
-			return nil, err
+			return s3response.CopyObjectOutput{}, err
 		}
 		res, err := srcClient.GetTags(ctx, nil)
 		if err != nil {
-			return nil, azureErrToS3Err(err)
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 		}
 
 		_, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil)
 		if err != nil {
-			return nil, azureErrToS3Err(err)
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 		}
 	}
 
-	return &s3.CopyObjectOutput{
-		CopyObjectResult: &types.CopyObjectResult{
+	return s3response.CopyObjectOutput{
+		CopyObjectResult: &s3response.CopyObjectResult{
 			ETag: &resp.ETag,
 		},
 	}, nil
@@ -1034,20 +1049,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.Cre
 	}
 
 	// parse object tags
-	tagsStr := getString(input.Tagging)
-	tags := map[string]string{}
-	if tagsStr != "" {
-		tagParts := strings.Split(tagsStr, "&")
-		for _, prt := range tagParts {
-			p := strings.Split(prt, "=")
-			if len(p) != 2 {
-				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTag)
-			}
-			if len(p[0]) > 128 || len(p[1]) > 256 {
-				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTag)
-			}
-			tags[p[0]] = p[1]
-		}
+	tags, err := backend.ParseObjectTags(getString(input.Tagging))
+	if err != nil {
+		return s3response.InitiateMultipartUploadResult{}, err
 	}
 
 	// set blob legal hold status in metadata
@@ -1087,7 +1091,7 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.Cre
 	// Create and empty blob in .sgwtmp/multipart/<uploadId>/<object hash>
 	// The blob indicates multipart upload initialization and holds the mp metadata
 	// e.g tagging, content-type, metadata, object lock status ...
-	_, err := az.client.UploadBuffer(ctx, *input.Bucket, tmpPath, []byte{}, opts)
+	_, err = az.client.UploadBuffer(ctx, *input.Bucket, tmpPath, []byte{}, opts)
 	if err != nil {
 		return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err)
 	}
@@ -1361,42 +1365,44 @@ func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultip
 // Copeies the multipart metadata from .sgwtmp namespace into the newly created blob
 // Deletes the multipart upload 'blob' from .sgwtmp namespace
 // It indicates the end of the multipart upload
-func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+	var res s3response.CompleteMultipartUploadResult
+
 	tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
 	blobClient, err := az.getBlobClient(*input.Bucket, tmpPath)
 	if err != nil {
-		return nil, err
+		return res, "", err
 	}
 
 	props, err := blobClient.GetProperties(ctx, nil)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 	tags, err := blobClient.GetTags(ctx, nil)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return nil, err
+		return res, "", err
 	}
 	blockIds := []string{}
 
 	blockList, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
 	if err != nil {
-		return nil, azureErrToS3Err(err)
+		return res, "", azureErrToS3Err(err)
 	}
 
 	if len(blockList.UncommittedBlocks) != len(input.MultipartUpload.Parts) {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 	}
 
 	uncommittedBlocks := map[int32]*blockblob.Block{}
 	for _, el := range blockList.UncommittedBlocks {
 		ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name))
 		if err != nil {
-			return nil, fmt.Errorf("invalid block name: %w", err)
+			return res, "", fmt.Errorf("invalid block name: %w", err)
 		}
 
 		uncommittedBlocks[int32(ptNumber)] = el
@@ -1408,35 +1414,35 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 	last := len(blockList.UncommittedBlocks) - 1
 	for i, part := range input.MultipartUpload.Parts {
 		if part.PartNumber == nil {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 		if *part.PartNumber < 1 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
 		}
 		if *part.PartNumber <= partNumber {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder)
 		}
 		partNumber = *part.PartNumber
 
 		block, ok := uncommittedBlocks[*part.PartNumber]
 		if !ok {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
 		if *part.ETag != *block.Name {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 		// all parts except the last need to be greater, than
 		// the minimum allowed size (5 Mib)
 		if i < last && *block.Size < backend.MinPartSize {
-			return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
+			return res, "", s3err.GetAPIError(s3err.ErrEntityTooSmall)
 		}
 		totalSize += *block.Size
 		blockIds = append(blockIds, *block.Name)
 	}
 
 	if input.MpuObjectSize != nil && totalSize != *input.MpuObjectSize {
-		return nil, s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize)
+		return res, "", s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize)
 	}
 
 	opts := &blockblob.CommitBlockListOptions{
@@ -1453,20 +1459,20 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 
 	resp, err := client.CommitBlockList(ctx, blockIds, opts)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 
 	// cleanup the multipart upload
 	_, err = blobClient.Delete(ctx, nil)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 
-	return &s3.CompleteMultipartUploadOutput{
+	return s3response.CompleteMultipartUploadResult{
 		Bucket: input.Bucket,
 		Key:    input.Key,
 		ETag:   (*string)(resp.ETag),
-	}, nil
+	}, "", nil
 }
 
 func (az *Azure) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
@@ -1818,24 +1824,6 @@ func parseAzMetadata(m map[string]*string) map[string]string {
 	return meta
 }
 
-func parseTags(tagstr *string) (map[string]string, error) {
-	tagsStr := getString(tagstr)
-	tags := make(map[string]string)
-
-	if tagsStr != "" {
-		tagParts := strings.Split(tagsStr, "&")
-		for _, prt := range tagParts {
-			p := strings.Split(prt, "=")
-			if len(p) != 2 {
-				return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
-			}
-			tags[p[0]] = p[1]
-		}
-	}
-
-	return tags, nil
-}
-
 func parseAzTags(tagSet []*blob.Tags) map[string]string {
 	tags := map[string]string{}
 	for _, tag := range tagSet {
@@ -1976,11 +1964,9 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string
 }
 
 func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
-	var acl auth.ACL
-
 	data, ok := meta[string(key)]
 	if !ok {
-		return &acl, nil
+		return &auth.ACL{}, nil
 	}
 
 	value, err := decodeString(*data)
@@ -1988,13 +1974,9 @@ func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 		return nil, err
 	}
 
-	if len(value) == 0 {
-		return &acl, nil
-	}
-
-	err = json.Unmarshal(value, &acl)
+	acl, err := auth.ParseACL(value)
 	if err != nil {
-		return nil, fmt.Errorf("unmarshal acl: %w", err)
+		return nil, err
 	}
 
 	return &acl, nil

backend/azure/err.go

@@ -40,7 +40,7 @@ func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
 	case "BlobNotFound":
 		return s3err.GetAPIError(s3err.ErrNoSuchKey)
 	case "TagsTooLarge":
-		return s3err.GetAPIError(s3err.ErrInvalidTag)
+		return s3err.GetAPIError(s3err.ErrInvalidTagValue)
 	case "Requested Range Not Satisfiable":
 		return s3err.GetAPIError(s3err.ErrInvalidRange)
 	}

backend/backend.go

@@ -52,7 +52,7 @@ type Backend interface {
 
 	// multipart operations
 	CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
-	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (_ s3response.CompleteMultipartUploadResult, versionid string, _ error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
@@ -65,7 +65,7 @@ type Backend interface {
 	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
-	CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error)
 	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
 	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
@@ -100,10 +100,6 @@ type Backend interface {
 	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
-// InterfaceVersion tracks changes to the Backend interface for plugins.
-// Increment this when the Backend interface changes.
-const InterfaceVersion = 1
-
 type BackendUnsupported struct{}
 
 var _ Backend = &BackendUnsupported{}
@@ -170,8 +166,8 @@ func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) err
 func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+	return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -204,8 +200,8 @@ func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (
 func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
+	return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)

backend/common.go

@@ -17,11 +17,17 @@ package backend
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
+	"hash"
 	"io"
+	"io/fs"
+	"net/url"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -83,11 +89,11 @@ var (
 	errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
 )
 
-// ParseGetObjectRange parses input range header and returns startoffset, length, isValid
+// ParseObjectRange parses input range header and returns startoffset, length, isValid
 // and error. If no endoffset specified, then length is set to the object size
 // for invalid inputs, it returns no error, but isValid=false
 // `InvalidRange` error is returnd, only if startoffset is greater than the object size
-func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
+func ParseObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
 	if acceptRange == "" {
 		return 0, size, false, nil
 	}
@@ -108,15 +114,17 @@ func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, er
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
-	if err != nil {
+	if err != nil && bRange[0] != "" {
 		return 0, size, false, nil
 	}
 
-	if startOffset >= size {
-		return 0, 0, false, errInvalidRange
-	}
-
 	if bRange[1] == "" {
+		if bRange[0] == "" {
+			return 0, size, false, nil
+		}
+		if startOffset >= size {
+			return 0, 0, false, errInvalidRange
+		}
 		return startOffset, size - startOffset, true, nil
 	}
 
@@ -125,12 +133,22 @@ func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, er
 		return 0, size, false, nil
 	}
 
-	if endOffset < startOffset {
+	if startOffset > endOffset {
 		return 0, size, false, nil
 	}
 
+	// for ranges like 'bytes=-100' return the last bytes specified with 'endOffset'
+	if bRange[0] == "" {
+		endOffset = min(endOffset, size)
+		return size - endOffset, endOffset, true, nil
+	}
+
+	if startOffset >= size {
+		return 0, 0, false, errInvalidRange
+	}
+
 	if endOffset >= size {
-		return startOffset, size - startOffset, true, nil
+		endOffset = size - 1
 	}
 
 	return startOffset, endOffset - startOffset + 1, true, nil
@@ -212,27 +230,81 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
 }
 
 // ParseObjectTags parses the url encoded input string into
-// map[string]string key-value tag set
-func ParseObjectTags(t string) (map[string]string, error) {
-	if t == "" {
+// map[string]string with unescaped key/value pair
+func ParseObjectTags(tagging string) (map[string]string, error) {
+	if tagging == "" {
 		return nil, nil
 	}
 
-	tagging := make(map[string]string)
+	tagSet := make(map[string]string)
 
-	tagParts := strings.Split(t, "&")
-	for _, prt := range tagParts {
-		p := strings.Split(prt, "=")
-		if len(p) != 2 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+	for tagging != "" {
+		var tag string
+		tag, tagging, _ = strings.Cut(tagging, "&")
+		// if 'tag' before the first appearance of '&' is empty continue
+		if tag == "" {
+			continue
 		}
-		if len(p[0]) > 128 || len(p[1]) > 256 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+
+		key, value, found := strings.Cut(tag, "=")
+		// if key is empty, but "=" is present, return invalid url ecnoding err
+		if found && key == "" {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
 		}
-		tagging[p[0]] = p[1]
+
+		// return invalid tag key, if the key is longer than 128
+		if len(key) > 128 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// return invalid tag value, if tag value is longer than 256
+		if len(value) > 256 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// query unescape tag key
+		key, err := url.QueryUnescape(key)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// query unescape tag value
+		value, err = url.QueryUnescape(value)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// check tag key to be valid
+		if !isValidTagComponent(key) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// check tag value to be valid
+		if !isValidTagComponent(value) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// duplicate keys are not allowed: return invalid url encoding err
+		_, ok := tagSet[key]
+		if ok {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		tagSet[key] = value
 	}
 
-	return tagging, nil
+	return tagSet, nil
+}
+
+var validTagComponent = regexp.MustCompile(`^[a-zA-Z0-9:/_.\-+ ]+$`)
+
+// isValidTagComponent matches strings which contain letters, decimal digits,
+// and special chars: '/', '_', '-', '+', '.', ' ' (space)
+func isValidTagComponent(str string) bool {
+	if str == "" {
+		return true
+	}
+	return validTagComponent.Match([]byte(str))
 }
 
 func GetMultipartMD5(parts []types.CompletedPart) string {
@@ -269,3 +341,65 @@ func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
 func (f *FileSectionReadCloser) Close() error {
 	return f.F.Close()
 }
+
+// MoveFile moves a file from source to destination.
+func MoveFile(source, destination string, perm os.FileMode) error {
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	err := os.Rename(source, destination)
+	if err == nil || !errors.Is(err, syscall.EXDEV) {
+		return err
+	}
+
+	// Rename can fail if the source and destination are not on the same
+	// filesystem. The fallback is to copy the file and then remove the source.
+	// We need to be careful that the desination does not exist before copying
+	// to prevent any other simultaneous writes to the file.
+	sourceFile, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("open source: %w", err)
+	}
+	defer sourceFile.Close()
+
+	var destFile *os.File
+	for {
+		destFile, err = os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+		if err != nil {
+			if errors.Is(err, fs.ErrExist) {
+				if removeErr := os.Remove(destination); removeErr != nil {
+					return fmt.Errorf("remove existing destination: %w", removeErr)
+				}
+				continue
+			}
+			return fmt.Errorf("create destination: %w", err)
+		}
+		break
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return fmt.Errorf("copy data: %w", err)
+	}
+
+	err = os.Remove(source)
+	if err != nil {
+		return fmt.Errorf("remove source: %w", err)
+	}
+
+	return nil
+}
+
+// GenerateEtag generates a new quoted etag from the provided hash.Hash
+func GenerateEtag(h hash.Hash) string {
+	dataSum := h.Sum(nil)
+	return fmt.Sprintf("\"%s\"", hex.EncodeToString(dataSum[:]))
+}
+
+// AreEtagsSame compares 2 etags by ignoring quotes
+func AreEtagsSame(e1, e2 string) bool {
+	return strings.Trim(e1, `"`) == strings.Trim(e2, `"`)
+}

backend/plugin/plugin.go

@@ -1,516 +0,0 @@
-// Copyright 2025 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package vgwplugin
-
-import (
-	"bufio"
-	"context"
-	"fmt"
-	"plugin"
-	"reflect"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3response"
-)
-
-// The plugin backend is used to dynamically load a Go plugin at runtime.
-// It loads the plugin and calls the InitPlugin function to initialize it.
-// A config string option is passed to init the plugin, it is expected that the
-// plugin will handle its own configuration and initialization from this.
-// If the plugin cannot be loaded or initialized, it returns an error.
-// The InitPlugin function should be defined in the plugin and should have
-// the signature func(configfile string) (version int, err error).
-// The plugin should also implement the backend.Backend interface functions.
-// However, the plugin does not need to implement all functions of the
-// backend.Backend interface. It can implement only the functions it needs.
-// Any non-implemented functions will return an error indicating that
-// the function is not implemented.
-// The plugin file should be compiled with the same Go version as the
-// application using it. The plugin file should be built with the
-// -buildmode=plugin flag.
-// Example: go build -buildmode=plugin -o myplugin.so myplugin.go
-// See the following for caveats and details:
-// https://pkg.go.dev/plugin#hdr-Warnings
-
-// PluginBackend implements the backend.Backend interface using Go plugins.
-type PluginBackend struct {
-	p *plugin.Plugin
-}
-
-// NewPluginBackend creates a new PluginBackend. The path parameter should
-// point to the compiled plugin file (e.g., .so file).
-func NewPluginBackend(path, config string) (*PluginBackend, error) {
-	p, err := plugin.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open plugin: %w", err)
-	}
-
-	initSymbol, err := p.Lookup("InitPlugin")
-	if err != nil {
-		return nil, fmt.Errorf("failed to lookup InitPlugin symbol: %w", err)
-	}
-
-	initFunc, ok := initSymbol.(func(string) (int, error))
-	if !ok {
-		return nil, fmt.Errorf("InitPlugin symbol is not a func() (int, error)")
-	}
-
-	version, err := initFunc(config)
-	if err != nil {
-		return nil, fmt.Errorf("InitPlugin failed: %w", err)
-	}
-
-	if version != backend.InterfaceVersion {
-		return nil, fmt.Errorf("plugin interface version mismatch: gateway %v, plugin %v",
-			backend.InterfaceVersion, version)
-	}
-
-	return &PluginBackend{p: p}, nil
-}
-
-func (p *PluginBackend) callPluginFunc(name string, args []any) ([]reflect.Value, error) {
-	symbol, err := p.p.Lookup(name)
-	if err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-	}
-
-	symbolValue := reflect.ValueOf(symbol)
-	if symbolValue.Kind() != reflect.Func {
-		return nil, fmt.Errorf("symbol %s is not a function", name)
-	}
-
-	numIn := symbolValue.Type().NumIn()
-	if len(args) != numIn {
-		return nil, fmt.Errorf("incorrect number of arguments for function %s, expected %d, got %d", name, numIn, len(args))
-	}
-
-	in := make([]reflect.Value, len(args))
-	for i := range args {
-		in[i] = reflect.ValueOf(args[i])
-	}
-
-	return symbolValue.Call(in), nil
-}
-
-func (p *PluginBackend) String() string { return "Plugin Gateway" }
-func (p *PluginBackend) Shutdown()      {}
-
-func (p *PluginBackend) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
-	results, err := p.callPluginFunc("ListBuckets", []any{ctx, input})
-	if err != nil {
-		return s3response.ListAllMyBucketsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListAllMyBucketsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
-	results, err := p.callPluginFunc("HeadBucket", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.HeadBucketOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
-	results, err := p.callPluginFunc("GetBucketAcl", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, defaultACL []byte) error {
-	_, err := p.callPluginFunc("CreateBucket", []any{ctx, input, defaultACL})
-	return err
-}
-
-func (p *PluginBackend) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
-	_, err := p.callPluginFunc("PutBucketAcl", []any{ctx, bucket, data})
-	return err
-}
-
-func (p *PluginBackend) DeleteBucket(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucket", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
-	_, err := p.callPluginFunc("PutBucketVersioning", []any{ctx, bucket, status})
-	return err
-}
-
-func (p *PluginBackend) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
-	results, err := p.callPluginFunc("GetBucketVersioning", []any{ctx, bucket})
-	if err != nil {
-		return s3response.GetBucketVersioningOutput{}, err
-	}
-
-	return results[0].Interface().(s3response.GetBucketVersioningOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
-	_, err := p.callPluginFunc("PutBucketPolicy", []any{ctx, bucket, policy})
-	return err
-}
-
-func (p *PluginBackend) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetBucketPolicy", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteBucketPolicy(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketPolicy", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
-	_, err := p.callPluginFunc("PutBucketOwnershipControls", []any{ctx, bucket, ownership})
-	return err
-}
-
-func (p *PluginBackend) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
-	results, err := p.callPluginFunc("GetBucketOwnershipControls", []any{ctx, bucket})
-	if err != nil {
-		return "", err
-	}
-
-	return results[0].Interface().(types.ObjectOwnership), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketOwnershipControls", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) PutBucketCors(ctx context.Context, data []byte) error {
-	_, err := p.callPluginFunc("PutBucketCors", []any{ctx, data})
-	return err
-}
-
-func (p *PluginBackend) GetBucketCors(ctx context.Context, bucket string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetBucketCors", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteBucketCors(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketCors", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
-	results, err := p.callPluginFunc("CreateMultipartUpload", []any{ctx, input})
-	if err != nil {
-		return s3response.InitiateMultipartUploadResult{}, err
-	}
-
-	return results[0].Interface().(s3response.InitiateMultipartUploadResult), convertError(results[1])
-}
-
-func (p *PluginBackend) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	results, err := p.callPluginFunc("CompleteMultipartUpload", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.CompleteMultipartUploadOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
-	_, err := p.callPluginFunc("AbortMultipartUpload", []any{ctx, input})
-	return err
-}
-
-func (p *PluginBackend) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
-	results, err := p.callPluginFunc("ListMultipartUploads", []any{ctx, input})
-	if err != nil {
-		return s3response.ListMultipartUploadsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListMultipartUploadsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
-	results, err := p.callPluginFunc("ListParts", []any{ctx, input})
-	if err != nil {
-		return s3response.ListPartsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListPartsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
-	results, err := p.callPluginFunc("UploadPart", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.UploadPartOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
-	results, err := p.callPluginFunc("UploadPartCopy", []any{ctx, input})
-	if err != nil {
-		return s3response.CopyPartResult{}, err
-	}
-
-	return results[0].Interface().(s3response.CopyPartResult), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObject(ctx context.Context, input s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
-	results, err := p.callPluginFunc("PutObject", []any{ctx, input})
-	if err != nil {
-		return s3response.PutObjectOutput{}, err
-	}
-
-	return results[0].Interface().(s3response.PutObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	results, err := p.callPluginFunc("HeadObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.HeadObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
-	results, err := p.callPluginFunc("GetObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.GetObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetObjectAcl(ctx context.Context, input *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
-	results, err := p.callPluginFunc("GetObjectAcl", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.GetObjectAclOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
-	results, err := p.callPluginFunc("GetObjectAttributes", []any{ctx, input})
-	if err != nil {
-		return s3response.GetObjectAttributesResponse{}, err
-	}
-
-	return results[0].Interface().(s3response.GetObjectAttributesResponse), convertError(results[1])
-}
-
-func (p *PluginBackend) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	results, err := p.callPluginFunc("CopyObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.CopyObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
-	results, err := p.callPluginFunc("ListObjects", []any{ctx, input})
-	if err != nil {
-		return s3response.ListObjectsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListObjectsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
-	results, err := p.callPluginFunc("ListObjectsV2", []any{ctx, input})
-	if err != nil {
-		return s3response.ListObjectsV2Result{}, err
-	}
-
-	return results[0].Interface().(s3response.ListObjectsV2Result), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
-	results, err := p.callPluginFunc("DeleteObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.DeleteObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
-	results, err := p.callPluginFunc("DeleteObjects", []any{ctx, input})
-	if err != nil {
-		return s3response.DeleteResult{}, err
-	}
-
-	return results[0].Interface().(s3response.DeleteResult), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectAcl(ctx context.Context, input *s3.PutObjectAclInput) error {
-	_, err := p.callPluginFunc("PutObjectAcl", []any{ctx, input})
-	return err
-}
-
-func (p *PluginBackend) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
-	results, err := p.callPluginFunc("ListObjectVersions", []any{ctx, input})
-	if err != nil {
-		return s3response.ListVersionsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListVersionsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) RestoreObject(ctx context.Context, input *s3.RestoreObjectInput) error {
-	_, err := p.callPluginFunc("RestoreObject", []any{ctx, input})
-	return err
-}
-
-func (p *PluginBackend) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
-	results, err := p.callPluginFunc("SelectObjectContent", []any{ctx, input})
-	if err != nil {
-		return func(w *bufio.Writer) {}
-	}
-
-	return results[0].Interface().(func(w *bufio.Writer))
-}
-
-func (p *PluginBackend) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) {
-	results, err := p.callPluginFunc("GetBucketTagging", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(map[string]string), convertError(results[1])
-}
-
-func (p *PluginBackend) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
-	_, err := p.callPluginFunc("PutBucketTagging", []any{ctx, bucket, tags})
-	return err
-}
-
-func (p *PluginBackend) DeleteBucketTagging(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketTagging", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
-	results, err := p.callPluginFunc("GetObjectTagging", []any{ctx, bucket, object})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(map[string]string), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
-	_, err := p.callPluginFunc("PutObjectTagging", []any{ctx, bucket, object, tags})
-	return err
-}
-
-func (p *PluginBackend) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
-	_, err := p.callPluginFunc("DeleteObjectTagging", []any{ctx, bucket, object})
-	return err
-}
-
-func (p *PluginBackend) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
-	_, err := p.callPluginFunc("PutObjectLockConfiguration", []any{ctx, bucket, config})
-	return err
-}
-
-func (p *PluginBackend) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetObjectLockConfiguration", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	_, err := p.callPluginFunc("PutObjectRetention", []any{ctx, bucket, object, versionId, bypass, retention})
-	return err
-}
-
-func (p *PluginBackend) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetObjectRetention", []any{ctx, bucket, object, versionId})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
-	_, err := p.callPluginFunc("PutObjectLegalHold", []any{ctx, bucket, object, versionId, status})
-	return err
-}
-
-func (p *PluginBackend) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
-	results, err := p.callPluginFunc("GetObjectLegalHold", []any{ctx, bucket, object, versionId})
-	if err != nil {
-		return nil, err
-	}
-
-	val := results[0].Interface()
-
-	if val == nil {
-		return nil, convertError(results[1])
-	}
-
-	return val.(*bool), convertError(results[1])
-}
-
-func (p *PluginBackend) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
-	_, err := p.callPluginFunc("ChangeBucketOwner", []any{ctx, bucket, acl})
-	return err
-}
-
-func (p *PluginBackend) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
-	results, err := p.callPluginFunc("ListBucketsAndOwners", []any{ctx})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]s3response.Bucket), convertError(results[1])
-}
-
-func convertError(result reflect.Value) error {
-	if result.IsNil() {
-		return nil
-	}
-
-	err, ok := result.Interface().(error)
-	if !ok {
-		return fmt.Errorf("expected error, got %T", result.Interface())
-	}
-
-	return err
-}
-
-var _ backend.Backend = &PluginBackend{}

backend/posix/with_otmpfile.go

@@ -52,9 +52,13 @@ var (
 	defaultFilePerm uint32 = 0644
 )
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool, forceNoTmpFile bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
+	if forceNoTmpFile {
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
+	}
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
@@ -68,37 +72,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		}
 
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
-		if err != nil {
-			return nil, fmt.Errorf("make temp dir: %w", err)
-		}
-		f, err := os.CreateTemp(dir,
-			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-		if err != nil {
-			return nil, err
-		}
-		tmp := &tmpfile{
-			f:          f,
-			bucket:     bucket,
-			objname:    obj,
-			size:       size,
-			needsChown: doChown,
-			uid:        uid,
-			gid:        gid,
-		}
-		// falloc is best effort, its fine if this fails
-		if size > 0 && dofalloc {
-			tmp.falloc()
-		}
-
-		if doChown {
-			err := f.Chown(uid, gid)
-			if err != nil {
-				return nil, fmt.Errorf("set temp file ownership: %w", err)
-			}
-		}
-
-		return tmp, nil
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
@@ -132,6 +106,46 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	return tmp, nil
 }
 
+func (p *Posix) openMkTemp(dir, bucket, obj string, size int64, dofalloc bool, uid, gid int, doChown bool) (*tmpfile, error) {
+	err := backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, err
+	}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+	// falloc is best effort, its fine if this fails
+	if size > 0 && dofalloc {
+		tmp.falloc()
+	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
+	return tmp, nil
+}
+
 func (tmp *tmpfile) falloc() error {
 	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
 	if err != nil {
@@ -222,7 +236,9 @@ func (tmp *tmpfile) fallbackLink() error {
 	objPath := filepath.Join(tmp.bucket, tmp.objname)
 	err = os.Rename(tempname, objPath)
 	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
+		// rename only works for files within the same filesystem
+		// if this fails fallback to copy
+		return backend.MoveFile(tempname, objPath, fs.FileMode(defaultFilePerm))
 	}
 
 	return nil

backend/posix/without_otmpfile.go

@@ -38,7 +38,7 @@ type tmpfile struct {
 	size    int64
 }
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool, _ bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
 	// Create a temp file for upload while in progress (see link comments below).
@@ -80,31 +80,17 @@ func (tmp *tmpfile) link() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
-	// We use Rename as the atomic operation for object puts. The upload is
-	// written to a temp file to not conflict with any other simultaneous
-	// uploads. The final operation is to move the temp file into place for
-	// the object. This ensures the object semantics of last upload completed
-	// wins and is not some combination of writes from simultaneous uploads.
 	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err := os.Remove(objPath)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove stale path: %w", err)
-	}
 
 	// reset default file mode because CreateTemp uses 0600
 	tmp.f.Chmod(defaultFilePerm)
 
-	err = tmp.f.Close()
+	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)
 	}
 
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
+	return backend.MoveFile(tempname, objPath, defaultFilePerm)
 }
 
 func (tmp *tmpfile) Write(b []byte) (int, error) {

backend/s3proxy/client.go

@@ -36,6 +36,11 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
 	if s.endpoint != "" {
 		return s3.NewFromConfig(cfg, func(o *s3.Options) {
 			o.BaseEndpoint = &s.endpoint
+			o.UsePathStyle = s.usePathStyle
+			// The http body stream is not seekable, so most operations cannot
+			// be retried. The error returned to the original client may be
+			// retried by the client.
+			o.Retryer = aws.NopRetryer{}
 		}), nil
 	}
 

backend/s3proxy/s3.go

@@ -15,9 +15,9 @@
 package s3proxy
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
-	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -25,7 +25,6 @@ import (
 	"io"
 	"net/http"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -40,7 +39,12 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
-const aclKey string = "versitygwAcl"
+type metaPrefix string
+
+const (
+	metaPrefixAcl    metaPrefix = "vgw-meta-acl-"
+	metaPrefixPolicy metaPrefix = "vgw-meta-policy-"
+)
 
 type S3Proxy struct {
 	backend.BackendUnsupported
@@ -51,29 +55,48 @@ type S3Proxy struct {
 	secret          string
 	endpoint        string
 	awsRegion       string
+	metaBucket      string
 	disableChecksum bool
 	sslSkipVerify   bool
+	usePathStyle    bool
 	debug           bool
 }
 
 var _ backend.Backend = &S3Proxy{}
 
-func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
+func NewWithClient(ctx context.Context, client *s3.Client, metaBucket string) (*S3Proxy, error) {
+	s := &S3Proxy{
+		metaBucket: metaBucket,
+	}
+	s.client = client
+	return s, s.validate(ctx)
+}
+
+func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, disableChecksum, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
 	s := &S3Proxy{
 		access:          access,
 		secret:          secret,
 		endpoint:        endpoint,
 		awsRegion:       region,
+		metaBucket:      metaBucket,
 		disableChecksum: disableChecksum,
 		sslSkipVerify:   sslSkipVerify,
+		usePathStyle:    usePathStyle,
 		debug:           debug,
 	}
-	client, err := s.getClientWithCtx(context.Background())
+	client, err := s.getClientWithCtx(ctx)
 	if err != nil {
 		return nil, err
 	}
 	s.client = client
-	return s, nil
+	return s, s.validate(ctx)
+}
+
+func (s *S3Proxy) validate(ctx context.Context) error {
+	if s.metaBucket != "" && !s.bucketExists(ctx, s.metaBucket) {
+		return fmt.Errorf("the provided meta bucket doesn't exist")
+	}
+	return nil
 }
 
 func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
@@ -88,10 +111,33 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsI
 
 	var buckets []s3response.ListAllMyBucketsEntry
 	for _, b := range output.Buckets {
-		buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-			Name:         *b.Name,
-			CreationDate: *b.CreationDate,
-		})
+		if *b.Name == s.metaBucket {
+			continue
+		}
+		if input.IsAdmin || s.metaBucket == "" {
+			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+				Name:         *b.Name,
+				CreationDate: *b.CreationDate,
+			})
+			continue
+		}
+
+		data, err := s.getMetaBucketObjData(ctx, *b.Name, metaPrefixAcl, false)
+		if err != nil {
+			return s3response.ListAllMyBucketsResult{}, handleError(err)
+		}
+
+		acl, err := auth.ParseACL(data)
+		if err != nil {
+			return s3response.ListAllMyBucketsResult{}, err
+		}
+
+		if acl.Owner == input.Owner {
+			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+				Name:         *b.Name,
+				CreationDate: *b.CreationDate,
+			})
+		}
 	}
 
 	return s3response.ListAllMyBucketsResult{
@@ -130,27 +176,52 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 	if input.GrantWriteACP != nil && *input.GrantWriteACP == "" {
 		input.GrantWriteACP = nil
 	}
+	if *input.Bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
+	if s.metaBucket != "" {
+		data, err := s.getMetaBucketObjData(ctx, *input.Bucket, metaPrefixAcl, true)
+		if err == nil {
+			acl, err := auth.ParseACL(data)
+			if err != nil {
+				return err
+			}
+
+			if acl.Owner == acct.Access {
+				return s3err.GetAPIError(s3err.ErrBucketAlreadyOwnedByYou)
+			}
+			return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
+		}
+	}
+
 	_, err := s.client.CreateBucket(ctx, input)
 	if err != nil {
 		return handleError(err)
 	}
 
-	var tagSet []types.Tag
-	tagSet = append(tagSet, types.Tag{
-		Key:   backend.GetPtrFromString(aclKey),
-		Value: backend.GetPtrFromString(base64Encode(acl)),
-	})
+	// Store bucket default acl
+	if s.metaBucket != "" {
+		err = s.putMetaBucketObj(ctx, *input.Bucket, acl, metaPrefixAcl)
+		if err != nil {
+			// attempt to cleanup
+			_ = s.DeleteBucket(ctx, *input.Bucket)
+			return handleError(err)
+		}
+	}
 
-	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
-		Bucket: input.Bucket,
-		Tagging: &types.Tagging{
-			TagSet: tagSet,
-		},
-	})
-	return handleError(err)
+	return nil
 }
 
 func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
+	if bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
 		Bucket: &bucket,
 	})
@@ -158,6 +229,9 @@ func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
 }
 
 func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
+	if bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	_, err := s.client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
 		Bucket: &bucket,
 		OwnershipControls: &types.OwnershipControls{
@@ -172,6 +246,9 @@ func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string,
 }
 
 func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
+	if bucket == s.metaBucket {
+		return "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	var ownship types.ObjectOwnership
 	resp, err := s.client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
 		Bucket: &bucket,
@@ -182,6 +259,9 @@ func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string)
 	return resp.OwnershipControls.Rules[0].ObjectOwnership, nil
 }
 func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
+	if bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	_, err := s.client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
 		Bucket: &bucket,
 	})
@@ -189,6 +269,9 @@ func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket stri
 }
 
 func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
+	if bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	_, err := s.client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
 		Bucket: &bucket,
 		VersioningConfiguration: &types.VersioningConfiguration{
@@ -200,6 +283,9 @@ func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status
 }
 
 func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	if bucket == s.metaBucket {
+		return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
 		Bucket: &bucket,
 	})
@@ -211,6 +297,9 @@ func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3res
 }
 
 func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.Delimiter != nil && *input.Delimiter == "" {
 		input.Delimiter = nil
 	}
@@ -248,13 +337,16 @@ func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVe
 		NextVersionIdMarker: out.NextVersionIdMarker,
 		Prefix:              out.Prefix,
 		VersionIdMarker:     input.VersionIdMarker,
-		Versions:            out.Versions,
+		Versions:            convertObjectVersions(out.Versions),
 	}, nil
 }
 
 var defTime = time.Time{}
 
 func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.CacheControl != nil && *input.CacheControl == "" {
 		input.CacheControl = nil
 	}
@@ -365,7 +457,12 @@ func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input s3response.Cr
 	}, nil
 }
 
-func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+	var res s3response.CompleteMultipartUploadResult
+
+	if *input.Bucket == s.metaBucket {
+		return res, "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 == "" {
 		input.ChecksumCRC32 = nil
 	}
@@ -403,11 +500,33 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		input.SSECustomerKeyMD5 = nil
 	}
 
+	var versionid string
 	out, err := s.client.CompleteMultipartUpload(ctx, input)
-	return out, handleError(err)
+	if out != nil {
+		res = s3response.CompleteMultipartUploadResult{
+			Location:          out.Location,
+			Bucket:            out.Bucket,
+			Key:               out.Key,
+			ETag:              out.ETag,
+			ChecksumCRC32:     out.ChecksumCRC32,
+			ChecksumCRC32C:    out.ChecksumCRC32C,
+			ChecksumCRC64NVME: out.ChecksumCRC64NVME,
+			ChecksumSHA1:      out.ChecksumSHA1,
+			ChecksumSHA256:    out.ChecksumSHA256,
+			ChecksumType:      &out.ChecksumType,
+		}
+		if out.VersionId != nil {
+			versionid = *out.VersionId
+		}
+	}
+
+	return res, versionid, handleError(err)
 }
 
 func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	if *input.Bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
 		input.ExpectedBucketOwner = nil
 	}
@@ -419,6 +538,9 @@ func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMulti
 }
 
 func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.Delimiter != nil && *input.Delimiter == "" {
 		input.Delimiter = nil
 	}
@@ -487,6 +609,9 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
 }
 
 func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
 		input.ExpectedBucketOwner = nil
 	}
@@ -561,6 +686,9 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
 }
 
 func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	if *input.Bucket == s.metaBucket {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 == "" {
 		input.ChecksumCRC32 = nil
 	}
@@ -601,6 +729,9 @@ func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s
 }
 
 func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.CopySourceIfMatch != nil && *input.CopySourceIfMatch == "" {
 		input.CopySourceIfMatch = nil
 	}
@@ -658,6 +789,9 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
 }
 
 func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.CacheControl != nil && *input.CacheControl == "" {
 		input.CacheControl = nil
 	}
@@ -812,6 +946,9 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
 }
 
 func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	if *input.Bucket == s.metaBucket {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
 		input.ExpectedBucketOwner = nil
 	}
@@ -869,6 +1006,9 @@ func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 }
 
 func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	if *input.Bucket == s.metaBucket {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
 		input.ExpectedBucketOwner = nil
 	}
@@ -930,6 +1070,9 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
 }
 
 func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
 		input.ExpectedBucketOwner = nil
 	}
@@ -988,7 +1131,10 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
 	}, handleError(err)
 }
 
-func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.CacheControl != nil && *input.CacheControl == "" {
 		input.CacheControl = nil
 	}
@@ -1123,10 +1269,39 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInp
 			StorageClass:                   input.StorageClass,
 			TaggingDirective:               input.TaggingDirective,
 		})
-	return out, handleError(err)
+	if err != nil {
+		return s3response.CopyObjectOutput{}, handleError(err)
+	}
+	if out.CopyObjectResult == nil {
+		out.CopyObjectResult = &types.CopyObjectResult{}
+	}
+	return s3response.CopyObjectOutput{
+		BucketKeyEnabled: out.BucketKeyEnabled,
+		CopyObjectResult: &s3response.CopyObjectResult{
+			ChecksumCRC32:     out.CopyObjectResult.ChecksumCRC32,
+			ChecksumCRC32C:    out.CopyObjectResult.ChecksumCRC32C,
+			ChecksumCRC64NVME: out.CopyObjectResult.ChecksumCRC64NVME,
+			ChecksumSHA1:      out.CopyObjectResult.ChecksumSHA1,
+			ChecksumSHA256:    out.CopyObjectResult.ChecksumSHA256,
+			ChecksumType:      out.CopyObjectResult.ChecksumType,
+			ETag:              out.CopyObjectResult.ETag,
+			LastModified:      out.CopyObjectResult.LastModified,
+		},
+		CopySourceVersionId:     out.CopySourceVersionId,
+		Expiration:              out.Expiration,
+		SSECustomerAlgorithm:    out.SSECustomerAlgorithm,
+		SSECustomerKeyMD5:       out.SSECustomerKeyMD5,
+		SSEKMSEncryptionContext: out.SSEKMSEncryptionContext,
+		SSEKMSKeyId:             out.SSEKMSKeyId,
+		ServerSideEncryption:    out.ServerSideEncryption,
+		VersionId:               out.VersionId,
+	}, handleError(err)
 }
 
 func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.Delimiter != nil && *input.Delimiter == "" {
 		input.Delimiter = nil
 	}
@@ -1164,6 +1339,9 @@ func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
 }
 
 func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ContinuationToken != nil && *input.ContinuationToken == "" {
 		input.ContinuationToken = nil
 	}
@@ -1205,6 +1383,9 @@ func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
 }
 
 func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if *input.Bucket == s.metaBucket {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
 		input.ExpectedBucketOwner = nil
 	}
@@ -1229,6 +1410,9 @@ func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput)
 }
 
 func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+	if *input.Bucket == s.metaBucket {
+		return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
 		input.ExpectedBucketOwner = nil
 	}
@@ -1252,77 +1436,22 @@ func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInpu
 }
 
 func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
-	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
-		input.ExpectedBucketOwner = nil
-	}
-
-	tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
-		Bucket: input.Bucket,
-	})
+	data, err := s.getMetaBucketObjData(ctx, *input.Bucket, metaPrefixAcl, false)
 	if err != nil {
-		var ae smithy.APIError
-		if errors.As(err, &ae) {
-			// sdk issue workaround for missing NoSuchTagSet error type
-			// https://github.com/aws/aws-sdk-go-v2/issues/2878
-			if strings.Contains(ae.ErrorCode(), "NoSuchTagSet") {
-				return []byte{}, nil
-			}
-			if strings.Contains(ae.ErrorCode(), "NotImplemented") {
-				return []byte{}, nil
-			}
-		}
 		return nil, handleError(err)
 	}
 
-	for _, tag := range tagout.TagSet {
-		if *tag.Key == aclKey {
-			acl, err := base64Decode(*tag.Value)
-			if err != nil {
-				return nil, handleError(err)
-			}
-			return acl, nil
-		}
-	}
-
-	return []byte{}, nil
+	return data, nil
 }
 
 func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
-	tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
-		Bucket: &bucket,
-	})
-	if err != nil {
-		return handleError(err)
-	}
-
-	var found bool
-	for i, tag := range tagout.TagSet {
-		if *tag.Key == aclKey {
-			tagout.TagSet[i] = types.Tag{
-				Key:   backend.GetPtrFromString(aclKey),
-				Value: backend.GetPtrFromString(base64Encode(data)),
-			}
-			found = true
-			break
-		}
-	}
-	if !found {
-		tagout.TagSet = append(tagout.TagSet, types.Tag{
-			Key:   backend.GetPtrFromString(aclKey),
-			Value: backend.GetPtrFromString(base64Encode(data)),
-		})
-	}
-
-	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
-		Bucket: &bucket,
-		Tagging: &types.Tagging{
-			TagSet: tagout.TagSet,
-		},
-	})
-	return handleError(err)
+	return handleError(s.putMetaBucketObj(ctx, bucket, data, metaPrefixAcl))
 }
 
 func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	if bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	tagging := &types.Tagging{
 		TagSet: []types.Tag{},
 	}
@@ -1342,6 +1471,9 @@ func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, t
 }
 
 func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	if bucket == s.metaBucket {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
 		Bucket: &bucket,
 		Key:    &object,
@@ -1359,6 +1491,9 @@ func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (
 }
 
 func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	if bucket == s.metaBucket {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
 	_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
 		Bucket: &bucket,
 		Key:    &object,
@@ -1367,34 +1502,29 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
 }
 
 func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
-	_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
-		Bucket: &bucket,
-		Policy: backend.GetPtrFromString(string(policy)),
-	})
-	return handleError(err)
+	return handleError(s.putMetaBucketObj(ctx, bucket, policy, metaPrefixPolicy))
 }
 
 func (s *S3Proxy) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
-	policy, err := s.client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
-		Bucket: &bucket,
-	})
+	data, err := s.getMetaBucketObjData(ctx, bucket, metaPrefixPolicy, false)
 	if err != nil {
 		return nil, handleError(err)
 	}
 
-	result := []byte{}
-	if policy.Policy != nil {
-		result = []byte(*policy.Policy)
-	}
-
-	return result, nil
+	return data, nil
 }
 
 func (s *S3Proxy) DeleteBucketPolicy(ctx context.Context, bucket string) error {
-	_, err := s.client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
-		Bucket: &bucket,
+	key := getMetaKey(bucket, metaPrefixPolicy)
+	_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: &s.metaBucket,
+		Key:    &key,
 	})
-	return handleError(err)
+	if err != nil && !areErrSame(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+		return handleError(err)
+	}
+
+	return nil
 }
 
 func (s *S3Proxy) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
@@ -1423,10 +1553,11 @@ func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versio
 }
 
 func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
-	var acll auth.ACL
-	if err := json.Unmarshal(acl, &acll); err != nil {
-		return fmt.Errorf("unmarshal acl: %w", err)
+	acll, err := auth.ParseACL(acl)
+	if err != nil {
+		return err
 	}
+
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, acll.Owner), nil)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
@@ -1502,6 +1633,102 @@ func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket
 	return buckets, nil
 }
 
+func (s *S3Proxy) bucketExists(ctx context.Context, bucket string) bool {
+	_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
+		Bucket: &bucket,
+	})
+	return err == nil
+}
+
+func (s *S3Proxy) putMetaBucketObj(ctx context.Context, bucket string, data []byte, prefix metaPrefix) error {
+	// if meta bucket is not provided, return successful response
+	if s.metaBucket == "" {
+		return nil
+	}
+
+	key := getMetaKey(bucket, prefix)
+	// store the provided bucket acl/policy as an object in meta bucket
+	_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: &s.metaBucket,
+		Key:    &key,
+		Body:   bytes.NewReader(data),
+	})
+	return err
+}
+
+// set checkExists to true if using to check for existence of bucket, in
+// this case it will not return default acl/policy if the metadata does
+// not exist
+func (s *S3Proxy) getMetaBucketObjData(ctx context.Context, bucket string, prefix metaPrefix, checkExists bool) ([]byte, error) {
+	// return default bahviour of get bucket policy/acl, if meta bucket is not provided
+	if s.metaBucket == "" {
+		switch prefix {
+		case metaPrefixAcl:
+			return []byte{}, nil
+		case metaPrefixPolicy:
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)
+		}
+	}
+
+	key := getMetaKey(bucket, prefix)
+	// get meta bucket object
+	res, err := s.client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: &s.metaBucket,
+		Key:    &key,
+	})
+	if areErrSame(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+		if checkExists {
+			return nil, err
+		}
+
+		switch prefix {
+		case metaPrefixAcl:
+			// If bucket acl is not found, return default acl
+			return []byte{}, nil
+		case metaPrefixPolicy:
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read meta object data: %w", err)
+	}
+
+	return data, nil
+}
+
+// Checks if the provided err is a type of smithy.APIError
+// and if the error code and message match with the provided apiErr
+func areErrSame(err error, apiErr s3err.APIError) bool {
+	if err == nil {
+		return false
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() != apiErr.Code {
+			return false
+		}
+
+		// 404 errors are not well serialized by aws-sdk-go-v2
+		if ae.ErrorCode() != "NoSuchKey" && ae.ErrorMessage() != apiErr.Description {
+			return false
+		}
+
+		return true
+	}
+
+	return false
+}
+
+// generates meta object key with bucket name and meta prefix
+func getMetaKey(bucket string, prefix metaPrefix) string {
+	return string(prefix) + bucket
+}
+
 func handleError(err error) error {
 	if err == nil {
 		return nil
@@ -1522,18 +1749,6 @@ func handleError(err error) error {
 	return err
 }
 
-func base64Encode(input []byte) string {
-	return base64.StdEncoding.EncodeToString(input)
-}
-
-func base64Decode(encoded string) ([]byte, error) {
-	decoded, err := base64.StdEncoding.DecodeString(encoded)
-	if err != nil {
-		return nil, err
-	}
-	return decoded, nil
-}
-
 func convertObjects(objs []types.Object) []s3response.Object {
 	result := make([]s3response.Object, 0, len(objs))
 
@@ -1553,3 +1768,24 @@ func convertObjects(objs []types.Object) []s3response.Object {
 
 	return result
 }
+
+func convertObjectVersions(versions []types.ObjectVersion) []s3response.ObjectVersion {
+	result := make([]s3response.ObjectVersion, 0, len(versions))
+	for _, v := range versions {
+		result = append(result, s3response.ObjectVersion{
+			ChecksumAlgorithm: v.ChecksumAlgorithm,
+			ChecksumType:      v.ChecksumType,
+			ETag:              v.ETag,
+			IsLatest:          v.IsLatest,
+			Key:               v.Key,
+			LastModified:      v.LastModified,
+			Owner:             v.Owner,
+			RestoreStatus:     v.RestoreStatus,
+			Size:              v.Size,
+			StorageClass:      v.StorageClass,
+			VersionId:         v.VersionId,
+		})
+	}
+
+	return result
+}

backend/scoutfs/scoutfs.go

@@ -193,23 +193,22 @@ func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
-func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
 	acct, ok := ctx.Value("account").(auth.Account)
 	if !ok {
 		acct = auth.Account{}
 	}
 
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
+	var res s3response.CompleteMultipartUploadResult
+
 	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		return res, "", s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 	if input.UploadId == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
+		return res, "", s3err.GetAPIError(s3err.ErrNoSuchUpload)
 	}
 	if input.MultipartUpload == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return res, "", s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
 	bucket := *input.Bucket
@@ -219,22 +218,22 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return res, "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return res, "", fmt.Errorf("stat bucket: %w", err)
 	}
 
 	sum, err := s.checkUploadIDExists(bucket, object, uploadID)
 	if err != nil {
-		return nil, err
+		return res, "", err
 	}
 
 	objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
 
 	checksums, err := s.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID))
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get mp checksums: %w", err)
+		return res, "", fmt.Errorf("get mp checksums: %w", err)
 	}
 
 	// ChecksumType should be the same as specified on CreateMultipartUpload
@@ -244,7 +243,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 			checksumType = types.ChecksumType("null")
 		}
 
-		return nil, s3err.GetChecksumTypeMismatchOnMpErr(checksumType)
+		return res, "", s3err.GetChecksumTypeMismatchOnMpErr(checksumType)
 	}
 
 	// check all parts ok
@@ -255,13 +254,13 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	var partNumber int32
 	for i, part := range parts {
 		if part.PartNumber == nil {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 		if *part.PartNumber < 1 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
 		}
 		if *part.PartNumber <= partNumber {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder)
 		}
 
 		partNumber = *part.PartNumber
@@ -270,14 +269,14 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		fullPartPath := filepath.Join(bucket, partObjPath)
 		fi, err := os.Lstat(fullPartPath)
 		if err != nil {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
 		totalsize += fi.Size()
 		// all parts except the last need to be greater, thena
 		// the minimum allowed size (5 Mib)
 		if i < last && fi.Size() < backend.MinPartSize {
-			return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
+			return res, "", s3err.GetAPIError(s3err.ErrEntityTooSmall)
 		}
 
 		b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
@@ -285,24 +284,24 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		if err != nil {
 			etag = ""
 		}
-		if parts[i].ETag == nil || etag != *parts[i].ETag {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		if parts[i].ETag == nil || !backend.AreEtagsSame(etag, *parts[i].ETag) {
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
 		partChecksum, err := s.retrieveChecksums(nil, bucket, partObjPath)
 		if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-			return nil, fmt.Errorf("get part checksum: %w", err)
+			return res, "", fmt.Errorf("get part checksum: %w", err)
 		}
 
 		// If checksum has been provided on mp initalization
 		err = validatePartChecksum(partChecksum, part)
 		if err != nil {
-			return nil, err
+			return res, "", err
 		}
 	}
 
 	if input.MpuObjectSize != nil && totalsize != *input.MpuObjectSize {
-		return nil, s3err.GetIncorrectMpObjectSizeErr(totalsize, *input.MpuObjectSize)
+		return res, "", s3err.GetIncorrectMpObjectSizeErr(totalsize, *input.MpuObjectSize)
 	}
 
 	// use totalsize=0 because we wont be writing to the file, only moving
@@ -310,22 +309,22 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	f, err := s.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0, acct)
 	if err != nil {
 		if errors.Is(err, syscall.EDQUOT) {
-			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+			return res, "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
 		}
-		return nil, fmt.Errorf("open temp file: %w", err)
+		return res, "", fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
 
 	for _, part := range parts {
 		if part.PartNumber == nil || *part.PartNumber < 1 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
 		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
 		fullPartPath := filepath.Join(bucket, partObjPath)
 		pf, err := os.Open(fullPartPath)
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
+			return res, "", fmt.Errorf("open part %v: %v", *part.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
@@ -334,7 +333,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		err = moveData(pf, f.File())
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
+			return res, "", fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
 		}
 	}
 
@@ -343,7 +342,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	objMeta := s.loadUserMetaData(bucket, upiddir, userMetaData)
 	err = s.storeObjectMetadata(f.File(), bucket, object, objMeta)
 	if err != nil {
-		return nil, err
+		return res, "", err
 	}
 
 	objname := filepath.Join(bucket, object)
@@ -352,50 +351,50 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		uid, gid, doChown := s.getChownIDs(acct)
 		err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
 		if err != nil {
-			return nil, err
+			return res, "", err
 		}
 	}
 
 	for k, v := range userMetaData {
 		err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
-			return nil, fmt.Errorf("set user attr %q: %w", k, err)
+			return res, "", fmt.Errorf("set user attr %q: %w", k, err)
 		}
 	}
 
 	// load and set tagging
 	tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object tagging: %w", err)
+		return res, "", fmt.Errorf("get object tagging: %w", err)
 	}
 	if err == nil {
 		err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
 		if err != nil {
-			return nil, fmt.Errorf("set object tagging: %w", err)
+			return res, "", fmt.Errorf("set object tagging: %w", err)
 		}
 	}
 
 	// load and set legal hold
 	lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object legal hold: %w", err)
+		return res, "", fmt.Errorf("get object legal hold: %w", err)
 	}
 	if err == nil {
 		err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
 		if err != nil {
-			return nil, fmt.Errorf("set object legal hold: %w", err)
+			return res, "", fmt.Errorf("set object legal hold: %w", err)
 		}
 	}
 
 	// load and set retention
 	ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object retention: %w", err)
+		return res, "", fmt.Errorf("get object retention: %w", err)
 	}
 	if err == nil {
 		err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
 		if err != nil {
-			return nil, fmt.Errorf("set object retention: %w", err)
+			return res, "", fmt.Errorf("set object retention: %w", err)
 		}
 	}
 
@@ -404,12 +403,12 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 
 	err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
 	if err != nil {
-		return nil, fmt.Errorf("set etag attr: %w", err)
+		return res, "", fmt.Errorf("set etag attr: %w", err)
 	}
 
 	err = f.link()
 	if err != nil {
-		return nil, fmt.Errorf("link object in namespace: %w", err)
+		return res, "", fmt.Errorf("link object in namespace: %w", err)
 	}
 
 	// cleanup tmp dirs
@@ -418,11 +417,11 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	// for same object name outstanding
 	os.Remove(filepath.Join(bucket, objdir))
 
-	return &s3.CompleteMultipartUploadOutput{
+	return s3response.CompleteMultipartUploadResult{
 		Bucket: &bucket,
 		ETag:   &s3MD5,
 		Key:    &object,
-	}, nil
+	}, "", nil
 }
 
 func (s *ScoutFS) storeObjectMetadata(f *os.File, bucket, object string, m objectMetadata) error {
@@ -728,9 +727,6 @@ func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
 }
 
 func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
-	if input.Bucket == nil {
-		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
 	bucket := *input.Bucket
 	prefix := ""
 	if input.Prefix != nil {
@@ -767,20 +763,17 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
 	return s3response.ListObjectsResult{
 		CommonPrefixes: results.CommonPrefixes,
 		Contents:       results.Objects,
-		Delimiter:      &delim,
+		Delimiter:      backend.GetPtrFromString(delim),
+		Marker:         backend.GetPtrFromString(marker),
+		NextMarker:     backend.GetPtrFromString(results.NextMarker),
+		Prefix:         backend.GetPtrFromString(prefix),
 		IsTruncated:    &results.Truncated,
-		Marker:         &marker,
 		MaxKeys:        &maxkeys,
 		Name:           &bucket,
-		NextMarker:     &results.NextMarker,
-		Prefix:         &prefix,
 	}, nil
 }
 
 func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
-	if input.Bucket == nil {
-		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
 	bucket := *input.Bucket
 	prefix := ""
 	if input.Prefix != nil {
@@ -788,7 +781,11 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
 	}
 	marker := ""
 	if input.ContinuationToken != nil {
-		marker = *input.ContinuationToken
+		if input.StartAfter != nil {
+			marker = max(*input.StartAfter, *input.ContinuationToken)
+		} else {
+			marker = *input.ContinuationToken
+		}
 	}
 	delim := ""
 	if input.Delimiter != nil {
@@ -814,16 +811,20 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
 		return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
+	count := int32(len(results.Objects))
+
 	return s3response.ListObjectsV2Result{
 		CommonPrefixes:        results.CommonPrefixes,
 		Contents:              results.Objects,
-		Delimiter:             &delim,
 		IsTruncated:           &results.Truncated,
-		ContinuationToken:     &marker,
 		MaxKeys:               &maxkeys,
 		Name:                  &bucket,
-		NextContinuationToken: &results.NextMarker,
-		Prefix:                &prefix,
+		KeyCount:              &count,
+		Delimiter:             backend.GetPtrFromString(delim),
+		ContinuationToken:     backend.GetPtrFromString(marker),
+		NextContinuationToken: backend.GetPtrFromString(results.NextMarker),
+		Prefix:                backend.GetPtrFromString(prefix),
+		StartAfter:            backend.GetPtrFromString(*input.StartAfter),
 	}, nil
 }
 

backend/scoutfs/scoutfs_compat.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 
 	"golang.org/x/sys/unix"
 
@@ -31,6 +32,7 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/s3err"
 )
 
 func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
@@ -90,6 +92,9 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 	// file descriptor into the namespace.
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, err
 	}
 
@@ -150,10 +155,20 @@ func (tmp *tmpfile) link() error {
 	}
 	defer dirf.Close()
 
-	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-	if err != nil {
-		return fmt.Errorf("link tmpfile: %w", err)
+	for {
+		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		if errors.Is(err, fs.ErrExist) {
+			err := os.Remove(objPath)
+			if err != nil && !errors.Is(err, fs.ErrNotExist) {
+				return fmt.Errorf("remove stale path: %w", err)
+			}
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("link tmpfile: %w", err)
+		}
+		break
 	}
 
 	err = tmp.f.Close()

backend/walk.go

@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
-	"sort"
 	"strings"
 	"syscall"
 
@@ -38,10 +37,38 @@ type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
 
 var ErrSkipObj = errors.New("skip this object")
 
+// map to store object common prefixes
+type cpMap map[string]int
+
+func (c cpMap) Add(key string) {
+	_, ok := c[key]
+	if !ok {
+		c[key] = len(c)
+	}
+}
+
+// Len returns the length of the map
+func (c cpMap) Len() int {
+	return len(c)
+}
+
+// CpArray converts the map into a sorted []types.CommonPrefixes array
+func (c cpMap) CpArray() []types.CommonPrefix {
+	commonPrefixes := make([]types.CommonPrefix, c.Len())
+	for cp, i := range c {
+		pfx := cp
+		commonPrefixes[i] = types.CommonPrefix{
+			Prefix: &pfx,
+		}
+	}
+
+	return commonPrefixes
+}
+
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
 func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
-	cpmap := make(map[string]struct{})
+	cpmap := cpMap{}
 	var objects []s3response.Object
 
 	// if max is 0, it should return empty non-truncated result
@@ -120,7 +147,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 						return fs.SkipAll
 					}
 					objects = append(objects, dirobj)
-					if (len(objects) + len(cpmap)) == int(max) {
+					if (len(objects) + cpmap.Len()) == int(max) {
 						newMarker = path
 						pastMax = true
 					}
@@ -174,7 +201,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 
 			objects = append(objects, obj)
 
-			if (len(objects) + len(cpmap)) == int(max) {
+			if (len(objects) + cpmap.Len()) == int(max) {
 				newMarker = path
 				pastMax = true
 			}
@@ -218,7 +245,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 				return fs.SkipAll
 			}
 			objects = append(objects, obj)
-			if (len(objects) + len(cpmap)) == int(max) {
+			if (len(objects) + cpmap.Len()) == int(max) {
 				newMarker = path
 				pastMax = true
 			}
@@ -244,8 +271,8 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			truncated = true
 			return fs.SkipAll
 		}
-		cpmap[cpref] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
+		cpmap.Add(cpref)
+		if (len(objects) + cpmap.Len()) == int(max) {
 			newMarker = cpref
 			pastMax = true
 		}
@@ -260,25 +287,12 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		return WalkResults{}, err
 	}
 
-	var commonPrefixStrings []string
-	for k := range cpmap {
-		commonPrefixStrings = append(commonPrefixStrings, k)
-	}
-	sort.Strings(commonPrefixStrings)
-	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
-	for _, cp := range commonPrefixStrings {
-		pfx := cp
-		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
-			Prefix: &pfx,
-		})
-	}
-
 	if !truncated {
 		newMarker = ""
 	}
 
 	return WalkResults{
-		CommonPrefixes: commonPrefixes,
+		CommonPrefixes: cpmap.CpArray(),
 		Objects:        objects,
 		Truncated:      truncated,
 		NextMarker:     newMarker,
@@ -296,7 +310,7 @@ func contains(a string, strs []string) bool {
 
 type WalkVersioningResults struct {
 	CommonPrefixes      []types.CommonPrefix
-	ObjectVersions      []types.ObjectVersion
+	ObjectVersions      []s3response.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
 	Truncated           bool
 	NextMarker          string
@@ -304,7 +318,7 @@ type WalkVersioningResults struct {
 }
 
 type ObjVersionFuncResult struct {
-	ObjectVersions      []types.ObjectVersion
+	ObjectVersions      []s3response.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
 	NextVersionIdMarker string
 	Truncated           bool
@@ -315,8 +329,8 @@ type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *boo
 // WalkVersions walks the supplied fs.FS and returns results compatible with
 // ListObjectVersions action response
 func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
-	cpmap := make(map[string]struct{})
-	var objects []types.ObjectVersion
+	cpmap := cpMap{}
+	var objects []s3response.ObjectVersion
 	var delMarkers []types.DeleteMarkerEntry
 
 	var pastMarker bool
@@ -371,11 +385,11 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 			if delimiter == "/" &&
 				prefix != path+"/" &&
 				strings.HasPrefix(path+"/", prefix) {
-				cpmap[path+"/"] = struct{}{}
+				cpmap.Add(path + "/")
 				return fs.SkipDir
 			}
 
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-cpmap.Len(), d)
 			if err == ErrSkipObj {
 				return nil
 			}
@@ -402,7 +416,7 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		if delimiter == "" {
 			// If no delimiter specified, then all files with matching
 			// prefix are included in results
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-cpmap.Len(), d)
 			if err == ErrSkipObj {
 				return nil
 			}
@@ -445,7 +459,7 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		suffix := strings.TrimPrefix(path, prefix)
 		before, _, found := strings.Cut(suffix, delimiter)
 		if !found {
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-cpmap.Len(), d)
 			if err == ErrSkipObj {
 				return nil
 			}
@@ -467,8 +481,8 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
 		// delimiter at the end.
-		cpmap[prefix+before+delimiter] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
+		cpmap.Add(prefix + before + delimiter)
+		if (len(objects) + cpmap.Len()) == int(max) {
 			nextMarker = path
 			truncated = true
 
@@ -481,21 +495,8 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		return WalkVersioningResults{}, err
 	}
 
-	var commonPrefixStrings []string
-	for k := range cpmap {
-		commonPrefixStrings = append(commonPrefixStrings, k)
-	}
-	sort.Strings(commonPrefixStrings)
-	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
-	for _, cp := range commonPrefixStrings {
-		pfx := cp
-		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
-			Prefix: &pfx,
-		})
-	}
-
 	return WalkVersioningResults{
-		CommonPrefixes:      commonPrefixes,
+		CommonPrefixes:      cpmap.CpArray(),
 		ObjectVersions:      objects,
 		DelMarkers:          delMarkers,
 		Truncated:           truncated,

cmd/versitygw/admin.go

@@ -100,6 +100,11 @@ func adminCommand() *cli.Command {
 						Usage:   "secret access key for the new user",
 						Aliases: []string{"s"},
 					},
+					&cli.StringFlag{
+						Name:    "role",
+						Usage:   "the new user role",
+						Aliases: []string{"r"},
+					},
 					&cli.IntFlag{
 						Name:    "user-id",
 						Usage:   "userID for the new user",
@@ -311,8 +316,14 @@ func deleteUser(ctx *cli.Context) error {
 }
 
 func updateUser(ctx *cli.Context) error {
-	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
+	access, secret, userId, groupId, role := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id"), auth.Role(ctx.String("role"))
 	props := auth.MutableProps{}
+	if ctx.IsSet("role") {
+		if !role.IsValid() {
+			return fmt.Errorf("invalid user role: %v", role)
+		}
+		props.Role = role
+	}
 	if ctx.IsSet("secret") {
 		props.Secret = &secret
 	}

cmd/versitygw/main.go

@@ -50,6 +50,7 @@ var (
 	logWebhookURL, accessLog                 string
 	adminLogFile                             string
 	healthPath                               string
+	virtualDomain                            string
 	debug                                    bool
 	pprof                                    string
 	quiet                                    bool
@@ -227,6 +228,13 @@ func initFlags() []cli.Flag {
 			Destination: &quiet,
 			Aliases:     []string{"q"},
 		},
+		&cli.StringFlag{
+			Name:        "virtual-domain",
+			Usage:       "enables the virtual host style bucket addressing with the specified arg as the base domain",
+			EnvVars:     []string{"VGW_VIRTUAL_DOMAIN"},
+			Destination: &virtualDomain,
+			Aliases:     []string{"vd"},
+		},
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
@@ -525,19 +533,19 @@ func initFlags() []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:        "ipa-user",
-			Usage:       "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
+			Usage:       "Username used to connect to FreeIPA (requires permissions to read user vault contents)",
 			EnvVars:     []string{"VGW_IPA_USER"},
 			Destination: &ipaUser,
 		},
 		&cli.StringFlag{
 			Name:        "ipa-password",
-			Usage:       "Password of the user used to connect to FreeIPA.",
+			Usage:       "Password of the user used to connect to FreeIPA",
 			EnvVars:     []string{"VGW_IPA_PASSWORD"},
 			Destination: &ipaPassword,
 		},
 		&cli.BoolFlag{
 			Name:        "ipa-insecure",
-			Usage:       "Verify TLS certificate of FreeIPA server. Default is 'true'.",
+			Usage:       "Disable verify TLS certificate of FreeIPA server",
 			EnvVars:     []string{"VGW_IPA_INSECURE"},
 			Destination: &ipaInsecure,
 		},
@@ -603,6 +611,9 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if readonly {
 		opts = append(opts, s3api.WithReadOnly())
 	}
+	if virtualDomain != "" {
+		opts = append(opts, s3api.WithHostStyle(virtualDomain))
+	}
 
 	admApp := fiber.New(fiber.Config{
 		AppName:               "versitygw",

cmd/versitygw/plugin.go

@@ -15,50 +15,60 @@
 package main
 
 import (
+	"errors"
 	"fmt"
+	"plugin"
 
 	"github.com/urfave/cli/v2"
-	vgwplugin "github.com/versity/versitygw/backend/plugin"
-)
-
-var (
-	pluginPath   string
-	pluginConfig string
+	"github.com/versity/versitygw/plugins"
 )
 
 func pluginCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "plugin",
-		Usage:       "plugin storage backend",
-		Description: `This tells the gateway to load the backend from a dynamic runtime plugin.`,
-		Action:      runPlugin,
+		Usage:       "load a backend from a plugin",
+		Description: "Runs a s3 gateway and redirects the requests to the backend defined in the plugin",
+		Action:      runPluginBackend,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
-				Name:        "file",
-				Usage:       "path to plugin shared object file",
-				Value:       "",
-				Required:    true,
-				EnvVars:     []string{"VGW_PLUGIN_FILE"},
-				Destination: &pluginPath,
-				Aliases:     []string{"f"},
-			},
-			&cli.StringFlag{
-				Name:        "config",
-				Usage:       "configuration option for the plugin",
-				Value:       "",
-				Required:    true,
-				EnvVars:     []string{"VGW_PLUGIN_CONFIG"},
-				Destination: &pluginConfig,
-				Aliases:     []string{"c"},
+				Name:    "config",
+				Usage:   "location of the config file",
+				Aliases: []string{"c"},
 			},
 		},
 	}
 }
 
-func runPlugin(ctx *cli.Context) error {
-	be, err := vgwplugin.NewPluginBackend(pluginPath, pluginConfig)
-	if err != nil {
-		return fmt.Errorf("init plugin backend: %w", err)
+func runPluginBackend(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no plugin file provided to be loaded")
 	}
+
+	pluginPath := ctx.Args().Get(0)
+	config := ctx.String("config")
+
+	p, err := plugin.Open(pluginPath)
+	if err != nil {
+		return err
+	}
+
+	backendSymbol, err := p.Lookup("Backend")
+	if err != nil {
+		return err
+	}
+	backendPluginPtr, ok := backendSymbol.(*plugins.BackendPlugin)
+	if !ok {
+		return errors.New("plugin is not of type *plugins.BackendPlugin")
+	}
+
+	if backendPluginPtr == nil {
+		return errors.New("variable Backend is nil")
+	}
+
+	be, err := (*backendPluginPtr).New(config)
+	if err != nil {
+		return err
+	}
+
 	return runGateway(ctx.Context, be)
 }

cmd/versitygw/posix.go

@@ -31,6 +31,7 @@ var (
 	dirPerms           uint
 	sidecar            string
 	nometa             bool
+	forceNoTmpFile     bool
 )
 
 func posixCommand() *cli.Command {
@@ -93,6 +94,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_META_NONE"},
 				Destination: &nometa,
 			},
+			&cli.BoolFlag{
+				Name:        "disableotmp",
+				Usage:       "disable O_TMPFILE support for new objects",
+				EnvVars:     []string{"VGW_DISABLE_OTMP"},
+				Destination: &forceNoTmpFile,
+			},
 		},
 	}
 }
@@ -113,11 +120,12 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	opts := posix.PosixOpts{
-		ChownUID:      chownuid,
-		ChownGID:      chowngid,
-		BucketLinks:   bucketlinks,
-		VersioningDir: versioningDir,
-		NewDirPerm:    fs.FileMode(dirPerms),
+		ChownUID:       chownuid,
+		ChownGID:       chowngid,
+		BucketLinks:    bucketlinks,
+		VersioningDir:  versioningDir,
+		NewDirPerm:     fs.FileMode(dirPerms),
+		ForceNoTmpFile: forceNoTmpFile,
 	}
 
 	var ms meta.MetadataStorer

cmd/versitygw/s3.go

@@ -26,8 +26,10 @@ var (
 	s3proxySecret          string
 	s3proxyEndpoint        string
 	s3proxyRegion          string
+	s3proxyMetaBucket      string
 	s3proxyDisableChecksum bool
 	s3proxySslSkipVerify   bool
+	s3proxyUsePathStyle    bool
 	s3proxyDebug           bool
 )
 
@@ -71,6 +73,12 @@ to an s3 storage backend service.`,
 				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
+			&cli.StringFlag{
+				Name:        "meta-bucket",
+				Usage:       "s3 service meta bucket to store buckets acl/policy",
+				EnvVars:     []string{"VGW_S3_META_BUCKET"},
+				Destination: &s3proxyMetaBucket,
+			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
@@ -85,6 +93,13 @@ to an s3 storage backend service.`,
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
+			&cli.BoolFlag{
+				Name:        "use-path-style",
+				Usage:       "use path style addressing for s3 proxy",
+				EnvVars:     []string{"VGW_S3_USE_PATH_STYLE"},
+				Value:       false,
+				Destination: &s3proxyUsePathStyle,
+			},
 			&cli.BoolFlag{
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
@@ -97,8 +112,8 @@ to an s3 storage backend service.`,
 }
 
 func runS3(ctx *cli.Context) error {
-	be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
-		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
 	if err != nil {
 		return fmt.Errorf("init s3 backend: %w", err)
 	}

cmd/versitygw/test.go

@@ -34,7 +34,7 @@ var (
 	totalReqs         int
 	upload            bool
 	download          bool
-	pathStyle         bool
+	hostStyle         bool
 	checksumDisable   bool
 	versioningEnabled bool
 	azureTests        bool
@@ -74,6 +74,12 @@ func initTestFlags() []cli.Flag {
 			Destination: &endpoint,
 			Aliases:     []string{"e"},
 		},
+		&cli.BoolFlag{
+			Name:        "host-style",
+			Usage:       "Use host-style bucket addressing",
+			Value:       false,
+			Destination: &hostStyle,
+		},
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug mode",
@@ -124,6 +130,11 @@ func initTestCommands() []*cli.Command {
 				},
 			},
 		},
+		{
+			Name:   "scoutfs",
+			Usage:  "Tests scoutfs full flow",
+			Action: getAction(integration.TestScoutfs),
+		},
 		{
 			Name:   "iam",
 			Usage:  "Tests iam service",
@@ -186,12 +197,6 @@ func initTestCommands() []*cli.Command {
 					Value:       1,
 					Destination: &concurrency,
 				},
-				&cli.BoolFlag{
-					Name:        "pathStyle",
-					Usage:       "Use Pathstyle bucket addressing",
-					Value:       false,
-					Destination: &pathStyle,
-				},
 				&cli.BoolFlag{
 					Name:        "checksumDis",
 					Usage:       "Disable server checksum",
@@ -223,8 +228,8 @@ func initTestCommands() []*cli.Command {
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
-				if pathStyle {
-					opts = append(opts, integration.WithPathStyle())
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
 				}
 				if checksumDisable {
 					opts = append(opts, integration.WithDisableChecksum())
@@ -287,6 +292,9 @@ func initTestCommands() []*cli.Command {
 				if checksumDisable {
 					opts = append(opts, integration.WithDisableChecksum())
 				}
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
+				}
 
 				s3conf := integration.NewS3Conf(opts...)
 
@@ -316,6 +324,9 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if azureTests {
 			opts = append(opts, integration.WithAzureMode())
 		}
+		if hostStyle {
+			opts = append(opts, integration.WithHostStyle())
+		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -351,6 +362,9 @@ func extractIntTests() (commands []*cli.Command) {
 				if versioningEnabled {
 					opts = append(opts, integration.WithVersioningEnabled())
 				}
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
+				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)

extra/example.conf

@@ -99,6 +99,26 @@ ROOT_SECRET_ACCESS_KEY=
 # endpoint is unauthenticated, and returns a 200 status for GET.
 #VGW_HEALTH=
 
+# Enable VGW_READ_ONLY to only allow read operations to the S3 server. No write
+# operations will be allowed.
+#VGW_READ_ONLY=false
+
+# The VGW_VIRTUAL_DOMAIN option enables the virtual host style bucket
+# addressing. The path style addressing is the default, and remains enabled
+# even when virtual host style is enabled. The VGW_VIRTUAL_DOMAIN option
+# specifies the domain name that will be used for the virtual host style
+# addressing. For virtual addressing, access to a bucket is in the request
+# form:
+# https://<bucket>.<VGW_VIRTUAL_DOMAIN>/
+# for example: https://mybucket.example.com/ where
+# VGW_VIRTUAL_DOMAIN=example.com
+# and all subdomains of VGW_VIRTUAL_DOMAIN should be reserved for buckets.
+# This means that virtual host addressing will generally require a DNS
+# entry for each bucket that needs to be accessed.
+# The default path style request is of the form:
+# https://<VGW_ENDPOINT>/<bucket>
+#VGW_VIRTUAL_DOMAIN=
+
 ###############
 # Access Logs #
 ###############
@@ -240,6 +260,24 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_LDAP_USER_ID_ATR=
 #VGW_IAM_LDAP_GROUP_ID_ATR=
 
+# The FreeIPA options will enable the FreeIPA IAM service with accounts stored
+# in an external FreeIPA service. Currently the FreeIPA IAM service only
+# supports account retrieval. Creating and modifying accounts must be done
+# outside of the versitygw service.
+# FreeIPA server url e.g. https://ipa.example.test
+#VGW_IPA_HOST=
+# A name of the user vault containing their secret
+#VGW_IPA_VAULT_NAME=
+# Username used to connect to FreeIPA (requires permissions to read user vault
+# contents)
+#VGW_IPA_USER=
+# Password of the user used to connect to FreeIPA
+#VGW_IPA_PASSWORD=
+# Disable verify TLS certificate of FreeIPA server
+#VGW_IPA_INSECURE=false
+# FreeIPA IAM debug output
+#VGW_IPA_DEBUG=false
+
 ###############
 # IAM caching #
 ###############
@@ -317,6 +355,40 @@ ROOT_SECRET_ACCESS_KEY=
 # as any parent directories automatically created with object uploads.
 #VGW_DIR_PERMS=0755
 
+# To enable object versions, the VGW_VERSIONING_DIR option must be set to the
+# directory that will be used to store the object versions. The version
+# directory must NOT be a subdirectory of the VGW_BACKEND_ARG directory.
+#VGW_VERSIONING_DIR=
+
+# The gateway uses xattrs to store metadata for objects by default. For systems
+# that do not support xattrs, the VGW_META_SIDECAR option can be set to a
+# directory that will be used to store the metadata for objects. This is
+# currently experimental, and may have issues for some edge cases.
+#VGW_META_SIDECAR=
+
+# The VGW_META_NONE option will disable the metadata functionality for the
+# gateway. This will cause the gateway to not store any metadata for objects
+# or buckets. This include bucket ACLs and Policy. This may be useful for
+# read only access to pre-existing data where the gateway should not modify
+# the data. It is recommened to enable VGW_READ_ONLY (Global Options) along
+# with this.
+#VGW_META_NONE=false
+
+# The gateway will use O_TMPFILE for writing objects while uploading and
+# link the file to the final object name when the upload is complete if the
+# filesystem supports O_TMPFILE. This creates an atomic object creation
+# that is not visible to other clients or racing uploads until the upload
+# is complete. This will not work if there is a different filesystem mounted
+# below the bucket level than where the bucket resides. The VGW_DISABLE_OTMP
+# option can be set to true to disable this functionality and force the fallback
+# mode when O_TMPFILE is not available. This fallback will create a temporary
+# file in the bucket directory and rename it to the final object name when
+# the upload is complete if the final location is in the same filesystem, or
+# copy the file to the final location if the final location is in a different
+# filesystem. This fallback mode is still atomic, but may be less efficient
+# than O_TMPFILE when the data needs to be copied into the final location.
+#VGW_DISABLE_OTMP=false
+
 ###########
 # scoutfs #
 ###########

go.mod

@@ -6,72 +6,72 @@ toolchain go1.24.1
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
 	github.com/DataDog/datadog-go/v5 v5.6.0
-	github.com/aws/aws-sdk-go-v2 v1.36.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1
-	github.com/aws/smithy-go v1.22.3
-	github.com/go-ldap/ldap/v3 v3.4.10
-	github.com/gofiber/fiber/v2 v2.52.6
+	github.com/aws/aws-sdk-go-v2 v1.36.5
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0
+	github.com/aws/smithy-go v1.22.4
+	github.com/go-ldap/ldap/v3 v3.4.11
+	github.com/gofiber/fiber/v2 v2.52.8
 	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.41.0
-	github.com/oklog/ulid/v2 v2.1.0
-	github.com/pkg/xattr v0.4.10
-	github.com/segmentio/kafka-go v0.4.47
+	github.com/nats-io/nats.go v1.43.0
+	github.com/oklog/ulid/v2 v2.1.1
+	github.com/pkg/xattr v0.4.12
+	github.com/segmentio/kafka-go v0.4.48
 	github.com/smira/go-statsd v1.3.4
-	github.com/urfave/cli/v2 v2.27.6
-	github.com/valyala/fasthttp v1.60.0
+	github.com/urfave/cli/v2 v2.27.7
+	github.com/valyala/fasthttp v1.62.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sync v0.13.0
-	golang.org/x/sys v0.32.0
+	golang.org/x/sync v0.15.0
+	golang.org/x/sys v0.33.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.10 // indirect
+	github.com/nats-io/nkeys v0.4.11 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.37.0 // indirect
-	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/text v0.24.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.29.13
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.66
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.17
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.70
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect

go.sum

@@ -1,15 +1,15 @@
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2/go.mod h1:SqINnQ9lVVdRlyC8cd1lCI0SdX4n2paeABd2K8ggfnE=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 h1:Bg8m3nq/X1DeePkAbCfb6ml6F3F0IunEhE8TMh+lY48=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
@@ -23,50 +23,50 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
-github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
-github.com/aws/aws-sdk-go-v2/config v1.29.13 h1:RgdPqWoE8nPpIekpVpDJsBckbqT4Liiaq9f35pbTh1Y=
-github.com/aws/aws-sdk-go-v2/config v1.29.13/go.mod h1:NI28qs/IOUIRhsR7GQ/JdexoqRN9tDxkIrYZq0SOF44=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.66 h1:aKpEKaTy6n4CEJeYI1MNj97oSDLi4xro3UzQfwf5RWE=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.66/go.mod h1:xQ5SusDmHb/fy55wU0QqTy0yNfLqxzec59YcsRZB+rI=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 h1:s43gLuY+zGmtpx+KybfFP4IckopmTfDOPdlf/L++N5I=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71/go.mod h1:KH6wWmY3O3c/jVAjHk0MGzVAFDxkOSt42Eoe4ZO4ge0=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0=
+github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY=
+github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0=
+github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 h1:EO13QJTCD1Ig2IrQnoHTRrn981H9mB7afXsZ89WptI4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82/go.mod h1:AGh1NCg0SH+uyJamiJA5tTQcql4MMRDXGRdMmCxCXzY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 h1:2Ku1xwAohSSXHR1tpAnyVDSQSxoDMA+/NZBytW+f4qg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
-github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3QasqBfuAFHY4Pk5g=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
-github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
-github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 h1:JubM8CGDDFaAOmBrd8CRYNr49ZNgEAiLwGwgNMdS0nw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E=
+github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w=
+github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw=
+github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
-github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -74,33 +74,29 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
-github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
-github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
-github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
-github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU=
+github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM=
+github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4=
+github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
 github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
 github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
-github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
-github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
+github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
+github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
@@ -117,8 +113,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
+github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
+github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
@@ -132,14 +128,14 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.41.0 h1:PzxEva7fflkd+n87OtQTXqCTyLfIIMFJBpyccHLE2Ko=
-github.com/nats-io/nats.go v1.41.0/go.mod h1:wV73x0FSI/orHPSYoyMeJB+KajMDoWyXmFaRrrYaaTo=
-github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc=
-github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U=
+github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
+github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
+github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
+github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
-github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
+github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
+github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
 github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
@@ -147,12 +143,12 @@ github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
-github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
+github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
-github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
+github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
+github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -160,8 +156,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
-github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
+github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs=
+github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
 github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
@@ -170,18 +166,17 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
-github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.60.0 h1:kBRYS0lOhVJ6V+bYN8PqAHELKHtXqwq9zNMLKx1MBsw=
-github.com/valyala/fasthttp v1.60.0/go.mod h1:iY4kDgV3Gc6EqhRZ8icqcmlG6bqhcDXfuHgTO4FXCvc=
+github.com/valyala/fasthttp v1.62.0 h1:8dKRBX/y2rCzyc6903Zu1+3qN0H/d2MsxPPmVNamiH0=
+github.com/valyala/fasthttp v1.62.0/go.mod h1:FCINgr4GKdKqV8Q0xv8b+UxPV+H/O5nNFo3D+r54Htg=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -199,46 +194,28 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
-golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
-golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -254,23 +231,14 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
-golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -278,25 +246,19 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
-golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
-golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
-golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

plugins/plugins.go

@@ -0,0 +1,35 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package plugins
+
+import "github.com/versity/versitygw/backend"
+
+// BackendPlugin defines an interface for creating backend
+// implementation instances.
+// Plugins implementing this interface can be built as shared
+// libraries using Go's plugin system (to build use `go build -buildmode=plugin`).
+// The shared library should export an instance of
+// this interface in a variable named `Backend`.
+type BackendPlugin interface {
+	// New creates and initializes a new backend.Backend instance.
+	// The config parameter specifies the path of the file containing
+	// the configuration for the backend.
+	//
+	// Implementations of this method should perform the necessary steps to
+	// establish a connection to the underlying storage system or service
+	// (e.g., network storage system, distributed storage system, cloud storage)
+	//  and configure it according to the provided configuration.
+	New(config string) (backend.Backend, error)
+}

s3api/controllers/admin.go

@@ -100,7 +100,16 @@ func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := c.iam.UpdateUserAccount(access, props)
+	err := props.Validate()
+	if err != nil {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
+	}
+
+	err = c.iam.UpdateUserAccount(access, props)
 	if err != nil {
 		if strings.Contains(err.Error(), "user not found") {
 			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)

s3api/controllers/backend_moq_test.go

@@ -29,10 +29,10 @@ var _ backend.Backend = &BackendMock{}
 //			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
 //				panic("mock out the ChangeBucketOwner method")
 //			},
-//			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+//			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
 //				panic("mock out the CompleteMultipartUpload method")
 //			},
-//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
 //				panic("mock out the CopyObject method")
 //			},
 //			CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
@@ -199,10 +199,10 @@ type BackendMock struct {
 	ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, acl []byte) error
 
 	// CompleteMultipartUploadFunc mocks the CompleteMultipartUpload method.
-	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error)
 
 	// CopyObjectFunc mocks the CopyObject method.
-	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (s3response.CopyObjectOutput, error)
 
 	// CreateBucketFunc mocks the CreateBucket method.
 	CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error
@@ -904,7 +904,7 @@ func (mock *BackendMock) ChangeBucketOwnerCalls() []struct {
 }
 
 // CompleteMultipartUpload calls CompleteMultipartUploadFunc.
-func (mock *BackendMock) CompleteMultipartUpload(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (mock *BackendMock) CompleteMultipartUpload(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
 	if mock.CompleteMultipartUploadFunc == nil {
 		panic("BackendMock.CompleteMultipartUploadFunc: method is nil but Backend.CompleteMultipartUpload was just called")
 	}
@@ -940,7 +940,7 @@ func (mock *BackendMock) CompleteMultipartUploadCalls() []struct {
 }
 
 // CopyObject calls CopyObjectFunc.
-func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
 	if mock.CopyObjectFunc == nil {
 		panic("BackendMock.CopyObjectFunc: method is nil but Backend.CopyObject was just called")
 	}

s3api/controllers/base_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
@@ -99,8 +100,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
-		ctx.Locals("isDebug", false)
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"})
 		return ctx.Next()
 	})
 	app.Get("/", s3ApiController.ListBuckets)
@@ -116,8 +116,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
-		ctx.Locals("isDebug", false)
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"})
 		return ctx.Next()
 	})
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
@@ -220,10 +219,9 @@ func TestS3ApiController_GetActions(t *testing.T) {
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
@@ -413,10 +411,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 
@@ -438,10 +435,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	}
 	appError := fiber.New()
 	appError.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
@@ -707,10 +703,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	}
 	// Mock ctx.Locals
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{Owner: "valid access"})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{Owner: "valid access"})
 		return ctx.Next()
 	})
 	app.Put("/:bucket", s3ApiController.PutBucketActions)
@@ -889,15 +884,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "Create-bucket-invalid-bucket-name",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
-			},
-			wantErr:    false,
-			statusCode: 400,
-		},
 		{
 			name: "Create-bucket-success",
 			app:  app,
@@ -974,9 +960,9 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
 				return nil
 			},
-			CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-				return &s3.CopyObjectOutput{
-					CopyObjectResult: &types.CopyObjectResult{},
+			CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
+				return s3response.CopyObjectOutput{
+					CopyObjectResult: &s3response.CopyObjectResult{},
 				}, nil
 			},
 			PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
@@ -1003,10 +989,9 @@ func TestS3ApiController_PutActions(t *testing.T) {
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
@@ -1292,10 +1277,9 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 
@@ -1378,10 +1362,9 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Post("/:bucket", s3ApiController.DeleteObjects)
@@ -1458,10 +1441,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
@@ -1482,10 +1464,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	}}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
@@ -1565,11 +1546,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
-		ctx.Locals("region", "us-east-1")
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
+		utils.ContextKeyRegion.Set(ctx, "us-east-1")
 		return ctx.Next()
 	})
 
@@ -1583,17 +1563,16 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 			return acldata, nil
 		},
 		HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
-			return nil, s3err.GetAPIError(3)
+			return nil, s3err.GetAPIError(s3err.ErrBucketNotEmpty)
 		},
 	},
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
-		ctx.Locals("region", "us-east-1")
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
+		utils.ContextKeyRegion.Set(ctx, "us-east-1")
 		return ctx.Next()
 	})
 
@@ -1670,10 +1649,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
@@ -1693,10 +1671,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
@@ -1765,8 +1742,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			RestoreObjectFunc: func(context.Context, *s3.RestoreObjectInput) error {
 				return nil
 			},
-			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-				return &s3.CompleteMultipartUploadOutput{}, nil
+			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+				return s3response.CompleteMultipartUploadResult{}, "", nil
 			},
 			CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 				return s3response.InitiateMultipartUploadResult{}, nil
@@ -1798,10 +1775,9 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	`
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)

s3api/debuglogger/logger.go

@@ -0,0 +1,226 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package debuglogger
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"sync/atomic"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type Color string
+
+const (
+	green  Color = "\033[32m"
+	yellow Color = "\033[33m"
+	blue   Color = "\033[34m"
+	Purple Color = "\033[0;35m"
+
+	reset      = "\033[0m"
+	borderChar = "─"
+	boxWidth   = 120
+)
+
+// Logs http request details: headers, body, params, query args
+func LogFiberRequestDetails(ctx *fiber.Ctx) {
+	// Log the full request url
+	fullURL := ctx.Protocol() + "://" + ctx.Hostname() + ctx.OriginalURL()
+	fmt.Printf("%s[URL]: %s%s\n", green, fullURL, reset)
+
+	// log request headers
+	wrapInBox(green, "REQUEST HEADERS", boxWidth, func() {
+		ctx.Request().Header.VisitAll(func(key, value []byte) {
+			printWrappedLine(yellow, string(key), string(value))
+		})
+	})
+	// skip request body log for PutObject and UploadPart
+	skipBodyLog := isLargeDataAction(ctx)
+	if !skipBodyLog {
+		body := ctx.Request().Body()
+		if len(body) != 0 {
+			printBoxTitleLine(blue, "REQUEST BODY", boxWidth, false)
+			fmt.Printf("%s%s%s\n", blue, body, reset)
+			printHorizontalBorder(blue, boxWidth, false)
+		}
+	}
+
+	if ctx.Request().URI().QueryArgs().Len() != 0 {
+		ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
+			log.Printf("%s: %s", key, val)
+		})
+	}
+}
+
+// Logs http response details: body, headers
+func LogFiberResponseDetails(ctx *fiber.Ctx) {
+	wrapInBox(green, "RESPONSE HEADERS", boxWidth, func() {
+		ctx.Response().Header.VisitAll(func(key, value []byte) {
+			printWrappedLine(yellow, string(key), string(value))
+		})
+	})
+
+	_, ok := ctx.Locals("skip-res-body-log").(bool)
+	if !ok {
+		body := ctx.Response().Body()
+		if len(body) != 0 {
+			PrintInsideHorizontalBorders(blue, "RESPONSE BODY", string(body), boxWidth)
+		}
+	}
+}
+
+var debugEnabled atomic.Bool
+
+// SetDebugEnabled sets the debug mode
+func SetDebugEnabled() {
+	debugEnabled.Store(true)
+}
+
+// Logf is the same as 'fmt.Printf' with debug prefix,
+// a color added and '\n' at the end
+func Logf(format string, v ...any) {
+	if !debugEnabled.Load() {
+		return
+	}
+	debugPrefix := "[DEBUG]: "
+	fmt.Printf(string(yellow)+debugPrefix+format+reset+"\n", v...)
+}
+
+// Infof prints out green info block with [INFO]: prefix
+func Infof(format string, v ...any) {
+	if !debugEnabled.Load() {
+		return
+	}
+	debugPrefix := "[INFO]: "
+	fmt.Printf(string(green)+debugPrefix+format+reset+"\n", v...)
+}
+
+// PrintInsideHorizontalBorders prints the text inside horizontal
+// border and title in the center of upper border
+func PrintInsideHorizontalBorders(color Color, title, text string, width int) {
+	if !debugEnabled.Load() {
+		return
+	}
+	printBoxTitleLine(color, title, width, false)
+	fmt.Printf("%s%s%s\n", color, text, reset)
+	printHorizontalBorder(color, width, false)
+}
+
+// Prints out box title either with closing characters or not:  "┌", "┐"
+// e.g ┌────────────────[ RESPONSE HEADERS ]────────────────┐
+func printBoxTitleLine(color Color, title string, length int, closing bool) {
+	leftCorner, rightCorner := "┌", "┐"
+
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	// Calculate how many border characters are needed
+	titleFormatted := fmt.Sprintf("[ %s ]", title)
+	borderSpace := length - len(titleFormatted) - 2 // 2 for corners
+	leftLen := borderSpace / 2
+	rightLen := borderSpace - leftLen
+
+	// Build the line
+	line := leftCorner +
+		strings.Repeat(borderChar, leftLen) +
+		titleFormatted +
+		strings.Repeat(borderChar, rightLen) +
+		rightCorner
+
+	fmt.Println(string(color) + line + reset)
+}
+
+// Prints out a horizontal line either with closing characters or not: "└", "┘"
+func printHorizontalBorder(color Color, length int, closing bool) {
+	leftCorner, rightCorner := "└", "┘"
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	line := leftCorner + strings.Repeat(borderChar, length-2) + rightCorner + reset
+	fmt.Println(string(color) + line)
+}
+
+// wrapInBox wraps the output of a function call (fn) inside a styled box with a title.
+func wrapInBox(color Color, title string, length int, fn func()) {
+	printBoxTitleLine(color, title, length, true)
+	fn()
+	printHorizontalBorder(color, length, true)
+}
+
+// returns the provided string length
+// defaulting to 13 for exceeding lengths
+func getLen(str string) int {
+	if len(str) < 13 {
+		return 13
+	}
+
+	return len(str)
+}
+
+// prints a formatted key-value pair within a box layout,
+// wrapping the value text if it exceeds the allowed width.
+func printWrappedLine(keyColor Color, key, value string) {
+	prefix := fmt.Sprintf("%s│%s %s%-13s%s : ", green, reset, keyColor, key, reset)
+	prefixLen := len(prefix) - len(green) - len(reset) - len(keyColor) - len(reset)
+	// the actual prefix size without colors
+	actualPrefixLen := getLen(key) + 5
+
+	lineWidth := boxWidth - prefixLen
+	valueLines := wrapText(value, lineWidth)
+
+	for i, line := range valueLines {
+		if i == 0 {
+			if len(line) < lineWidth {
+				line += strings.Repeat(" ", lineWidth-len(line))
+			}
+			fmt.Printf("%s%s%s %s│%s\n", prefix, reset, line, green, reset)
+		} else {
+			line = strings.Repeat(" ", actualPrefixLen-2) + line
+			if len(line) < boxWidth-4 {
+				line += strings.Repeat(" ", boxWidth-len(line)-4)
+			}
+			fmt.Printf("%s│ %s%s %s│%s\n", green, reset, line, green, reset)
+		}
+	}
+}
+
+// wrapText splits the input text into lines of at most `width` characters each.
+func wrapText(text string, width int) []string {
+	var lines []string
+	for len(text) > width {
+		lines = append(lines, text[:width])
+		text = text[width:]
+	}
+	if text != "" {
+		lines = append(lines, text)
+	}
+	return lines
+}
+
+// TODO: remove this and use utils.IsBidDataAction after refactoring
+// and creating 'internal' package
+func isLargeDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}

s3api/middlewares/acl-parser.go

@@ -24,6 +24,7 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
@@ -34,7 +35,6 @@ var (
 
 func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
 		path := ctx.Path()
 		pathParts := strings.Split(path, "/")
 		bucket := pathParts[1]
@@ -53,6 +53,7 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
 			!ctx.Request().URI().QueryArgs().Has("ownershipControls") &&
 			!ctx.Request().URI().QueryArgs().Has("cors") {
+			isRoot, acct := utils.ContextKeyIsRoot.Get(ctx).(bool), utils.ContextKeyAccount.Get(ctx).(auth.Account)
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
@@ -77,10 +78,10 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 
 		// if owner is not set, set default owner to root account
 		if parsedAcl.Owner == "" {
-			parsedAcl.Owner = ctx.Locals("rootAccess").(string)
+			parsedAcl.Owner = utils.ContextKeyRootAccessKey.Get(ctx).(string)
 		}
 
-		ctx.Locals("parsedAcl", parsedAcl)
+		utils.ContextKeyParsedAcl.Set(ctx, parsedAcl)
 		return ctx.Next()
 	}
 }

s3api/middlewares/admin.go

@@ -21,13 +21,14 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
 func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		acct := ctx.Locals("account").(auth.Account)
+		acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
 		if acct.Role != auth.RoleAdmin {
 			path := ctx.Path()
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),

s3api/middlewares/authentication.go

@@ -33,7 +33,8 @@ import (
 )
 
 const (
-	iso8601Format = "20060102T150405Z"
+	iso8601Format   = "20060102T150405Z"
+	maxObjSizeLimit = 5 * 1024 * 1024 * 1024 // 5gb
 )
 
 type RootUserConfig struct {
@@ -45,14 +46,15 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
-		// If account is set in context locals, it means it was presigned url case
-		_, ok := ctx.Locals("account").(auth.Account)
-		if ok {
+		// The bucket is public, no need to check this signature
+		if utils.ContextKeyPublicBucket.IsSet(ctx) {
+			return ctx.Next()
+		}
+		// If ContextKeyAuthenticated is set in context locals, it means it was presigned url case
+		if utils.ContextKeyAuthenticated.IsSet(ctx) {
 			return ctx.Next()
 		}
 
-		ctx.Locals("region", region)
-		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
 		if authorization == "" {
 			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
@@ -71,8 +73,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			}, logger, mm)
 		}
 
-		ctx.Locals("isRoot", authData.Access == root.Access)
-		ctx.Locals("rootAccess", root.Access)
+		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
@@ -81,7 +82,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)
 		}
-		ctx.Locals("account", account)
+
+		utils.ContextKeyAccount.Set(ctx, account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
@@ -105,6 +107,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return sendResponse(ctx, err, logger, mm)
 		}
 
+		var contentLength int64
+		contentLengthStr := ctx.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+			//TODO: not sure if InvalidRequest should be returned in this case
+			if err != nil {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
+			}
+		}
+
 		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if utils.IsBigDataAction(ctx) {
 			// for streaming PUT actions, authorization is deferred
@@ -126,6 +138,18 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 				if err != nil {
 					return sendResponse(ctx, err, logger, mm)
 				}
+
+				return ctx.Next()
+			}
+
+			// Content-Length has to be set for data uploads: PutObject, UploadPart
+			if contentLengthStr == "" {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingContentLength), logger, mm)
+			}
+			// the upload limit for big data actions: PutObject, UploadPart
+			// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
+			if contentLength > maxObjSizeLimit {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrEntityTooLarge), logger, mm)
 			}
 
 			return ctx.Next()
@@ -142,15 +166,6 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			}
 		}
 
-		var contentLength int64
-		contentLengthStr := ctx.Get("Content-Length")
-		if contentLengthStr != "" {
-			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
-			if err != nil {
-				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
-			}
-		}
-
 		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)

s3api/middlewares/body-reader.go

@@ -18,14 +18,15 @@ import (
 	"io"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/utils"
 )
 
 func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
-	r, ok := ctx.Locals("body-reader").(io.Reader)
+	r, ok := utils.ContextKeyBodyReader.Get(ctx).(io.Reader)
 	if !ok {
 		r = ctx.Request().BodyStream()
 	}
 
 	r = wr(r)
-	ctx.Locals("body-reader", r)
+	utils.ContextKeyBodyReader.Set(ctx, r)
 }

s3api/middlewares/bucket-object-name-validator.go

@@ -0,0 +1,58 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/http"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+// BucketObjectNameValidator extracts and validates
+// the bucket and object names from the request URI.
+func BucketObjectNameValidator(l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		// skip the check for admin apis
+		if ctx.Method() == http.MethodPatch {
+			return ctx.Next()
+		}
+
+		path := ctx.Path()
+		// skip the check if the operation isn't bucket/object scoped
+		// e.g ListBuckets
+		if path == "/" {
+			return ctx.Next()
+		}
+
+		bucket, object := parsePath(path)
+
+		// check if the provided bucket name is valid
+		if !utils.IsValidBucketName(bucket) {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketName), l, mm)
+		}
+
+		// check if the provided object name is valid
+		// skip for empty objects: e.g bucket operations: HeadBucket...
+		if object != "" && !utils.IsObjectNameValid(object) {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrBadRequest), l, mm)
+		}
+
+		return ctx.Next()
+	}
+}

s3api/middlewares/host-style-parser.go

@@ -0,0 +1,40 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+// HostStyleParser is a middleware which parses the bucket name
+// from the 'Host' header and appends in the request URL path
+func HostStyleParser(virtualDomain string) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		host := string(ctx.Request().Host())
+		// the host should match this pattern: '<bucket_name>.<virtual_domain>'
+		bucket, _, found := strings.Cut(host, "."+virtualDomain)
+		if !found || bucket == "" {
+			return ctx.Next()
+		}
+		path := ctx.Path()
+		pathStyleUrl := fmt.Sprintf("/%v%v", bucket, path)
+		ctx.Path(pathStyleUrl)
+
+		return ctx.Next()
+	}
+}

s3api/middlewares/logger.go

@@ -15,30 +15,15 @@
 package middlewares
 
 import (
-	"fmt"
-	"log"
-
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/debuglogger"
 )
 
-func RequestLogger(isDebug bool) fiber.Handler {
+func DebugLogger() fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		ctx.Locals("isDebug", isDebug)
-		if isDebug {
-			log.Println("Request headers: ")
-			ctx.Request().Header.VisitAll(func(key, val []byte) {
-				log.Printf("%s: %s", key, val)
-			})
-
-			if ctx.Request().URI().QueryArgs().Len() != 0 {
-				fmt.Println()
-				log.Println("Request query arguments: ")
-				ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
-					log.Printf("%s: %s", key, val)
-				})
-			}
-		}
-
-		return ctx.Next()
+		debuglogger.LogFiberRequestDetails(ctx)
+		err := ctx.Next()
+		debuglogger.LogFiberResponseDetails(ctx)
+		return err
 	}
 }

s3api/middlewares/presign-auth.go

@@ -16,7 +16,7 @@ package middlewares
 
 import (
 	"io"
-	"time"
+	"strconv"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
@@ -30,20 +30,24 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
+		// The bucket is public, no need to check this signature
+		if utils.ContextKeyPublicBucket.IsSet(ctx) {
+			return ctx.Next()
+		}
 		if ctx.Query("X-Amz-Signature") == "" {
 			return ctx.Next()
 		}
 
-		ctx.Locals("region", region)
-		ctx.Locals("startTime", time.Now())
+		// Set in the context the "authenticated" key, in case the authentication succeeds,
+		// otherwise the middleware will return the caucht error
+		utils.ContextKeyAuthenticated.Set(ctx, true)
 
 		authData, err := utils.ParsePresignedURIParts(ctx)
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)
 		}
 
-		ctx.Locals("isRoot", authData.Access == root.Access)
-		ctx.Locals("rootAccess", root.Access)
+		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
@@ -52,9 +56,28 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)
 		}
-		ctx.Locals("account", account)
+		utils.ContextKeyAccount.Set(ctx, account)
+
+		var contentLength int64
+		contentLengthStr := ctx.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+			//TODO: not sure if InvalidRequest should be returned in this case
+			if err != nil {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
+			}
+		}
 
 		if utils.IsBigDataAction(ctx) {
+			// Content-Length has to be set for data uploads: PutObject, UploadPart
+			if contentLengthStr == "" {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingContentLength), logger, mm)
+			}
+			// the upload limit for big data actions: PutObject, UploadPart
+			// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
+			if contentLength > maxObjSizeLimit {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrEntityTooLarge), logger, mm)
+			}
 			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
 				return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
 			})

s3api/middlewares/public-bucket.go

@@ -0,0 +1,298 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func AuthorizePublicBucketAccess(be backend.Backend, l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		// skip for auhtneicated requests
+		if ctx.Query("X-Amz-Algorithm") != "" || ctx.Get("Authorization") != "" {
+			return ctx.Next()
+		}
+
+		bucket, object := parsePath(ctx.Path())
+
+		action, permission, err := detectS3Action(ctx, object == "")
+		if err != nil {
+			return sendResponse(ctx, err, l, mm)
+		}
+
+		err = auth.VerifyPublicAccess(ctx.Context(), be, action, permission, bucket, object)
+		if err != nil {
+			return sendResponse(ctx, err, l, mm)
+		}
+
+		if utils.IsBigDataAction(ctx) {
+			payloadType := ctx.Get("X-Amz-Content-Sha256")
+			if utils.IsUnsignedStreamingPayload(payloadType) {
+				checksumType, err := utils.ExtractChecksumType(ctx)
+				if err != nil {
+					return sendResponse(ctx, err, l, mm)
+				}
+
+				wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+					var cr io.Reader
+					cr, err = utils.NewUnsignedChunkReader(r, checksumType)
+					return cr
+				})
+				if err != nil {
+					return sendResponse(ctx, err, l, mm)
+				}
+			}
+		}
+
+		utils.ContextKeyPublicBucket.Set(ctx, true)
+
+		return ctx.Next()
+	}
+}
+
+func detectS3Action(ctx *fiber.Ctx, isBucketAction bool) (auth.Action, auth.Permission, error) {
+	path := ctx.Path()
+	// ListBuckets is not publically available
+	if path == "/" {
+		//TODO: Still not clear what kind of error should be returned in this case(ListBuckets)
+		return "", auth.PermissionRead, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	queryArgs := ctx.Context().QueryArgs()
+
+	switch ctx.Method() {
+	case fiber.MethodPatch:
+		// Admin apis should always be protected
+		return "", "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	case fiber.MethodHead:
+		// HeadBucket
+		if isBucketAction {
+			return auth.ListBucketAction, auth.PermissionRead, nil
+		}
+
+		// HeadObject
+		return auth.GetObjectAction, auth.PermissionRead, nil
+	case fiber.MethodGet:
+		if isBucketAction {
+			if queryArgs.Has("tagging") {
+				// GetBucketTagging
+				return auth.GetBucketTaggingAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("ownershipControls") {
+				// GetBucketOwnershipControls
+				return auth.GetBucketOwnershipControlsAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousGetBucketOwnership)
+			} else if queryArgs.Has("versioning") {
+				// GetBucketVersioning
+				return auth.GetBucketVersioningAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("policy") {
+				// GetBucketPolicy
+				return auth.GetBucketPolicyAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("cors") {
+				// GetBucketCors
+				return auth.GetBucketCorsAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("versions") {
+				// ListObjectVersions
+				return auth.ListBucketVersionsAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("object-lock") {
+				// GetObjectLockConfiguration
+				return auth.GetBucketObjectLockConfigurationAction, auth.PermissionReadAcp, nil
+			} else if queryArgs.Has("acl") {
+				// GetBucketAcl
+				return auth.GetBucketAclAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("uploads") {
+				// ListMultipartUploads
+				return auth.ListBucketMultipartUploadsAction, auth.PermissionRead, nil
+			} else if queryArgs.GetUintOrZero("list-type") == 2 {
+				// ListObjectsV2
+				return auth.ListBucketAction, auth.PermissionRead, nil
+			}
+			// All the other requests are considerd as ListObjects in the router
+			// no matter what kind of query arguments are provided apart from the ones above
+
+			return auth.ListBucketAction, auth.PermissionRead, nil
+		}
+
+		if queryArgs.Has("tagging") {
+			// GetObjectTagging
+			return auth.GetObjectTaggingAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("retention") {
+			// GetObjectRetention
+			return auth.GetObjectRetentionAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("legal-hold") {
+			// GetObjectLegalHold
+			return auth.GetObjectLegalHoldAction, auth.PermissionReadAcp, nil
+		} else if queryArgs.Has("acl") {
+			// GetObjectAcl
+			return auth.GetObjectAclAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("attributes") {
+			// GetObjectAttributes
+			return auth.GetObjectAttributesAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("uploadId") {
+			// ListParts
+			return auth.ListMultipartUploadPartsAction, auth.PermissionRead, nil
+		}
+
+		// All the other requests are considerd as GetObject in the router
+		// no matter what kind of query arguments are provided apart from the ones above
+		if queryArgs.Has("versionId") {
+			return auth.GetObjectVersionAction, auth.PermissionRead, nil
+		}
+		return auth.GetObjectAction, auth.PermissionRead, nil
+	case fiber.MethodPut:
+		if isBucketAction {
+			if queryArgs.Has("tagging") {
+				// PutBucketTagging
+				return auth.PutBucketTaggingAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("ownershipControls") {
+				// PutBucketOwnershipControls
+				return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership)
+			}
+			if queryArgs.Has("versioning") {
+				// PutBucketVersioning
+				return auth.PutBucketVersioningAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("object-lock") {
+				// PutObjectLockConfiguration
+				return auth.PutBucketObjectLockConfigurationAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("cors") {
+				// PutBucketCors
+				return auth.PutBucketCorsAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("policy") {
+				// PutBucketPolicy
+				return auth.PutBucketPolicyAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("acl") {
+				// PutBucketAcl
+				return auth.PutBucketAclAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousRequest)
+			}
+
+			// All the other rquestes are considered as 'CreateBucket' in the router
+			return "", "", s3err.GetAPIError(s3err.ErrAnonymousRequest)
+		}
+
+		if queryArgs.Has("tagging") {
+			// PutObjectTagging
+			return auth.PutObjectTaggingAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("retention") {
+			// PutObjectRetention
+			return auth.PutObjectRetentionAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("legal-hold") {
+			// PutObjectLegalHold
+			return auth.PutObjectLegalHoldAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("acl") {
+			// PutObjectAcl
+			return auth.PutObjectAclAction, auth.PermissionWriteAcp, s3err.GetAPIError(s3err.ErrAnonymousRequest)
+		}
+		if queryArgs.Has("uploadId") && queryArgs.Has("partNumber") {
+			if ctx.Get("X-Amz-Copy-Source") != "" {
+				// UploadPartCopy
+				//TODO: Add public access check for copy-source
+				// Return AccessDenied for now
+				return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied)
+			}
+
+			utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream())
+			// UploadPart
+			return auth.PutObjectAction, auth.PermissionWrite, nil
+		}
+		if ctx.Get("X-Amz-Copy-Source") != "" {
+			return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousCopyObject)
+		}
+
+		utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream())
+		// All the other requests are considered as 'PutObject' in the router
+		return auth.PutObjectAction, auth.PermissionWrite, nil
+	case fiber.MethodPost:
+		if isBucketAction {
+			// DeleteObjects
+			// FIXME: should be fixed with https://github.com/versity/versitygw/issues/1327
+			// Return AccessDenied for now
+			return auth.DeleteObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+
+		if queryArgs.Has("restore") {
+			return auth.RestoreObjectAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("select") && ctx.Query("select-type") == "2" {
+			// SelectObjectContent
+			return auth.GetObjectAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousRequest)
+		}
+		if queryArgs.Has("uploadId") {
+			// CompleteMultipartUpload
+			return auth.PutObjectAction, auth.PermissionWrite, nil
+		}
+
+		// All the other requests are considered as 'CreateMultipartUpload' in the router
+		return "", "", s3err.GetAPIError(s3err.ErrAnonymousCreateMp)
+	case fiber.MethodDelete:
+		if isBucketAction {
+			if queryArgs.Has("tagging") {
+				// DeleteBucketTagging
+				return auth.PutBucketTaggingAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("ownershipControls") {
+				// DeleteBucketOwnershipControls
+				return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership)
+			}
+			if queryArgs.Has("policy") {
+				// DeleteBucketPolicy
+				return auth.PutBucketPolicyAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("cors") {
+				// DeleteBucketCors
+				return auth.PutBucketCorsAction, auth.PermissionWrite, nil
+			}
+
+			// All the other requests are considered as 'DeleteBucket' in the router
+			return auth.DeleteBucketAction, auth.PermissionWrite, nil
+		}
+
+		if queryArgs.Has("tagging") {
+			// DeleteObjectTagging
+			return auth.PutObjectTaggingAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("uploadId") {
+			// AbortMultipartUpload
+			return auth.AbortMultipartUploadAction, auth.PermissionWrite, nil
+		}
+		// All the other requests are considered as 'DeleteObject' in the router
+		return auth.DeleteObjectAction, auth.PermissionWrite, nil
+	default:
+		// In no action is detected, return AccessDenied ?
+		return "", "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+}
+
+// parsePath extracts the bucket and object names from the path
+func parsePath(path string) (string, string) {
+	p := strings.TrimPrefix(path, "/")
+	bucket, object, _ := strings.Cut(p, "/")
+
+	return bucket, object
+}

s3api/middlewares/set-default-keys.go

@@ -0,0 +1,37 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
+)
+
+func SetDefaultValues(root RootUserConfig, region string) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		// These are necessary for the server access logs
+		utils.ContextKeyRegion.Set(ctx, region)
+		utils.ContextKeyStartTime.Set(ctx, time.Now())
+		utils.ContextKeyRootAccessKey.Set(ctx, root.Access)
+		// Set the account and isRoot to some defulat values, to avoid panics
+		// in case of public buckets
+		utils.ContextKeyAccount.Set(ctx, auth.Account{})
+		utils.ContextKeyIsRoot.Set(ctx, false)
+		return ctx.Next()
+	}
+}

s3api/middlewares/url-decoder.go

@@ -26,7 +26,7 @@ import (
 
 func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
+		unescp, err := url.PathUnescape(string(ctx.Request().URI().PathOriginal()))
 		if err != nil {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}

s3api/router.go

@@ -42,7 +42,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
 
 		// UpdateUser admin api
-		app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
+		app.Patch("/update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
 
 		// ListUsers admin api
 		app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)

s3api/server.go

@@ -29,15 +29,16 @@ import (
 )
 
 type S3ApiServer struct {
-	app      *fiber.App
-	backend  backend.Backend
-	router   *S3ApiRouter
-	port     string
-	cert     *tls.Certificate
-	quiet    bool
-	debug    bool
-	readonly bool
-	health   string
+	app           *fiber.App
+	backend       backend.Backend
+	router        *S3ApiRouter
+	port          string
+	cert          *tls.Certificate
+	quiet         bool
+	debug         bool
+	readonly      bool
+	health        string
+	virtualDomain string
 }
 
 func New(
@@ -76,7 +77,25 @@ func New(
 		})
 	}
 	app.Use(middlewares.DecodeURL(l, mm))
-	app.Use(middlewares.RequestLogger(server.debug))
+
+	// initialize host-style parser in virtual domain is specified
+	if server.virtualDomain != "" {
+		app.Use(middlewares.HostStyleParser(server.virtualDomain))
+	}
+
+	// initilaze the default value setter middleware
+	app.Use(middlewares.SetDefaultValues(root, region))
+
+	// initialize the debug logger in debug mode
+	if server.debug {
+		app.Use(middlewares.DebugLogger())
+	}
+
+	// initialize the bucket/object name validator
+	app.Use(middlewares.BucketObjectNameValidator(l, mm))
+
+	// Public buckets access checker
+	app.Use(middlewares.AuthorizePublicBucketAccess(be, l, mm))
 
 	// Authentication middlewares
 	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
@@ -121,6 +140,11 @@ func WithReadOnly() Option {
 	return func(s *S3ApiServer) { s.readonly = true }
 }
 
+// WithHostStyle enabled host-style bucket addressing on the server
+func WithHostStyle(virtualDomain string) Option {
+	return func(s *S3ApiServer) { s.virtualDomain = virtualDomain }
+}
+
 func (sa *S3ApiServer) Serve() (err error) {
 	if sa.cert != nil {
 		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

s3api/utils/chunk-reader.go

@@ -18,13 +18,19 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/debuglogger"
 	"github.com/versity/versitygw/s3err"
 )
 
+const (
+	maxObjSizeLimit = 5 * 1024 * 1024 * 1024 // 5gb
+)
+
 type payloadType string
 
 const (
@@ -82,11 +88,28 @@ func (c checksumType) isValid() bool {
 		c == checksumTypeCrc64nvme
 }
 
+// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
+func ExtractChecksumType(ctx *fiber.Ctx) (checksumType, error) {
+	trailer := ctx.Get("X-Amz-Trailer")
+	chType := checksumType(strings.ToLower(trailer))
+	if chType != "" && !chType.isValid() {
+		debuglogger.Logf("invalid value for 'X-Amz-Trailer': %v", chType)
+		return "", s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
+	}
+
+	return chType, nil
+}
+
 // IsSpecialPayload checks for special authorization types
 func IsSpecialPayload(str string) bool {
 	return specialValues[payloadType(str)]
 }
 
+// Checks if the provided string is unsigned payload trailer type
+func IsUnsignedStreamingPayload(str string) bool {
+	return payloadType(str) == payloadTypeStreamingUnsignedTrailer
+}
+
 // IsChunkEncoding checks for streaming/unsigned authorization types
 func IsStreamingPayload(str string) bool {
 	pt := payloadType(str)
@@ -96,18 +119,36 @@ func IsStreamingPayload(str string) bool {
 }
 
 func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
-	decContLength := ctx.Get("X-Amz-Decoded-Content-Length")
-	if decContLength == "" {
-		return nil, s3err.GetAPIError(s3err.ErrMissingDecodedContentLength)
+	decContLengthStr := ctx.Get("X-Amz-Decoded-Content-Length")
+	if decContLengthStr == "" {
+		debuglogger.Logf("missing required header 'X-Amz-Decoded-Content-Length'")
+		return nil, s3err.GetAPIError(s3err.ErrMissingContentLength)
 	}
+	decContLength, err := strconv.ParseInt(decContLengthStr, 10, 64)
+	//TODO: not sure if InvalidRequest should be returned in this case
+	if err != nil {
+		debuglogger.Logf("invalid value for 'X-Amz-Decoded-Content-Length': %v", decContLengthStr)
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	if decContLength > maxObjSizeLimit {
+		debuglogger.Logf("the object size exceeds the allowed limit: (size): %v, (limit): %v", decContLength, maxObjSizeLimit)
+		return nil, s3err.GetAPIError(s3err.ErrEntityTooLarge)
+	}
+
 	contentSha256 := payloadType(ctx.Get("X-Amz-Content-Sha256"))
 	if !contentSha256.isValid() {
 		//TODO: Add proper APIError
+		debuglogger.Logf("invalid value for 'X-Amz-Content-Sha256': %v", contentSha256)
 		return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
 	}
 
-	checksumType := checksumType(strings.ToLower(ctx.Get("X-Amz-Trailer")))
-	if contentSha256 != payloadTypeStreamingSigned && !checksumType.isValid() {
+	checksumType, err := ExtractChecksumType(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if contentSha256 != payloadTypeStreamingSigned && checksumType == "" {
+		debuglogger.Logf("empty value for required trailer header 'X-Amz-Trailer': %v", checksumType)
 		return nil, s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
 	}
 
@@ -122,6 +163,7 @@ func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secr
 	// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
 	// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
 	default:
+		debuglogger.Logf("unsupported chunk reader algorithm: %v", contentSha256)
 		return nil, getPayloadTypeNotSupportedErr(contentSha256)
 	}
 }

s3api/utils/context-keys.go

@@ -0,0 +1,65 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"github.com/gofiber/fiber/v2"
+)
+
+// Region, StartTime, IsRoot, Account, AccessKey context locals
+// are set to defualut values in middlewares.SetDefaultValues
+// to avoid the nil interface conversions
+type ContextKey string
+
+const (
+	ContextKeyRegion         ContextKey = "region"
+	ContextKeyStartTime      ContextKey = "start-time"
+	ContextKeyIsRoot         ContextKey = "is-root"
+	ContextKeyRootAccessKey  ContextKey = "root-access-key"
+	ContextKeyAccount        ContextKey = "account"
+	ContextKeyAuthenticated  ContextKey = "authenticated"
+	ContextKeyPublicBucket   ContextKey = "public-bucket"
+	ContextKeyParsedAcl      ContextKey = "parsed-acl"
+	ContextKeySkipResBodyLog ContextKey = "skip-res-body-log"
+	ContextKeyBodyReader     ContextKey = "body-reader"
+)
+
+func (ck ContextKey) Values() []ContextKey {
+	return []ContextKey{
+		ContextKeyRegion,
+		ContextKeyStartTime,
+		ContextKeyIsRoot,
+		ContextKeyRootAccessKey,
+		ContextKeyAccount,
+		ContextKeyAuthenticated,
+		ContextKeyPublicBucket,
+		ContextKeyParsedAcl,
+		ContextKeySkipResBodyLog,
+		ContextKeyBodyReader,
+	}
+}
+
+func (ck ContextKey) Set(ctx *fiber.Ctx, val any) {
+	ctx.Locals(string(ck), val)
+}
+
+func (ck ContextKey) IsSet(ctx *fiber.Ctx) bool {
+	val := ctx.Locals(string(ck))
+	return val != nil
+}
+
+func (ck ContextKey) Get(ctx *fiber.Ctx) any {
+	return ctx.Locals(string(ck))
+}

s3api/utils/crc.go

@@ -0,0 +1,180 @@
+// Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+//
+// This software is provided 'as-is', without any express or implied
+// warranty.  In no event will the authors be held liable for any damages
+// arising from the use of this software.
+//
+// Permission is granted to anyone to use this software for any purpose,
+// including commercial applications, and to alter it and redistribute it
+// freely, subject to the following restrictions:
+//
+// 1. The origin of this software must not be misrepresented; you must not
+//    claim that you wrote the original software. If you use this software
+//    in a product, an acknowledgment in the product documentation would be
+//    appreciated but is not required.
+// 2. Altered source versions must be plainly marked as such, and must not be
+//    misrepresented as being the original software.
+// 3. This notice may not be removed or altered from any source distribution.
+//
+// Jean-loup Gailly        Mark Adler
+// jloup@gzip.org          madler@alumni.caltech.edu
+
+// Original implementation is from
+// https://github.com/vimeo/go-util/blob/8cd4c737f091d9317f72b25df78ce6cf869f7d30/crc32combine/crc32combine.go
+// extended for crc64 support.
+
+// Following is ported from C to Go in 2016 by Justin Ruggles, with minimal alteration.
+// Used uint for unsigned long. Used uint32 for input arguments in order to match
+// the Go hash/crc32 package. zlib CRC32 combine (https://github.com/madler/zlib)
+
+package utils
+
+import (
+	"hash/crc64"
+)
+
+const crc64NVME = 0x9a6c_9329_ac4b_c9b5
+
+var crc64NVMETable = crc64.MakeTable(crc64NVME)
+
+func gf2MatrixTimes(mat []uint64, vec uint64) uint64 {
+	var sum uint64
+
+	for vec != 0 {
+		if vec&1 != 0 {
+			sum ^= mat[0]
+		}
+		vec >>= 1
+		mat = mat[1:]
+	}
+	return sum
+}
+
+func gf2MatrixSquare(square, mat []uint64) {
+	if len(square) != len(mat) {
+		panic("square matrix size mismatch")
+	}
+	for n := range mat {
+		square[n] = gf2MatrixTimes(mat, mat[n])
+	}
+}
+
+// crc32Combine returns the combined CRC-32 hash value of the two passed CRC-32
+// hash values crc1 and crc2. poly represents the generator polynomial
+// and len2 specifies the byte length that the crc2 hash covers.
+func crc32Combine(poly uint32, crc1, crc2 uint32, len2 int64) uint32 {
+	// degenerate case (also disallow negative lengths)
+	if len2 <= 0 {
+		return crc1
+	}
+
+	even := make([]uint64, 32) // even-power-of-two zeros operator
+	odd := make([]uint64, 32)  // odd-power-of-two zeros operator
+
+	// put operator for one zero bit in odd
+	odd[0] = uint64(poly) // CRC-32 polynomial
+	row := uint64(1)
+	for n := 1; n < 32; n++ {
+		odd[n] = row
+		row <<= 1
+	}
+
+	// put operator for two zero bits in even
+	gf2MatrixSquare(even, odd)
+
+	// put operator for four zero bits in odd
+	gf2MatrixSquare(odd, even)
+
+	// apply len2 zeros to crc1 (first square will put the operator for one
+	// zero byte, eight zero bits, in even)
+	crc1n := uint64(crc1)
+	for {
+		// apply zeros operator for this bit of len2
+		gf2MatrixSquare(even, odd)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(even, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+
+		// another iteration of the loop with odd and even swapped
+		gf2MatrixSquare(odd, even)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(odd, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+	}
+
+	// return combined crc
+	crc1n ^= uint64(crc2)
+	return uint32(crc1n)
+}
+
+// crc64Combine returns the combined CRC-64 hash value of the two passed CRC-64
+// hash values crc1 and crc2. poly represents the generator polynomial
+// and len2 specifies the byte length that the crc2 hash covers.
+func crc64Combine(poly uint64, crc1, crc2 uint64, len2 int64) uint64 {
+	// degenerate case (also disallow negative lengths)
+	if len2 <= 0 {
+		return crc1
+	}
+
+	even := make([]uint64, 64) // even-power-of-two zeros operator
+	odd := make([]uint64, 64)  // odd-power-of-two zeros operator
+
+	// put operator for one zero bit in odd
+	odd[0] = poly // CRC-64 polynomial
+	row := uint64(1)
+	for n := 1; n < 64; n++ {
+		odd[n] = row
+		row <<= 1
+	}
+
+	// put operator for two zero bits in even
+	gf2MatrixSquare(even, odd)
+
+	// put operator for four zero bits in odd
+	gf2MatrixSquare(odd, even)
+
+	// apply len2 zeros to crc1 (first square will put the operator for one
+	// zero byte, eight zero bits, in even)
+	crc1n := crc1
+	for {
+		// apply zeros operator for this bit of len2
+		gf2MatrixSquare(even, odd)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(even, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+
+		// another iteration of the loop with odd and even swapped
+		gf2MatrixSquare(odd, even)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(odd, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+	}
+
+	// return combined crc
+	crc1n ^= crc2
+	return crc1n
+}

s3api/utils/crc_test.go

@@ -0,0 +1,57 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"hash/crc32"
+	"hash/crc64"
+	"testing"
+)
+
+func TestCRC32Combine(t *testing.T) {
+	data := []byte("The quick brown fox jumps over the lazy dog")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	var poly uint32 = crc32.IEEE
+	tab := crc32.MakeTable(poly)
+	crc1 := crc32.Checksum(part1, tab)
+	crc2 := crc32.Checksum(part2, tab)
+	combined := crc32Combine(poly, crc1, crc2, int64(len(part2)))
+	full := crc32.Checksum(data, tab)
+
+	if combined != full {
+		t.Errorf("crc32Combine failed: got %08x, want %08x", combined, full)
+	}
+}
+
+func TestCRC64Combine(t *testing.T) {
+	data := []byte("The quick brown fox jumps over the lazy dog")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	var poly uint64 = crc64NVME
+	tab := crc64NVMETable
+	crc1 := crc64.Checksum(part1, tab)
+	crc2 := crc64.Checksum(part2, tab)
+	combined := crc64Combine(poly, crc1, crc2, int64(len(part2)))
+	full := crc64.Checksum(data, tab)
+
+	if combined != full {
+		t.Errorf("crc64Combine failed: got %016x, want %016x", combined, full)
+	}
+}

s3api/utils/csum-reader.go

@@ -26,7 +26,6 @@ import (
 	"hash/crc32"
 	"hash/crc64"
 	"io"
-	"math/bits"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
@@ -89,7 +88,7 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
 	case HashTypeCRC32C:
 		hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
 	case HashTypeCRC64NVME:
-		hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
+		hash = crc64.New(crc64NVMETable)
 	case HashTypeNone:
 		hash = noop{}
 	default:
@@ -185,7 +184,7 @@ func (hr *HashReader) Type() HashType {
 	return hr.hashType
 }
 
-// Md5SumString converts the hash bytes to the string checksum value
+// Base64SumString converts the hash bytes to the b64 encoded string checksum value
 func Base64SumString(b []byte) string {
 	return base64.StdEncoding.EncodeToString(b)
 }
@@ -198,6 +197,108 @@ func (n noop) Reset()                      {}
 func (n noop) Size() int                   { return 0 }
 func (n noop) BlockSize() int              { return 1 }
 
+// IsChecksumComposable tests if the final foll object crc can be calculated
+// based on the part crc values.
+func IsChecksumComposable(algo types.ChecksumAlgorithm) bool {
+	switch algo {
+	case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmCrc64nvme:
+		return true
+	default:
+		return false
+	}
+}
+
+// AddCRCChecksum calculates the composite CRC checksum after adding the part crc.
+// Only CRC32, CRC32C, and CRC64NVME are supported. The input checksums must be base64-encoded strings.
+func AddCRCChecksum(algo types.ChecksumAlgorithm, crc, partCrc string, partLen int64) (string, error) {
+	switch algo {
+	case types.ChecksumAlgorithmCrc32:
+		data, err := base64.StdEncoding.DecodeString(partCrc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode partCrc: %w", err)
+		}
+		if len(data) != 4 {
+			return "", fmt.Errorf("invalid crc32 part checksum length: %d", len(data))
+		}
+		currentCRC, err := base64.StdEncoding.DecodeString(crc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode crc: %w", err)
+		}
+		if len(currentCRC) != 4 {
+			return "", fmt.Errorf("invalid crc32 checksum length: %d", len(currentCRC))
+		}
+
+		currentVal := uint32(currentCRC[0])<<24 | uint32(currentCRC[1])<<16 | uint32(currentCRC[2])<<8 | uint32(currentCRC[3])
+		val := uint32(data[0])<<24 | uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3])
+		composite := crc32Combine(crc32.IEEE, currentVal, val, partLen)
+
+		out := []byte{
+			byte(composite >> 24),
+			byte(composite >> 16),
+			byte(composite >> 8),
+			byte(composite),
+		}
+		return base64.StdEncoding.EncodeToString(out), nil
+	case types.ChecksumAlgorithmCrc32c:
+		data, err := base64.StdEncoding.DecodeString(partCrc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode partCrc: %w", err)
+		}
+		if len(data) != 4 {
+			return "", fmt.Errorf("invalid crc32 part checksum length: %d", len(data))
+		}
+		currentCRC, err := base64.StdEncoding.DecodeString(crc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode crc: %w", err)
+		}
+		if len(currentCRC) != 4 {
+			return "", fmt.Errorf("invalid crc32 checksum length: %d", len(currentCRC))
+		}
+
+		currentVal := uint32(currentCRC[0])<<24 | uint32(currentCRC[1])<<16 | uint32(currentCRC[2])<<8 | uint32(currentCRC[3])
+		val := uint32(data[0])<<24 | uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3])
+		composite := crc32Combine(crc32.Castagnoli, currentVal, val, partLen)
+
+		// Convert composite to big-endian bytes
+		out := []byte{
+			byte(composite >> 24),
+			byte(composite >> 16),
+			byte(composite >> 8),
+			byte(composite),
+		}
+		return base64.StdEncoding.EncodeToString(out), nil
+	case types.ChecksumAlgorithmCrc64nvme:
+		data, err := base64.StdEncoding.DecodeString(partCrc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode partCrc: %w", err)
+		}
+		if len(data) != 8 {
+			return "", fmt.Errorf("invalid crc64 part checksum length: %d", len(data))
+		}
+		currentCRC, err := base64.StdEncoding.DecodeString(crc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode crc: %w", err)
+		}
+		if len(currentCRC) != 8 {
+			return "", fmt.Errorf("invalid crc64 checksum length: %d", len(currentCRC))
+		}
+
+		currentVal := uint64(currentCRC[0])<<56 | uint64(currentCRC[1])<<48 | uint64(currentCRC[2])<<40 | uint64(currentCRC[3])<<32 |
+			uint64(currentCRC[4])<<24 | uint64(currentCRC[5])<<16 | uint64(currentCRC[6])<<8 | uint64(currentCRC[7])
+		val := uint64(data[0])<<56 | uint64(data[1])<<48 | uint64(data[2])<<40 | uint64(data[3])<<32 |
+			uint64(data[4])<<24 | uint64(data[5])<<16 | uint64(data[6])<<8 | uint64(data[7])
+		composite := crc64Combine(crc64NVME, currentVal, val, partLen)
+
+		out := []byte{
+			byte(composite >> 56), byte(composite >> 48), byte(composite >> 40), byte(composite >> 32),
+			byte(composite >> 24), byte(composite >> 16), byte(composite >> 8), byte(composite),
+		}
+		return base64.StdEncoding.EncodeToString(out), nil
+	default:
+		return "", fmt.Errorf("composite checksum not supported for algorithm: %v", algo)
+	}
+}
+
 // NewCompositeChecksumReader initializes a composite checksum
 // processor, which decodes and validates the provided
 // checksums and returns the final checksum based on

s3api/utils/csum-reader_test.go

@@ -0,0 +1,120 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"encoding/base64"
+	"hash/crc32"
+	"hash/crc64"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+func TestAddCRCChecksum_CRC32(t *testing.T) {
+	data := []byte("this is a test buffer for crc32")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	crc1 := crc32.Checksum(part1, crc32.IEEETable)
+	crc2 := crc32.Checksum(part2, crc32.IEEETable)
+	crcFull := crc32.Checksum(data, crc32.IEEETable)
+
+	crc1b := []byte{byte(crc1 >> 24), byte(crc1 >> 16), byte(crc1 >> 8), byte(crc1)}
+	crc2b := []byte{byte(crc2 >> 24), byte(crc2 >> 16), byte(crc2 >> 8), byte(crc2)}
+	crc1b64 := base64.StdEncoding.EncodeToString(crc1b)
+	crc2b64 := base64.StdEncoding.EncodeToString(crc2b)
+
+	combined, err := AddCRCChecksum(types.ChecksumAlgorithmCrc32, crc1b64, crc2b64, int64(len(part2)))
+	if err != nil {
+		t.Fatalf("AddCRCChecksum failed: %v", err)
+	}
+	combinedBytes, err := base64.StdEncoding.DecodeString(combined)
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	combinedVal := uint32(combinedBytes[0])<<24 | uint32(combinedBytes[1])<<16 | uint32(combinedBytes[2])<<8 | uint32(combinedBytes[3])
+	if combinedVal != crcFull {
+		t.Errorf("CRC32 combine mismatch: got %x, want %x", combinedVal, crcFull)
+	}
+}
+
+func TestAddCRCChecksum_CRC32c(t *testing.T) {
+	data := []byte("this is a test buffer for crc32c")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	castagnoli := crc32.MakeTable(crc32.Castagnoli)
+	crc1 := crc32.Checksum(part1, castagnoli)
+	crc2 := crc32.Checksum(part2, castagnoli)
+	crcFull := crc32.Checksum(data, castagnoli)
+
+	crc1b := []byte{byte(crc1 >> 24), byte(crc1 >> 16), byte(crc1 >> 8), byte(crc1)}
+	crc2b := []byte{byte(crc2 >> 24), byte(crc2 >> 16), byte(crc2 >> 8), byte(crc2)}
+	crc1b64 := base64.StdEncoding.EncodeToString(crc1b)
+	crc2b64 := base64.StdEncoding.EncodeToString(crc2b)
+
+	combined, err := AddCRCChecksum(types.ChecksumAlgorithmCrc32c, crc1b64, crc2b64, int64(len(part2)))
+	if err != nil {
+		t.Fatalf("AddCRCChecksum failed: %v", err)
+	}
+	combinedBytes, err := base64.StdEncoding.DecodeString(combined)
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	combinedVal := uint32(combinedBytes[0])<<24 | uint32(combinedBytes[1])<<16 | uint32(combinedBytes[2])<<8 | uint32(combinedBytes[3])
+	if combinedVal != crcFull {
+		t.Errorf("CRC32c combine mismatch: got %x, want %x", combinedVal, crcFull)
+	}
+}
+
+func TestAddCRCChecksum_CRC64NVME(t *testing.T) {
+	data := []byte("this is a test buffer for crc64nvme")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	table := crc64NVMETable
+	crc1 := crc64.Checksum(part1, table)
+	crc2 := crc64.Checksum(part2, table)
+	crcFull := crc64.Checksum(data, table)
+
+	crc1b := []byte{
+		byte(crc1 >> 56), byte(crc1 >> 48), byte(crc1 >> 40), byte(crc1 >> 32),
+		byte(crc1 >> 24), byte(crc1 >> 16), byte(crc1 >> 8), byte(crc1),
+	}
+	crc2b := []byte{
+		byte(crc2 >> 56), byte(crc2 >> 48), byte(crc2 >> 40), byte(crc2 >> 32),
+		byte(crc2 >> 24), byte(crc2 >> 16), byte(crc2 >> 8), byte(crc2),
+	}
+	crc1b64 := base64.StdEncoding.EncodeToString(crc1b)
+	crc2b64 := base64.StdEncoding.EncodeToString(crc2b)
+
+	combined, err := AddCRCChecksum(types.ChecksumAlgorithmCrc64nvme, crc1b64, crc2b64, int64(len(part2)))
+	if err != nil {
+		t.Fatalf("AddCRCChecksum failed: %v", err)
+	}
+	combinedBytes, err := base64.StdEncoding.DecodeString(combined)
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	combinedVal := uint64(combinedBytes[0])<<56 | uint64(combinedBytes[1])<<48 | uint64(combinedBytes[2])<<40 | uint64(combinedBytes[3])<<32 |
+		uint64(combinedBytes[4])<<24 | uint64(combinedBytes[5])<<16 | uint64(combinedBytes[6])<<8 | uint64(combinedBytes[7])
+	if combinedVal != crcFull {
+		t.Errorf("CRC64NVME combine mismatch: got %x, want %x", combinedVal, crcFull)
+	}
+}

s3api/utils/logger.go

@@ -1,55 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package utils
-
-import (
-	"fmt"
-	"log"
-
-	"github.com/gofiber/fiber/v2"
-)
-
-func LogCtxDetails(ctx *fiber.Ctx, respBody []byte) {
-	isDebug, ok := ctx.Locals("isDebug").(bool)
-	_, notLogReqBody := ctx.Locals("logReqBody").(bool)
-	_, notLogResBody := ctx.Locals("logResBody").(bool)
-	if isDebug && ok {
-		// Log request body
-		if !notLogReqBody {
-			fmt.Println()
-			log.Printf("Request Body: %s", ctx.Request().Body())
-		}
-
-		// Log path parameters
-		fmt.Println()
-		log.Println("Path parameters: ")
-		for key, val := range ctx.AllParams() {
-			log.Printf("%s: %s", key, val)
-		}
-
-		// Log response headers
-		fmt.Println()
-		log.Println("Response Headers: ")
-		ctx.Response().Header.VisitAll(func(key, val []byte) {
-			log.Printf("%s: %s", key, val)
-		})
-
-		// Log response body
-		if !notLogResBody && len(respBody) > 0 {
-			fmt.Println()
-			log.Printf("Response body %s", ctx.Response().Body())
-		}
-	}
-}

s3api/utils/name_validate.go

@@ -0,0 +1,24 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+func IsObjectNameValid(name string) bool {
+	switch clean(name) {
+	case "", ".", "..", "/":
+		return false
+	}
+
+	return isObjectLocal(name)
+}

s3api/utils/path.go

@@ -0,0 +1,171 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// code modified from golang std library src/internal/filepathlite/path.go
+// to support path separator '/' for all platforms.
+package utils
+
+import (
+	"strings"
+)
+
+const separator = '/'
+
+// isObjectLocal checks if the given path would result in an object
+// that is local to the bucket.
+func isObjectLocal(path string) bool {
+	if path == "" || path == "." {
+		return true
+	}
+
+	path = strings.Join([]string{".", path}, string(separator))
+
+	hasDots := false
+	for p := path; p != ""; {
+		var part string
+		part, p, _ = strings.Cut(p, "/")
+		if part == "." || part == ".." {
+			hasDots = true
+			break
+		}
+	}
+	if hasDots {
+		path = clean(path)
+	}
+	if path == ".." || strings.HasPrefix(path, "../") {
+		return false
+	}
+	return true
+}
+
+func clean(path string) string {
+	originalPath := path
+	if path == "" {
+		return originalPath + "."
+	}
+	rooted := isPathSeparator(path[0])
+
+	// Invariants:
+	//	reading from path; r is index of next byte to process.
+	//	writing to buf; w is index of next byte to write.
+	//	dotdot is index in buf where .. must stop, either because
+	//		it is the leading slash or it is a leading ../../.. prefix.
+	n := len(path)
+	out := lazybuf{path: path, volAndPath: originalPath, volLen: 0}
+	r, dotdot := 0, 0
+	if rooted {
+		out.append(separator)
+		r, dotdot = 1, 1
+	}
+
+	for r < n {
+		switch {
+		case isPathSeparator(path[r]):
+			// empty path element
+			r++
+		case path[r] == '.' && (r+1 == n || isPathSeparator(path[r+1])):
+			// . element
+			r++
+		case path[r] == '.' && path[r+1] == '.' && (r+2 == n || isPathSeparator(path[r+2])):
+			// .. element: remove to last separator
+			r += 2
+			switch {
+			case out.w > dotdot:
+				// can backtrack
+				out.w--
+				for out.w > dotdot && !isPathSeparator(out.index(out.w)) {
+					out.w--
+				}
+			case !rooted:
+				// cannot backtrack, but not rooted, so append .. element.
+				if out.w > 0 {
+					out.append(separator)
+				}
+				out.append('.')
+				out.append('.')
+				dotdot = out.w
+			}
+		default:
+			// real path element.
+			// add slash if needed
+			if rooted && out.w != 1 || !rooted && out.w != 0 {
+				out.append(separator)
+			}
+			// copy element
+			for ; r < n && !isPathSeparator(path[r]); r++ {
+				out.append(path[r])
+			}
+		}
+	}
+
+	// Turn empty string into "."
+	if out.w == 0 {
+		out.append('.')
+	}
+
+	return FromSlash(out.string())
+}
+
+func isPathSeparator(c uint8) bool {
+	return c == '/'
+}
+
+func FromSlash(path string) string {
+	if separator == '/' {
+		return path
+	}
+	return replaceStringByte(path, '/', separator)
+}
+
+func replaceStringByte(s string, old, new byte) string {
+	if strings.IndexByte(s, old) == -1 {
+		return s
+	}
+	n := []byte(s)
+	for i := range n {
+		if n[i] == old {
+			n[i] = new
+		}
+	}
+	return string(n)
+}
+
+// A lazybuf is a lazily constructed path buffer.
+// It supports append, reading previously appended bytes,
+// and retrieving the final string. It does not allocate a buffer
+// to hold the output until that output diverges from s.
+type lazybuf struct {
+	path       string
+	buf        []byte
+	w          int
+	volAndPath string
+	volLen     int
+}
+
+func (b *lazybuf) index(i int) byte {
+	if b.buf != nil {
+		return b.buf[i]
+	}
+	return b.path[i]
+}
+
+func (b *lazybuf) append(c byte) {
+	if b.buf == nil {
+		if b.w < len(b.path) && b.path[b.w] == c {
+			b.w++
+			return
+		}
+		b.buf = make([]byte, len(b.path))
+		copy(b.buf, b.path[:b.w])
+	}
+	b.buf[b.w] = c
+	b.w++
+}
+
+func (b *lazybuf) string() string {
+	if b.buf == nil {
+		return b.volAndPath[:b.volLen+b.w]
+	}
+	return b.volAndPath[:b.volLen] + string(b.buf[:b.w])
+}

s3api/utils/path_test.go

@@ -0,0 +1,64 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils_test
+
+import (
+	"testing"
+
+	"github.com/versity/versitygw/s3api/utils"
+)
+
+func TestIsObjectNameValid(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  bool
+	}{
+		// valid names
+		{"simple file", "file.txt", true},
+		{"nested file", "dir/file.txt", true},
+		{"absolute nested file", "/dir/file.txt", true},
+		{"trailing slash", "dir/", true},
+		{"slash prefix", "/file.txt", true}, // treated as local after joined with bucket
+		{"dot slash prefix", "./file.txt", true},
+
+		// invalid names
+		{"dot dot only", "..", false},
+		{"dot only", ".", false},
+		{"dot slash", "./", false},
+		{"dot slash dot dot", "./..", false},
+		{"cleans to dot", "./../.", false},
+		{"empty", "", false},
+		{"file escapes 1", "../file.txt", false},
+		{"file escapes 2", "dir/../../file.txt", false},
+		{"file escapes 3", "../../../file.txt", false},
+		{"dir escapes 1", "../dir/", false},
+		{"dir escapes 2", "dir/../../dir/", false},
+		{"dir escapes 3", "../../../dir/", false},
+		{"dot escapes 1", "../.", false},
+		{"dot escapes 2", "dir/../../.", false},
+		{"dot escapes 3", "../../../.", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := utils.IsObjectNameValid(tt.input)
+			if got != tt.want {
+				t.Errorf("%v: IsObjectNameValid(%q) = %v, want %v",
+					tt.name, tt.input, got, tt.want)
+			}
+		})
+	}
+}

s3api/utils/presign-auth-reader.go

@@ -180,7 +180,7 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
 		return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
 	}
 
-	if ctx.Locals("region") != creds[2] {
+	if ContextKeyRegion.Get(ctx) != creds[2] {
 		return a, s3err.APIError{
 			Code:           "SignatureDoesNotMatch",
 			Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),

s3api/utils/signed-chunk-reader.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3api/debuglogger"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -89,11 +90,17 @@ func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string,
 	if chType != "" {
 		checksumHasher, err := getHasher(chType)
 		if err != nil {
+			debuglogger.Logf("failed to initialize hash calculator: %v", err)
 			return nil, err
 		}
 
 		chRdr.checksumHash = checksumHasher
 	}
+	if chType == "" {
+		debuglogger.Infof("initializing signed chunk reader")
+	} else {
+		debuglogger.Infof("initializing signed chunk reader with '%v' trailing checksum", chType)
+	}
 	return chRdr, nil
 }
 
@@ -150,11 +157,13 @@ func (cr *ChunkReader) getStringToSignPrefix(algo string) string {
 func (cr *ChunkReader) getChunkStringToSign() string {
 	prefix := cr.getStringToSignPrefix(streamPayloadAlgo)
 	chunkHash := cr.chunkHash.Sum(nil)
-	return fmt.Sprintf("%s\n%s\n%s\n%s",
+	strToSign := fmt.Sprintf("%s\n%s\n%s\n%s",
 		prefix,
 		cr.prevSig,
 		zeroLenSig,
 		hex.EncodeToString(chunkHash))
+	debuglogger.PrintInsideHorizontalBorders(debuglogger.Purple, "STRING TO SIGN", strToSign, 64)
+	return strToSign
 }
 
 // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
@@ -166,11 +175,15 @@ func (cr *ChunkReader) getTrailerChunkStringToSign() string {
 
 	prefix := cr.getStringToSignPrefix(streamPayloadTrailerAlgo)
 
-	return fmt.Sprintf("%s\n%s\n%s",
+	strToSign := fmt.Sprintf("%s\n%s\n%s",
 		prefix,
 		cr.prevSig,
 		sig,
 	)
+
+	debuglogger.PrintInsideHorizontalBorders(debuglogger.Purple, "TRAILER STRING TO SIGN", strToSign, 64)
+
+	return strToSign
 }
 
 // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
@@ -180,6 +193,7 @@ func (cr *ChunkReader) verifyTrailerSignature() error {
 	sig := hex.EncodeToString(hmac256(cr.signingKey, []byte(strToSign)))
 
 	if sig != cr.trailerSig {
+		debuglogger.Logf("incorrect trailing signature: (calculated): %v, (got): %v", sig, cr.trailerSig)
 		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
 	}
 
@@ -192,6 +206,7 @@ func (cr *ChunkReader) verifyChecksum() error {
 	checksum := base64.StdEncoding.EncodeToString(checksumHash)
 	if checksum != cr.parsedChecksum {
 		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(cr.trailer), "x-amz-checksum-")))
+		debuglogger.Logf("incorrect trailing checksum: (calculated): %v, (got): %v", checksum, cr.parsedChecksum)
 		return s3err.GetChecksumBadDigestErr(algo)
 	}
 
@@ -205,6 +220,7 @@ func (cr *ChunkReader) checkSignature() error {
 	cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
 
 	if cr.prevSig != cr.parsedSig {
+		debuglogger.Logf("incorrect signature: (calculated): %v, (got) %v", cr.prevSig, cr.parsedSig)
 		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
 	}
 	cr.parsedSig = ""
@@ -230,18 +246,20 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		}
 	}
 
-	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n], &n)
+	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
 	if err == errskipHeader {
 		cr.chunkDataLeft = 0
 		return 0, nil
 	}
 	if err != nil {
+		debuglogger.Logf("failed to parse chunk headers: %v", err)
 		return 0, err
 	}
 	cr.parsedSig = sig
 	// If we hit the final chunk, calculate and validate the final
 	// chunk signature and finish reading
 	if chunkSize == 0 {
+		debuglogger.Infof("final chunk parsed:\nchunk size: %v\nsignature: %v\nbuffer offset: %v", chunkSize, sig, bufOffset)
 		cr.chunkHash.Reset()
 		err := cr.checkSignature()
 		if err != nil {
@@ -249,6 +267,7 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		}
 
 		if cr.trailer != "" {
+			debuglogger.Infof("final chunk trailers parsed:\nchecksum: %v\ntrailing signature: %v", cr.parsedChecksum, cr.trailerSig)
 			err := cr.verifyChecksum()
 			if err != nil {
 				return 0, err
@@ -261,6 +280,7 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 
 		return 0, io.EOF
 	}
+	debuglogger.Infof("chunk headers parsed:\nchunk size: %v\nsignature: %v\nbuffer offset: %v", chunkSize, sig, bufOffset)
 
 	// move data up to remove chunk header
 	copy(p, p[bufOffset:n])
@@ -276,6 +296,7 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		}
 		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
 		if (chunkSize + int64(n)) > math.MaxInt {
+			debuglogger.Logf("exceeding the limit of maximum integer allowed: (value): %v, (limit): %v", chunkSize+int64(n), math.MaxInt)
 			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
 		}
 		return n + int(chunkSize), err
@@ -298,6 +319,7 @@ func getSigningKey(secret, region string, date time.Time) []byte {
 	dateRegionKey := hmac256(dateKey, []byte(region))
 	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
 	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
+	debuglogger.Infof("signing key: %s", hex.EncodeToString(signingKey))
 	return signingKey
 }
 
@@ -319,12 +341,14 @@ const (
 // This returns the chunk payload size, signature, data start offset, and
 // error if any. See the AWS documentation for the chunk header format. The
 // header[0] byte is expected to be the first byte of the chunk size here.
-func (cr *ChunkReader) parseChunkHeaderBytes(header []byte, l *int) (int64, string, int, error) {
+func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
 	stashLen := len(cr.stash)
 	if stashLen > maxHeaderSize {
+		debuglogger.Logf("the stash length exceeds the maximum allowed chunk header size: (stash len): %v, (header limit): %v", stashLen, maxHeaderSize)
 		return 0, "", 0, errInvalidChunkFormat
 	}
 	if cr.stash != nil {
+		debuglogger.Logf("recovering the stash: (stash len): %v", stashLen)
 		tmp := make([]byte, stashLen+len(header))
 		copy(tmp, cr.stash)
 		copy(tmp[len(cr.stash):], header)
@@ -336,33 +360,35 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte, l *int) (int64, stri
 
 	// After the first chunk each chunk header should start
 	// with "\n\r\n"
-	if !cr.isFirstHeader && stashLen == 0 {
+	if !cr.isFirstHeader {
 		err := readAndSkip(rdr, '\r', '\n')
 		if err != nil {
+			debuglogger.Logf("failed to read chunk header first 2 bytes: (should be): \\r\\n, (got): %q", header[:2])
 			return cr.handleRdrErr(err, header)
 		}
-
-		copy(header, header[2:])
-		*l = *l - 2
 	}
 
 	// read and parse the chunk size
 	chunkSizeStr, err := readAndTrim(rdr, ';')
 	if err != nil {
+		debuglogger.Logf("failed to read chunk size: %v", err)
 		return cr.handleRdrErr(err, header)
 	}
 	chunkSize, err := strconv.ParseInt(chunkSizeStr, 16, 64)
 	if err != nil {
+		debuglogger.Logf("failed to parse chunk size: (size): %v, (err): %v", chunkSizeStr, err)
 		return 0, "", 0, errInvalidChunkFormat
 	}
 
 	// read the chunk signature
 	err = readAndSkip(rdr, 'c', 'h', 'u', 'n', 'k', '-', 's', 'i', 'g', 'n', 'a', 't', 'u', 'r', 'e', '=')
 	if err != nil {
+		debuglogger.Logf("failed to read 'chunk-signature=': %v", err)
 		return cr.handleRdrErr(err, header)
 	}
 	sig, err := readAndTrim(rdr, '\r')
 	if err != nil {
+		debuglogger.Logf("failed to read '\\r', after chunk signature: %v", err)
 		return cr.handleRdrErr(err, header)
 	}
 
@@ -371,14 +397,17 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte, l *int) (int64, stri
 		if cr.trailer != "" {
 			err = readAndSkip(rdr, '\n')
 			if err != nil {
+				debuglogger.Logf("failed to read \\n before the trailer: %v", err)
 				return cr.handleRdrErr(err, header)
 			}
 			// parse and validate the trailing header
 			trailer, err := readAndTrim(rdr, ':')
 			if err != nil {
+				debuglogger.Logf("failed to read trailer prefix: %v", err)
 				return cr.handleRdrErr(err, header)
 			}
 			if trailer != string(cr.trailer) {
+				debuglogger.Logf("incorrect trailer prefix: (expected): %v, (got): %v", cr.trailer, trailer)
 				return 0, "", 0, errInvalidChunkFormat
 			}
 
@@ -387,30 +416,36 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte, l *int) (int64, stri
 			// parse the checksum
 			checksum, err := readAndTrim(rdr, '\r')
 			if err != nil {
+				debuglogger.Logf("failed to read checksum value: %v", err)
 				return cr.handleRdrErr(err, header)
 			}
 
 			if !IsValidChecksum(checksum, algo) {
+				debuglogger.Logf("invalid checksum value: %v", checksum)
 				return 0, "", 0, s3err.GetInvalidTrailingChecksumHeaderErr(trailer)
 			}
 
 			err = readAndSkip(rdr, '\n')
 			if err != nil {
+				debuglogger.Logf("failed to read \\n after checksum: %v", err)
 				return cr.handleRdrErr(err, header)
 			}
 
 			// parse the trailing signature
 			trailerSigPrefix, err := readAndTrim(rdr, ':')
 			if err != nil {
+				debuglogger.Logf("failed to read trailing signature prefix: %v", err)
 				return cr.handleRdrErr(err, header)
 			}
 
 			if trailerSigPrefix != trailerSignatureHeader {
+				debuglogger.Logf("invalid trailing signature prefix: (expected): %v, (got): %v", trailerSignatureHeader, trailerSigPrefix)
 				return 0, "", 0, errInvalidChunkFormat
 			}
 
 			trailerSig, err := readAndTrim(rdr, '\r')
 			if err != nil {
+				debuglogger.Logf("failed to read trailing signature: %v", err)
 				return cr.handleRdrErr(err, header)
 			}
 
@@ -421,6 +456,7 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte, l *int) (int64, stri
 		// "\r\n\r\n" is followed after the last chunk
 		err = readAndSkip(rdr, '\n', '\r', '\n')
 		if err != nil {
+			debuglogger.Logf("failed to read \\n\\r\\n at the end of chunk header: %v", err)
 			return cr.handleRdrErr(err, header)
 		}
 
@@ -429,19 +465,30 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte, l *int) (int64, stri
 
 	err = readAndSkip(rdr, '\n')
 	if err != nil {
+		debuglogger.Logf("failed to read \\n at the end of chunk header: %v", err)
 		return cr.handleRdrErr(err, header)
 	}
 
-	ind := bytes.Index(header, []byte{'\r', '\n'})
+	// find the index of chunk ending: '\r\n'
+	// skip the first 2 bytes as it is the starting '\r\n'
+	// the first chunk doesn't contain the starting '\r\n', but
+	// anyway, trimming the first 2 bytes doesn't pollute the logic.
+	ind := bytes.Index(header[2:], []byte{'\r', '\n'})
 	cr.isFirstHeader = false
 
-	return chunkSize, sig, ind + len(chunkHdrDelim) - stashLen, nil
+	// the offset is the found index + 4 - the stash length
+	// where:
+	// ind is the index of '\r\n'
+	// 4 specifies the trimmed 2 bytes plus 2 to shift the index at the end of '\r\n'
+	offset := ind + 4 - stashLen
+	return chunkSize, sig, offset, nil
 }
 
 // Stashes the header in cr.stash and returns "errskipHeader"
 func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, error) {
 	cr.stash = make([]byte, len(header))
 	copy(cr.stash, header)
+	debuglogger.Logf("stashing the header: (header length): %v", len(header))
 	return 0, "", 0, errskipHeader
 }
 
@@ -451,6 +498,7 @@ func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, er
 func (cr *ChunkReader) handleRdrErr(err error, header []byte) (int64, string, int, error) {
 	if err == io.EOF {
 		if cr.isEOF {
+			debuglogger.Logf("incomplete chunk encoding, EOF reached")
 			return 0, "", 0, errInvalidChunkFormat
 		}
 		return cr.stashAndSkipHeader(header)

s3api/utils/unsigned-chunk-reader.go

@@ -29,6 +29,8 @@ import (
 	"math/bits"
 	"strconv"
 	"strings"
+
+	"github.com/versity/versitygw/s3api/debuglogger"
 )
 
 var (
@@ -42,27 +44,28 @@ type UnsignedChunkReader struct {
 	expectedChecksum string
 	hasher           hash.Hash
 	stash            []byte
-	chunkCounter     int
 	offset           int
 }
 
 func NewUnsignedChunkReader(r io.Reader, ct checksumType) (*UnsignedChunkReader, error) {
 	hasher, err := getHasher(ct)
 	if err != nil {
+		debuglogger.Logf("failed to initialize hash calculator: %v", err)
 		return nil, err
 	}
+	debuglogger.Infof("initializing unsigned chunk reader")
 	return &UnsignedChunkReader{
 		reader:       bufio.NewReader(r),
 		checksumType: ct,
 		stash:        make([]byte, 0),
 		hasher:       hasher,
-		chunkCounter: 1,
 	}, nil
 }
 
 func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
 	// First read any stashed data
 	if len(ucr.stash) != 0 {
+		debuglogger.Infof("recovering the stash: (stash length): %v", len(ucr.stash))
 		n := copy(p, ucr.stash)
 		ucr.offset += n
 
@@ -89,22 +92,24 @@ func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
 		// Read and cache the payload
 		_, err = io.ReadFull(rdr, payload)
 		if err != nil {
+			debuglogger.Logf("failed to read chunk data: %v", err)
 			return 0, err
 		}
 
 		// Skip the trailing "\r\n"
 		if err := ucr.readAndSkip('\r', '\n'); err != nil {
+			debuglogger.Logf("failed to read trailing \\r\\n after chunk data: %v", err)
 			return 0, err
 		}
 
 		// Copy the payload into the io.Reader buffer
 		n := copy(p[ucr.offset:], payload)
 		ucr.offset += n
-		ucr.chunkCounter++
 
 		if int64(n) < chunkSize {
 			// stash the remaining data
 			ucr.stash = payload[n:]
+			debuglogger.Infof("stashing the remaining data: (stash length): %v", len(ucr.stash))
 			dataRead := ucr.offset
 			ucr.offset = 0
 			return dataRead, nil
@@ -113,6 +118,7 @@ func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
 
 	// Read and validate trailers
 	if err := ucr.readTrailer(); err != nil {
+		debuglogger.Logf("failed to read trailer: %v", err)
 		return 0, err
 	}
 
@@ -142,15 +148,19 @@ func (ucr *UnsignedChunkReader) readAndSkip(data ...byte) error {
 func (ucr *UnsignedChunkReader) extractChunkSize() (int64, error) {
 	line, err := ucr.reader.ReadString('\n')
 	if err != nil {
+		debuglogger.Logf("failed to parse chunk size: %v", err)
 		return 0, errMalformedEncoding
 	}
 	line = strings.TrimSpace(line)
 
 	chunkSize, err := strconv.ParseInt(line, 16, 64)
 	if err != nil {
+		debuglogger.Logf("failed to convert chunk size: %v", err)
 		return 0, errMalformedEncoding
 	}
 
+	debuglogger.Infof("chunk size extracted: %v", chunkSize)
+
 	return chunkSize, nil
 }
 
@@ -161,6 +171,7 @@ func (ucr *UnsignedChunkReader) readTrailer() error {
 	for {
 		v, err := ucr.reader.ReadByte()
 		if err != nil {
+			debuglogger.Logf("failed to read byte: %v", err)
 			if err == io.EOF {
 				return io.ErrUnexpectedEOF
 			}
@@ -173,12 +184,14 @@ func (ucr *UnsignedChunkReader) readTrailer() error {
 		var tmp [3]byte
 		_, err = io.ReadFull(ucr.reader, tmp[:])
 		if err != nil {
+			debuglogger.Logf("failed to read chunk ending: \\n\\r\\n: %v", err)
 			if err == io.EOF {
 				return io.ErrUnexpectedEOF
 			}
 			return err
 		}
 		if !bytes.Equal(tmp[:], trailerDelim) {
+			debuglogger.Logf("incorrect trailer delimiter: (expected): \\n\\r\\n, (got): %q", tmp[:])
 			return errMalformedEncoding
 		}
 		break
@@ -189,15 +202,18 @@ func (ucr *UnsignedChunkReader) readTrailer() error {
 	trailerHeader = strings.TrimSpace(trailerHeader)
 	trailerHeaderParts := strings.Split(trailerHeader, ":")
 	if len(trailerHeaderParts) != 2 {
+		debuglogger.Logf("invalid trailer header parts: %v", trailerHeaderParts)
 		return errMalformedEncoding
 	}
 
 	if trailerHeaderParts[0] != string(ucr.checksumType) {
+		debuglogger.Logf("invalid checksum type: %v", trailerHeaderParts[0])
 		//TODO: handle the error
 		return errMalformedEncoding
 	}
 
 	ucr.expectedChecksum = trailerHeaderParts[1]
+	debuglogger.Infof("parsed the trailing header:\n%v:%v", trailerHeaderParts[0], trailerHeaderParts[1])
 
 	// Validate checksum
 	return ucr.validateChecksum()
@@ -209,6 +225,7 @@ func (ucr *UnsignedChunkReader) validateChecksum() error {
 	checksum := base64.StdEncoding.EncodeToString(csum)
 
 	if checksum != ucr.expectedChecksum {
+		debuglogger.Logf("incorrect checksum: (expected): %v, (got): %v", ucr.expectedChecksum, checksum)
 		return fmt.Errorf("actual checksum: %v, expected checksum: %v", checksum, ucr.expectedChecksum)
 	}
 

s3api/utils/utils.go

@@ -17,6 +17,7 @@ package utils
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
@@ -28,9 +29,9 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/aws/smithy-go/encoding/httpbinding"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3api/debuglogger"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
@@ -40,10 +41,6 @@ var (
 	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
-const (
-	upperhex = "0123456789ABCDEF"
-)
-
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
 	headers.DisableNormalizing()
@@ -69,9 +66,9 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		body = bytes.NewReader(req.Body())
 	}
 
-	escapedURI := escapeOriginalURI(ctx)
+	uri := ctx.OriginalURL()
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
+	httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -124,8 +121,7 @@ func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, cont
 		body = bytes.NewReader(req.Body())
 	}
 
-	uri := string(ctx.Request().URI().Path())
-	uri = httpbinding.EscapePath(uri, false)
+	uri, _, _ := strings.Cut(ctx.OriginalURL(), "?")
 	isFirst := true
 
 	ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
@@ -181,9 +177,11 @@ func ParseUint(str string) (int32, error) {
 	}
 	num, err := strconv.ParseInt(str, 10, 32)
 	if err != nil {
+		debuglogger.Logf("invalid intager provided: %v\n", err)
 		return 1000, fmt.Errorf("invalid int: %w", err)
 	}
 	if num < 0 {
+		debuglogger.Logf("negative intager provided: %v\n", num)
 		return 1000, fmt.Errorf("negative uint: %v", num)
 	}
 	if num > 1000 {
@@ -212,15 +210,18 @@ func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser, bodysize int) {
 
 func IsValidBucketName(bucket string) bool {
 	if len(bucket) < 3 || len(bucket) > 63 {
+		debuglogger.Logf("bucket name length should be in 3-63 range, got: %v\n", len(bucket))
 		return false
 	}
 	// Checks to contain only digits, lowercase letters, dot, hyphen.
 	// Checks to start and end with only digits and lowercase letters.
 	if !bucketNameRegexp.MatchString(bucket) {
+		debuglogger.Logf("invalid bucket name: %v\n", bucket)
 		return false
 	}
 	// Checks not to be a valid IP address
 	if bucketNameIpRegexp.MatchString(bucket) {
+		debuglogger.Logf("bucket name is an ip address: %v\n", bucket)
 		return false
 	}
 	return true
@@ -305,6 +306,7 @@ func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]stru
 			for _, a := range oattrs {
 				attr := s3response.ObjectAttributes(a)
 				if !attr.IsValid() {
+					debuglogger.Logf("invalid object attribute: %v\n", attr)
 					err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
 					break
 				}
@@ -318,6 +320,7 @@ func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]stru
 	}
 
 	if len(attrs) == 0 {
+		debuglogger.Logf("empty get object attributes")
 		return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
 	}
 
@@ -336,6 +339,7 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 	objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")
 
 	if (objLockDate != "" && objLockModeHdr == "") || (objLockDate == "" && objLockModeHdr != "") {
+		debuglogger.Logf("one of 2 required params is missing: (lock date): %v, (lock mode): %v\n", objLockDate, objLockModeHdr)
 		return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)
 	}
 
@@ -343,9 +347,11 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 	if objLockDate != "" {
 		rDate, err := time.Parse(time.RFC3339, objLockDate)
 		if err != nil {
+			debuglogger.Logf("failed to parse retain until date: %v\n", err)
 			return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 		}
 		if rDate.Before(time.Now()) {
+			debuglogger.Logf("expired retain until date: %v\n", rDate.Format(time.RFC3339))
 			return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
 		}
 		retainUntilDate = rDate
@@ -356,12 +362,14 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 	if objLockMode != "" &&
 		objLockMode != types.ObjectLockModeCompliance &&
 		objLockMode != types.ObjectLockModeGovernance {
+		debuglogger.Logf("invalid object lock mode: %v\n", objLockMode)
 		return nil, s3err.GetAPIError(s3err.ErrInvalidObjectLockMode)
 	}
 
 	legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)
 
 	if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
+		debuglogger.Logf("invalid object lock legal hold status: %v\n", legalHold)
 		return nil, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus)
 	}
 
@@ -381,90 +389,40 @@ func IsValidOwnership(val types.ObjectOwnership) bool {
 	case types.ObjectOwnershipObjectWriter:
 		return true
 	default:
+		debuglogger.Logf("invalid object ownership: %v\n", val)
 		return false
 	}
 }
 
-func escapeOriginalURI(ctx *fiber.Ctx) string {
-	path := ctx.Path()
+type ChecksumValues map[types.ChecksumAlgorithm]string
 
-	// Escape the URI original path
-	escapedURI := escapePath(path)
+// Headers concatinates checksum algorithm by prefixing each
+// with 'x-amz-checksum-'
+// e.g.
+// "x-amz-checksum-crc64nvme, x-amz-checksum-sha1"
+func (cv ChecksumValues) Headers() string {
+	result := ""
+	isFirst := false
 
-	// Add the URI query params
-	query := string(ctx.Request().URI().QueryArgs().QueryString())
-	if query != "" {
-		escapedURI = escapedURI + "?" + query
-	}
-
-	return escapedURI
-}
-
-// Escapes the path string
-// Most of the parts copied from std url
-func escapePath(s string) string {
-	hexCount := 0
-	for i := 0; i < len(s); i++ {
-		c := s[i]
-		if shouldEscape(c) {
-			hexCount++
+	for key := range cv {
+		if !isFirst {
+			result += ", "
 		}
+		result += fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(key)))
 	}
-
-	if hexCount == 0 {
-		return s
-	}
-
-	var buf [64]byte
-	var t []byte
-
-	required := len(s) + 2*hexCount
-	if required <= len(buf) {
-		t = buf[:required]
-	} else {
-		t = make([]byte, required)
-	}
-
-	j := 0
-	for i := 0; i < len(s); i++ {
-		switch c := s[i]; {
-		case shouldEscape(c):
-			t[j] = '%'
-			t[j+1] = upperhex[c>>4]
-			t[j+2] = upperhex[c&15]
-			j += 3
-		default:
-			t[j] = s[i]
-			j++
-		}
-	}
-
-	return string(t)
+	return result
 }
 
-// Checks if the character needs to be escaped
-func shouldEscape(c byte) bool {
-	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
-		return false
-	}
-
-	switch c {
-	case '-', '_', '.', '~', '/':
-		return false
-	}
-
-	return true
-}
-
-func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) {
+func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, ChecksumValues, error) {
 	sdkAlgorithm := types.ChecksumAlgorithm(strings.ToUpper(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")))
 
 	err := IsChecksumAlgorithmValid(sdkAlgorithm)
 	if err != nil {
+		debuglogger.Logf("invalid checksum algorithm: %v\n", sdkAlgorithm)
 		return "", nil, err
 	}
 
-	checksums := map[types.ChecksumAlgorithm]string{}
+	checksums := ChecksumValues{}
 
 	var hdrErr error
 	// Parse and validate checksum headers
@@ -477,6 +435,7 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch
 		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(key), "X-Amz-Checksum-")))
 		err := IsChecksumAlgorithmValid(algo)
 		if err != nil {
+			debuglogger.Logf("invalid checksum header: %s\n", key)
 			hdrErr = s3err.GetAPIError(s3err.ErrInvalidChecksumHeader)
 			return
 		}
@@ -489,6 +448,7 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch
 	}
 
 	if len(checksums) > 1 {
+		debuglogger.Logf("multiple checksum headers provided: %v\n", checksums.Headers())
 		return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
 	}
 
@@ -518,15 +478,22 @@ var checksumLengths = map[types.ChecksumAlgorithm]int{
 func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
 	decoded, err := base64.StdEncoding.DecodeString(checksum)
 	if err != nil {
+		debuglogger.Logf("failed to parse checksum base64: %v\n", err)
 		return false
 	}
 
 	expectedLength, exists := checksumLengths[algorithm]
 	if !exists {
+		debuglogger.Logf("unknown checksum algorithm: %v\n", algorithm)
 		return false
 	}
 
-	return len(decoded) == expectedLength
+	isValid := len(decoded) == expectedLength
+	if !isValid {
+		debuglogger.Logf("decoded checksum length: (expected): %v, (got): %v\n", expectedLength, len(decoded))
+	}
+
+	return isValid
 }
 
 func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
@@ -537,6 +504,7 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
 		alg != types.ChecksumAlgorithmSha1 &&
 		alg != types.ChecksumAlgorithmSha256 &&
 		alg != types.ChecksumAlgorithmCrc64nvme {
+		debuglogger.Logf("invalid checksum algorithm: %v\n", alg)
 		return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
 	}
 
@@ -548,6 +516,7 @@ func IsChecksumTypeValid(t types.ChecksumType) error {
 	if t != "" &&
 		t != types.ChecksumTypeComposite &&
 		t != types.ChecksumTypeFullObject {
+		debuglogger.Logf("invalid checksum type: %v\n", t)
 		return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")
 	}
 	return nil
@@ -591,6 +560,7 @@ func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType
 	typeSchema := checksumMap[algo]
 	_, ok := typeSchema[t]
 	if !ok {
+		debuglogger.Logf("checksum type and algorithm mismatch: (type): %v, (algorithm): %v\n", t, algo)
 		return s3err.GetChecksumSchemaMismatchErr(algo, t)
 	}
 
@@ -612,6 +582,7 @@ func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, type
 	// Verify if checksum algorithm is provided, if
 	// checksum type is specified
 	if chType != "" && algo == "" {
+		debuglogger.Logf("checksum type can only be used with checksum algorithm: (type): %v\n", chType)
 		return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)
 	}
 
@@ -634,3 +605,62 @@ func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, type
 
 	return algo, chType, nil
 }
+
+// TagLimit specifies the allowed tag count in a tag set
+type TagLimit int
+
+const (
+	// Tag limit for bucket tagging
+	TagLimitBucket TagLimit = 50
+	// Tag limit for object tagging
+	TagLimitObject TagLimit = 10
+)
+
+// Parses and validates tagging
+func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) {
+	var tagging s3response.TaggingInput
+	err := xml.Unmarshal(data, &tagging)
+	if err != nil {
+		debuglogger.Logf("invalid taggging: %s", data)
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	tLen := len(tagging.TagSet.Tags)
+	if tLen > int(limit) {
+		switch limit {
+		case TagLimitObject:
+			debuglogger.Logf("bucket tagging length exceeds %v: %v", limit, tLen)
+			return nil, s3err.GetAPIError(s3err.ErrObjectTaggingLimited)
+		case TagLimitBucket:
+			debuglogger.Logf("object tagging length exceeds %v: %v", limit, tLen)
+			return nil, s3err.GetAPIError(s3err.ErrBucketTaggingLimited)
+		}
+	}
+
+	tagSet := make(map[string]string, tLen)
+
+	for _, tag := range tagging.TagSet.Tags {
+		// validate tag key
+		if len(tag.Key) == 0 || len(tag.Key) > 128 {
+			debuglogger.Logf("tag key should 0 < tag.Key <= 128, key: %v", tag.Key)
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// validate tag value
+		if len(tag.Value) > 256 {
+			debuglogger.Logf("invalid long tag value: (length): %v, (value): %v", len(tag.Value), tag.Value)
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// make sure there are no duplicate keys
+		_, ok := tagSet[tag.Key]
+		if ok {
+			debuglogger.Logf("duplicate tag key: %v", tag.Key)
+			return nil, s3err.GetAPIError(s3err.ErrDuplicateTagKey)
+		}
+
+		tagSet[tag.Key] = tag.Value
+	}
+
+	return tagSet, nil
+}

s3api/utils/utils_test.go

@@ -16,6 +16,9 @@ package utils
 
 import (
 	"bytes"
+	"encoding/xml"
+	"errors"
+	"math/rand"
 	"net/http"
 	"reflect"
 	"testing"
@@ -25,6 +28,7 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
 
@@ -414,128 +418,6 @@ func TestIsValidOwnership(t *testing.T) {
 	}
 }
 
-func Test_shouldEscape(t *testing.T) {
-	type args struct {
-		c byte
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			name: "shouldn't-escape-alphanum",
-			args: args{
-				c: 'h',
-			},
-			want: false,
-		},
-		{
-			name: "shouldn't-escape-unreserved-char",
-			args: args{
-				c: '_',
-			},
-			want: false,
-		},
-		{
-			name: "shouldn't-escape-unreserved-number",
-			args: args{
-				c: '0',
-			},
-			want: false,
-		},
-		{
-			name: "shouldn't-escape-path-separator",
-			args: args{
-				c: '/',
-			},
-			want: false,
-		},
-		{
-			name: "should-escape-special-char-1",
-			args: args{
-				c: '&',
-			},
-			want: true,
-		},
-		{
-			name: "should-escape-special-char-2",
-			args: args{
-				c: '*',
-			},
-			want: true,
-		},
-		{
-			name: "should-escape-special-char-3",
-			args: args{
-				c: '(',
-			},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := shouldEscape(tt.args.c); got != tt.want {
-				t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_escapePath(t *testing.T) {
-	type args struct {
-		s string
-	}
-	tests := []struct {
-		name string
-		args args
-		want string
-	}{
-		{
-			name: "empty-string",
-			args: args{
-				s: "",
-			},
-			want: "",
-		},
-		{
-			name: "alphanum-path",
-			args: args{
-				s: "/test-bucket/test-key",
-			},
-			want: "/test-bucket/test-key",
-		},
-		{
-			name: "path-with-unescapable-chars",
-			args: args{
-				s: "/test~bucket/test.key",
-			},
-			want: "/test~bucket/test.key",
-		},
-		{
-			name: "path-with-escapable-chars",
-			args: args{
-				s: "/bucket-*(/test=key&",
-			},
-			want: "/bucket-%2A%28/test%3Dkey%26",
-		},
-		{
-			name: "path-with-space",
-			args: args{
-				s: "/test-bucket/my key",
-			},
-			want: "/test-bucket/my%20key",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := escapePath(tt.args.s); got != tt.want {
-				t.Errorf("escapePath() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func TestIsChecksumAlgorithmValid(t *testing.T) {
 	type args struct {
 		alg types.ChecksumAlgorithm
@@ -857,3 +739,162 @@ func Test_checkChecksumTypeAndAlgo(t *testing.T) {
 		})
 	}
 }
+
+func TestParseTagging(t *testing.T) {
+	genRandStr := func(lgth int) string {
+		b := make([]byte, lgth)
+		for i := range b {
+			b[i] = byte(rand.Intn(95) + 32) // 126 - 32 + 1 = 95 printable characters
+		}
+		return string(b)
+	}
+	getTagSet := func(lgth int) s3response.TaggingInput {
+		res := s3response.TaggingInput{
+			TagSet: s3response.TagSet{
+				Tags: []s3response.Tag{},
+			},
+		}
+
+		for i := 0; i < lgth; i++ {
+			res.TagSet.Tags = append(res.TagSet.Tags, s3response.Tag{
+				Key:   genRandStr(10),
+				Value: genRandStr(20),
+			})
+		}
+
+		return res
+	}
+	type args struct {
+		data        s3response.TaggingInput
+		overrideXML []byte
+		limit       TagLimit
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    map[string]string
+		wantErr error
+	}{
+		{
+			name: "valid tags within limit",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{
+							{Key: "key1", Value: "value1"},
+							{Key: "key2", Value: "value2"},
+						},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    map[string]string{"key1": "value1", "key2": "value2"},
+			wantErr: nil,
+		},
+		{
+			name: "malformed XML",
+			args: args{
+				overrideXML: []byte("invalid xml"),
+				limit:       TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrMalformedXML),
+		},
+		{
+			name: "exceeds bucket tag limit",
+			args: args{
+				data:  getTagSet(51),
+				limit: TagLimitBucket,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrBucketTaggingLimited),
+		},
+		{
+			name: "exceeds object tag limit",
+			args: args{
+				data:  getTagSet(11),
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrObjectTaggingLimited),
+		},
+		{
+			name: "invalid 0 length tag key",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{{Key: "", Value: "value1"}},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrInvalidTagKey),
+		},
+		{
+			name: "invalid long tag key",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{{Key: genRandStr(130), Value: "value1"}},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrInvalidTagKey),
+		},
+		{
+			name: "invalid long tag value",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{{Key: "key", Value: genRandStr(257)}},
+					},
+				},
+				limit: TagLimitBucket,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrInvalidTagValue),
+		},
+		{
+			name: "duplicate tag key",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{
+							{Key: "key", Value: "value1"},
+							{Key: "key", Value: "value2"},
+						},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrDuplicateTagKey),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var data []byte
+			if tt.args.overrideXML != nil {
+				data = tt.args.overrideXML
+			} else {
+				var err error
+				data, err = xml.Marshal(tt.args.data)
+				if err != nil {
+					t.Fatalf("error marshalling input: %v", err)
+				}
+			}
+			got, err := ParseTagging(data, tt.args.limit)
+
+			if !errors.Is(err, tt.wantErr) {
+				t.Errorf("expected error %v, got %v", tt.wantErr, err)
+			}
+			if err == nil && !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("expected result %v, got %v", tt.want, got)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -59,6 +59,11 @@ type ErrorCode int
 const (
 	ErrNone ErrorCode = iota
 	ErrAccessDenied
+	ErrAnonymousRequest
+	ErrAnonymousCreateMp
+	ErrAnonymousCopyObject
+	ErrAnonymousPutBucketOwnership
+	ErrAnonymousGetBucketOwnership
 	ErrMethodNotAllowed
 	ErrBucketNotEmpty
 	ErrVersionedBucketNotEmpty
@@ -84,7 +89,12 @@ const (
 	ErrInvalidCopyDest
 	ErrInvalidCopySource
 	ErrInvalidCopySourceRange
-	ErrInvalidTag
+	ErrInvalidTagKey
+	ErrInvalidTagValue
+	ErrDuplicateTagKey
+	ErrBucketTaggingLimited
+	ErrObjectTaggingLimited
+	ErrInvalidURLEncodedTagging
 	ErrAuthHeaderEmpty
 	ErrSignatureVersionNotSupported
 	ErrMalformedPOSTRequest
@@ -113,7 +123,7 @@ const (
 	ErrSignatureTerminationStr
 	ErrSignatureIncorrService
 	ErrContentSHA256Mismatch
-	ErrMissingDecodedContentLength
+	ErrMissingContentLength
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
 	ErrMissingDateHeader
@@ -157,6 +167,7 @@ const (
 	ErrChecksumTypeWithAlgo
 	ErrInvalidChecksumHeader
 	ErrTrailerHeaderNotSupported
+	ErrBadRequest
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
@@ -181,6 +192,31 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Access Denied.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrAnonymousRequest: {
+		Code:           "AccessDenied",
+		Description:    "Anonymous users cannot invoke this API. Please authenticate.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAnonymousCreateMp: {
+		Code:           "AccessDenied",
+		Description:    "Anonymous users cannot initiate multipart uploads. Please authenticate.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAnonymousCopyObject: {
+		Code:           "AccessDenied",
+		Description:    "Anonymous users cannot copy objects. Please authenticate.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAnonymousPutBucketOwnership: {
+		Code:           "AccessDenied",
+		Description:    "s3:PutBucketOwnershipControls does not support Anonymous requests!",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAnonymousGetBucketOwnership: {
+		Code:           "AccessDenied",
+		Description:    "s3:GetBucketOwnershipControls does not support Anonymous requests!",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrMethodNotAllowed: {
 		Code:           "MethodNotAllowed",
 		Description:    "The specified method is not allowed against this resource.",
@@ -306,9 +342,34 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidTag: {
+	ErrInvalidTagKey: {
+		Code:           "InvalidTag",
+		Description:    "The TagKey you have provided is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidTagValue: {
+		Code:           "InvalidTag",
+		Description:    "The TagValue you have provided is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrDuplicateTagKey: {
+		Code:           "InvalidTag",
+		Description:    "Cannot provide multiple Tags with the same key",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketTaggingLimited: {
+		Code:           "BadRequest",
+		Description:    "Bucket tag count cannot be greater than 50",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectTaggingLimited: {
+		Code:           "BadRequest",
+		Description:    "Object tags cannot be greater than 10",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidURLEncodedTagging: {
 		Code:           "InvalidArgument",
-		Description:    "The Tag value you have provided is invalid",
+		Description:    "The header 'x-amz-tagging' shall be encoded as UTF-8 then URLEncoded URL query parameters without tag name duplicates.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMalformedXML: {
@@ -456,7 +517,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrMissingDecodedContentLength: {
+	ErrMissingContentLength: {
 		Code:           "MissingContentLength",
 		Description:    "You must provide the Content-Length HTTP header.",
 		HTTPStatusCode: http.StatusLengthRequired,
@@ -493,7 +554,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrInvalidRange: {
 		Code:           "InvalidRange",
-		Description:    "The requested range is not valid for the request. Try another range.",
+		Description:    "The requested range is not satisfiable",
 		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
 	},
 	ErrInvalidURI: {
@@ -666,6 +727,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The value specified in the x-amz-trailer header is not supported",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrBadRequest: {
+		Code:           "400",
+		Description:    "Bad Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {

s3event/event.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
 )
 
 type S3EventSender interface {
@@ -141,15 +142,20 @@ func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
 
 func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) EventSchema {
 	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
-	acc := ctx.Locals("account").(auth.Account)
+
+	var bucket, object string
+	if len(path) > 1 {
+		bucket, object = path[1], strings.Join(path[2:], "/")
+	}
+
+	acc := utils.ContextKeyAccount.Get(ctx).(auth.Account)
 
 	return EventSchema{
 		Records: []EventRecord{
 			{
 				EventVersion: "2.2",
 				EventSource:  "aws:s3",
-				AwsRegion:    ctx.Locals("region").(string),
+				AwsRegion:    utils.ContextKeyRegion.Get(ctx).(string),
 				EventTime:    time.Now().Format(time.RFC3339),
 				EventName:    meta.EventName,
 				UserIdentity: EventUserIdentity{

s3log/file.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -68,10 +69,16 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 	access := "-"
 	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
+	var bucket, object string
+	if len(path) > 1 {
+		bucket, object = path[1], strings.Join(path[2:], "/")
+	}
 	errorCode := ""
 	httpStatus := 200
-	startTime := ctx.Locals("startTime").(time.Time)
+	startTime, ok := utils.ContextKeyStartTime.Get(ctx).(time.Time)
+	if !ok {
+		startTime = time.Now()
+	}
 	tlsConnState := ctx.Context().TLSConnectionState()
 	if tlsConnState != nil {
 		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
@@ -89,9 +96,9 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 		}
 	}
 
-	switch ctx.Locals("account").(type) {
-	case auth.Account:
-		access = ctx.Locals("account").(auth.Account).Access
+	acct, ok := utils.ContextKeyAccount.Get(ctx).(auth.Account)
+	if ok {
+		access = acct.Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner
@@ -115,7 +122,7 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 	lf.HostID = ctx.Get("X-Amz-Id-2")
 	lf.SignatureVersion = "SigV4"
 	lf.AuthenticationType = "AuthHeader"
-	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", utils.ContextKeyRegion.Get(ctx).(string))
 	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
 	lf.AclRequired = "Yes"
 

s3log/file_admin.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
 )
 
 // FileLogger is a local file audit log
@@ -57,7 +58,10 @@ func (f *AdminFileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMe
 	access := "-"
 	reqURI := ctx.OriginalURL()
 	errorCode := ""
-	startTime := ctx.Locals("startTime").(time.Time)
+	startTime, ok := utils.ContextKeyStartTime.Get(ctx).(time.Time)
+	if !ok {
+		startTime = time.Now()
+	}
 	tlsConnState := ctx.Context().TLSConnectionState()
 	if tlsConnState != nil {
 		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
@@ -68,9 +72,9 @@ func (f *AdminFileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMe
 		errorCode = err.Error()
 	}
 
-	switch ctx.Locals("account").(type) {
+	switch utils.ContextKeyAccount.Get(ctx).(type) {
 	case auth.Account:
-		access = ctx.Locals("account").(auth.Account).Access
+		access = utils.ContextKeyAccount.Get(ctx).(auth.Account).Access
 	}
 
 	lf.Time = time.Now()

s3log/webhook.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -65,10 +66,16 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 	access := "-"
 	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
+	var bucket, object string
+	if len(path) > 1 {
+		bucket, object = path[1], strings.Join(path[2:], "/")
+	}
 	errorCode := ""
 	httpStatus := 200
-	startTime := ctx.Locals("startTime").(time.Time)
+	startTime, ok := utils.ContextKeyStartTime.Get(ctx).(time.Time)
+	if !ok {
+		startTime = time.Now()
+	}
 	tlsConnState := ctx.Context().TLSConnectionState()
 	if tlsConnState != nil {
 		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
@@ -86,9 +93,9 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 		}
 	}
 
-	switch ctx.Locals("account").(type) {
-	case auth.Account:
-		access = ctx.Locals("account").(auth.Account).Access
+	acct, ok := utils.ContextKeyAccount.Get(ctx).(auth.Account)
+	if ok {
+		access = acct.Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner
@@ -112,7 +119,7 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 	lf.HostID = ctx.Get("X-Amz-Id-2")
 	lf.SignatureVersion = "SigV4"
 	lf.AuthenticationType = "AuthHeader"
-	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", utils.ContextKeyRegion.Get(ctx).(string))
 	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
 	lf.AclRequired = "Yes"
 

s3response/s3response.go

@@ -62,7 +62,7 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 		Alias: (*Alias)(&p),
 	}
 
-	aux.LastModified = p.LastModified.UTC().Format(iso8601TimeFormat)
+	aux.LastModified = p.LastModified.UTC().Format(time.RFC3339)
 
 	return e.EncodeElement(aux, start)
 }
@@ -172,7 +172,7 @@ type ListObjectsV2Result struct {
 	Name                  *string
 	Prefix                *string
 	StartAfter            *string
-	ContinuationToken     *string
+	ContinuationToken     *string `xml:"ContinuationToken,omitempty"`
 	NextContinuationToken *string
 	KeyCount              *int32
 	MaxKeys               *int32
@@ -198,15 +198,14 @@ type Object struct {
 func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Object
 	aux := &struct {
-		LastModified *string `xml:"LastModified,omitempty"`
+		LastModified string `xml:"LastModified,omitempty"`
 		*Alias
 	}{
 		Alias: (*Alias)(&o),
 	}
 
 	if o.LastModified != nil {
-		formattedTime := o.LastModified.UTC().Format(iso8601TimeFormat)
-		aux.LastModified = &formattedTime
+		aux.LastModified = o.LastModified.UTC().Format(time.RFC3339)
 	}
 
 	return e.EncodeElement(aux, start)
@@ -233,7 +232,7 @@ func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 		Alias: (*Alias)(&u),
 	}
 
-	aux.Initiated = u.Initiated.UTC().Format(iso8601TimeFormat)
+	aux.Initiated = u.Initiated.UTC().Format(time.RFC3339)
 
 	return e.EncodeElement(aux, start)
 }
@@ -330,7 +329,7 @@ func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement
 		Alias: (*Alias)(&r),
 	}
 
-	aux.CreationDate = r.CreationDate.UTC().Format(iso8601TimeFormat)
+	aux.CreationDate = r.CreationDate.UTC().Format(time.RFC3339)
 
 	return e.EncodeElement(aux, start)
 }
@@ -344,11 +343,44 @@ type CanonicalUser struct {
 	DisplayName string
 }
 
+type CopyObjectOutput struct {
+	BucketKeyEnabled        *bool
+	CopyObjectResult        *CopyObjectResult
+	CopySourceVersionId     *string
+	Expiration              *string
+	SSECustomerAlgorithm    *string
+	SSECustomerKeyMD5       *string
+	SSEKMSEncryptionContext *string
+	SSEKMSKeyId             *string
+	ServerSideEncryption    types.ServerSideEncryption
+	VersionId               *string
+}
+
 type CopyObjectResult struct {
-	XMLName             xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
-	LastModified        time.Time
-	ETag                string
-	CopySourceVersionId string `xml:"-"`
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumCRC64NVME *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumType      types.ChecksumType
+	ETag              *string
+	LastModified      *time.Time
+}
+
+func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias CopyObjectResult
+	aux := &struct {
+		LastModified string `xml:"LastModified,omitempty"`
+		*Alias
+	}{
+		Alias: (*Alias)(&r),
+	}
+	if r.LastModified != nil {
+		aux.LastModified = r.LastModified.UTC().Format(time.RFC3339)
+	}
+
+	return e.EncodeElement(aux, start)
 }
 
 type CopyPartResult struct {
@@ -365,20 +397,35 @@ type CopyPartResult struct {
 	CopySourceVersionId string `xml:"-"`
 }
 
-func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	type Alias CopyObjectResult
+func (r CopyPartResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias CopyPartResult
 	aux := &struct {
-		LastModified string `xml:"LastModified"`
+		LastModified string `xml:"LastModified,omitempty"`
 		*Alias
 	}{
 		Alias: (*Alias)(&r),
 	}
-
-	aux.LastModified = r.LastModified.UTC().Format(iso8601TimeFormat)
+	if !r.LastModified.IsZero() {
+		aux.LastModified = r.LastModified.UTC().Format(time.RFC3339)
+	}
 
 	return e.EncodeElement(aux, start)
 }
 
+type CompleteMultipartUploadResult struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
+	Location          *string
+	Bucket            *string
+	Key               *string
+	ETag              *string
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+	ChecksumType      *types.ChecksumType
+}
+
 type AccessControlPolicy struct {
 	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
 	Owner             CanonicalUser
@@ -433,7 +480,37 @@ type ListVersionsResult struct {
 	NextVersionIdMarker *string
 	Prefix              *string
 	VersionIdMarker     *string
-	Versions            []types.ObjectVersion `xml:"Version"`
+	Versions            []ObjectVersion `xml:"Version"`
+}
+
+type ObjectVersion struct {
+	ChecksumAlgorithm []types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
+	ETag              *string
+	IsLatest          *bool
+	Key               *string
+	LastModified      *time.Time
+	Owner             *types.Owner
+	RestoreStatus     *types.RestoreStatus
+	Size              *int64
+	StorageClass      types.ObjectVersionStorageClass
+	VersionId         *string
+}
+
+func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias ObjectVersion
+	aux := &struct {
+		LastModified string `xml:"LastModified"`
+		*Alias
+	}{
+		Alias: (*Alias)(&o),
+	}
+
+	if o.LastModified != nil {
+		aux.LastModified = o.LastModified.UTC().Format(time.RFC3339)
+	}
+
+	return e.EncodeElement(aux, start)
 }
 
 type GetBucketVersioningOutput struct {

tests/.env.docker.default

@@ -25,5 +25,8 @@ USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 RECREATE_BUCKETS=true
+DELETE_BUCKETS_AFTER_TEST=true
 REMOVE_TEST_FILE_FOLDER=true
+AUTOGENERATE_USERS=true
+USER_AUTOGENERATION_PREFIX=versitygw-docker-
 VERSIONING_DIR=/tmp/versioning

tests/Dockerfile_test_bats

@@ -21,6 +21,7 @@ RUN apt-get update && \
     jq \
     bc \
     libxml2-utils \
+    xmlstarlet \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*

tests/README.md

@@ -110,6 +110,11 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 **ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**:  for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account.
 
+**USER_ID_{role}_{id}**, **USERNAME_{role}_{id}**, **PASSWORD_{role}_{id}**:  for setup_user_v2 non-autocreated users, the format for the user.
+* example:  USER_ID_USER_1={name}:  user ID corresponding to the first user with **user** permissions in the test.
+
+#### 
+
 ### Non-Secret
 
 **VERSITY_EXE**:  location of the versity executable relative to test folder.
@@ -146,11 +151,13 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 **DIRECT**:  if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes).
 
-**DIRECT_DISPLAY_NAME**:  username if **DIRECT** is set to **true**.
+**DIRECT_DISPLAY_NAME**:  AWS ACL main user display name if **DIRECT** is set to **true**.
+
+**DIRECT_AWS_USER_ID**:  AWS policy 12-digit user ID if **DIRECT** is set to **true**.
 
 **COVERAGE_DB**:  database to store client command coverage info and usage counts, if using.
 
-**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  credentials for users created and tested for non-root user **versitygw** operations.
+**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  setup_user (v1), credentials for users created and tested for non-root user **versitygw** operations (non-setup_user_v2).
 
 **TEST_FILE_FOLDER**:  where to put temporary test files.
 
@@ -162,10 +169,22 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 **TIME_LOG**:  optional log to show duration of individual tests
 
-**DIRECT_S3_ROOT_ACCOUNT_NAME**:  for direct mode, S3 username
+**DIRECT_S3_ROOT_ACCOUNT_NAME**:  for direct mode, S3 username for user with root permissions
 
 **DELETE_BUCKETS_AFTER_TEST**:  whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked
 
+**AUTOGENERATE_USERS**:  setup_user_v2, whether or not to autocreate users for tests.  If set to **false**, users must be pre-created (see `Secret` section above).
+
+**USER_AUTOGENERATION_PREFIX**:  setup_user_v2, if **AUTOCREATE_USERS** is set to **true**, the prefix for the autocreated username. 
+
+**CREATE_STATIC_USERS_IF_NONEXISTENT**:  setup_user_v2, if **AUTOCREATE_USERS** is set to **false**, generate non-existing users if they don't exist, but don't delete them, as with user autogeneration
+
+**DIRECT_POST_COMMAND_DELAY**:  in direct mode, time to wait before sending new commands to try to prevent propagation delay issues
+
+**SKIP_ACL_TESTING**:  avoid ACL tests for systems which do not use ACLs
+
+**MAX_FILE_DOWNLOAD_CHUNK_SIZE**:  when set, will divide the download of large files with GetObject into chunks of the given size.  Useful for direct testing with slower connections.
+
 ## REST Scripts
 
 REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder.  To call a script, the following parameters are needed:

tests/commands/abort_multipart_upload.sh

@@ -27,6 +27,21 @@ abort_multipart_upload() {
   return 0
 }
 
+abort_multipart_upload_rest() {
+  if ! check_param_count "abort_multipart_upload_rest" "bucket, key, upload ID" 3 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then
+    log 2 "error aborting multipart upload: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/result.txt)"
+    return 1
+  fi
+  return 0
+}
+
 abort_multipart_upload_with_user() {
   if [ $# -ne 5 ]; then
     log 2 "'abort multipart upload' command requires bucket, key, upload ID, username, password"

tests/commands/command.sh

@@ -27,8 +27,18 @@ send_command() {
     fi
     # shellcheck disable=SC2154
     echo "${masked_args[*]}" >> "$COMMAND_LOG"
-    "$@"
-    return $?
   fi
-  "$@"
+  local command_result=0
+  "$@" || command_result=$?
+  if [ "$command_result" -ne 0 ]; then
+    if [ "$1" == "curl" ]; then
+      echo ", curl response code: $command_result"
+    elif [ "$command_result" -ne 1 ]; then
+      echo " ($1 response code: $command_result)"
+    fi
+  fi
+  if [ "$DIRECT" == "true" ]; then
+    sleep "$DIRECT_POST_COMMAND_DELAY"
+  fi
+  return $command_result
 }

tests/commands/copy_object.sh

@@ -44,13 +44,13 @@ copy_object() {
 }
 
 copy_object_empty() {
-  record-command "copy-object" "client:s3api"
+  record_command "copy-object" "client:s3api"
   error=$(send_command aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
   if [[ $result -eq 0 ]]; then
     log 2 "copy object with empty parameters returned no error"
     return 1
   fi
-  if [[ $error != *"the following arguments are required: --bucket, --copy-source, --key" ]]; then
+  if [[ $error != *"the following arguments are required: --bucket, --copy-source, --key"* ]]; then
     log 2 "copy object with no params returned mismatching error: $error"
     return 1
   fi

tests/commands/create_bucket.sh

@@ -20,15 +20,14 @@ source ./tests/report.sh
 # param:  bucket name
 # return 0 for success, 1 for failure
 create_bucket() {
-  if [ $# -ne 2 ]; then
-    log 2 "create bucket missing command type, bucket name"
+  log 6 "create_bucket"
+  if ! check_param_count "create_bucket" "command type, bucket" 2 $#; then
     return 1
   fi
 
   record_command "create-bucket" "client:$1"
   local exit_code=0
   local error
-  log 6 "create bucket"
   if [[ $1 == 's3' ]]; then
     error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]]; then
@@ -50,8 +49,8 @@ create_bucket() {
 }
 
 create_bucket_with_user() {
-  if [ $# -ne 4 ]; then
-    log 2 "create bucket missing command type, bucket name, access, secret"
+  log 6 "create_bucket_with_user"
+  if ! check_param_count "create_bucket_with_user" "command type, bucket, access ID, secret key" 4 $#; then
     return 1
   fi
   local exit_code=0
@@ -73,9 +72,9 @@ create_bucket_with_user() {
 }
 
 create_bucket_object_lock_enabled() {
+  log 6 "create_bucket_object_lock_enabled"
   record_command "create-bucket" "client:s3api"
-  if [ $# -ne 1 ]; then
-    log 2 "create bucket missing bucket name"
+  if ! check_param_count "create_bucket_object_lock_enabled" "bucket" 1 $#; then
     return 1
   fi
 

tests/commands/create_multipart_upload.sh

@@ -122,5 +122,11 @@ create_multipart_upload_rest() {
     log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")"
     return 1
   fi
+  log 5 "result: $(cat "$TEST_FILE_FOLDER/output.txt")"
+  if ! upload_id=$(get_element_text "$TEST_FILE_FOLDER/output.txt" "InitiateMultipartUploadResult" "UploadId"); then
+    log 2 "error getting upload ID: $upload_id"
+    return 1
+  fi
+  echo "$upload_id"
   return 0
 }

tests/commands/delete_bucket.sh

@@ -50,4 +50,19 @@ delete_bucket() {
     return 1
   fi
   return 0
+}
+
+delete_bucket_rest() {
+  if ! check_param_count "delete_bucket_rest" "bucket" 1 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_bucket.sh 2>&1); then
+    log 2 "error deleting bucket: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    log 2 "expected '204', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
 }

tests/commands/delete_bucket_policy.sh

@@ -38,6 +38,21 @@ delete_bucket_policy() {
   return 0
 }
 
+delete_bucket_policy_rest() {
+  if ! check_param_count "delete_bucket_policy_rest" "bucket" 1 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_bucket_policy.sh 2>&1); then
+    log 2 "error deleting bucket policy: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    log 2 "expected '204', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
+}
+
 delete_bucket_policy_with_user() {
   record_command "delete-bucket-policy" "client:s3api"
   if [[ $# -ne 3 ]]; then

tests/commands/delete_object.sh

@@ -18,8 +18,7 @@
 delete_object() {
   log 6 "delete_object"
   record_command "delete-object" "client:$1"
-  if [ $# -ne 3 ]; then
-    log 2 "delete object command requires command type, bucket, key"
+  if ! check_param_count "delete_object" "command type, bucket, key" 3 $#; then
     return 1
   fi
   local exit_code=0
@@ -46,21 +45,38 @@ delete_object() {
   return 0
 }
 
-delete_object_bypass_retention() {
-  if [[ $# -ne 4 ]]; then
-    log 2 "'delete-object with bypass retention' requires bucket, key, user, password"
+# shellcheck disable=SC2317
+delete_object_rest() {
+  if ! check_param_count "delete_object_rest" "bucket, key" 2 $#; then
     return 1
   fi
-  if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
-    log 2 "error deleting object with bypass retention: $delete_object_error"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
+    log 2 "expected '204', was '$result' ($delete_object_error)"
+    return 1
+  fi
+  return 0
+}
+
+delete_object_bypass_retention() {
+  if ! check_param_count "delete_object_bypass_retention" "bucket, key, user, password" 4 $#; then
+    return 1
+  fi
+  if ! result=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" \
+      COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" BYPASS_GOVERNANCE_RETENTION="true" \
+      OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
     return 1
   fi
   return 0
 }
 
 delete_object_version() {
-  if [[ $# -ne 3 ]]; then
-    log 2 "'delete_object_version' requires bucket, key, version ID"
+  if ! check_param_count "delete_object_version" "bucket, key, version ID" 3 $#; then
     return 1
   fi
   if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then
@@ -70,9 +86,24 @@ delete_object_version() {
   return 0
 }
 
+delete_object_version_rest() {
+  if ! check_param_count "delete_object_version_rest" "bucket name, object name, version ID" 3 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
+    log 2 "expected '204', was '$result' ($delete_object_error)"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_version_bypass_retention() {
-  if [[ $# -ne 3 ]]; then
-    log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID"
+  if ! check_param_count "delete_object_version_bypass_retention" "bucket, key, version ID" 3 $#; then
     return 1
   fi
   if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then
@@ -82,17 +113,32 @@ delete_object_version_bypass_retention() {
   return 0
 }
 
+delete_object_version_rest_bypass_retention() {
+  if ! check_param_count "delete_object_version_rest_bypass_retention" "bucket, key, version ID" 3 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" BYPASS_GOVERNANCE_RETENTION="true" \
+      OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    log 2 "expected '204', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_with_user() {
   record_command "delete-object" "client:$1"
-  if [ $# -ne 5 ]; then
-    log 2 "delete object with user command requires command type, bucket, key, access ID, secret key"
+  if ! check_param_count "delete_object_version_bypass_retention" "command type, bucket, key, access ID, secret key" 5 $#; then
     return 1
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
     delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
   else
@@ -101,48 +147,23 @@ delete_object_with_user() {
   fi
   if [ $exit_code -ne 0 ]; then
     log 2 "error deleting object: $delete_object_error"
-    export delete_object_error
     return 1
   fi
   return 0
 }
 
 delete_object_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'delete_object_rest' requires bucket name, object name"
+  if ! check_param_count "delete_object_rest" "bucket, key" 2 $#; then
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="DELETE
-/$1/$2
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER"/delete_object_error.txt 2>&1)
-  if [[ "$reply" != "204" ]]; then
-    log 2 "delete object command returned error: $(cat "$TEST_FILE_FOLDER"/delete_object_error.txt)"
+  if [ "$result" != "204" ]; then
+    delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
+    log 2 "expected '204', was '$result' ($delete_object_error)"
     return 1
   fi
   return 0
-}
+}

tests/commands/get_bucket_ownership_controls.sh

@@ -37,6 +37,26 @@ get_bucket_ownership_controls() {
   return 0
 }
 
+get_bucket_ownership_controls_rest() {
+  if ! check_param_count "get_bucket_ownership_controls_rest" "bucket" 1 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then
+    log 2 "error getting bucket ownership controls: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "GetBucketOwnershipControls returned response code: $result, reply:  $(cat "$TEST_FILE_FOLDER/ownershipControls.txt")"
+    return 1
+  fi
+  log 5 "controls: $(cat "$TEST_FILE_FOLDER/ownershipControls.txt")"
+  if ! rule=$(xmllint --xpath '//*[local-name()="ObjectOwnership"]/text()' "$TEST_FILE_FOLDER/ownershipControls.txt" 2>&1); then
+    log 2 "error getting ownership rule: $rule"
+    return 1
+  fi
+  echo "$rule"
+}
+
 get_object_ownership_rule() {
   if [[ -n "$SKIP_BUCKET_OWNERSHIP_CONTROLS" ]]; then
     log 5 "Skipping get bucket ownership controls"

tests/commands/get_bucket_policy.sh

@@ -15,18 +15,20 @@
 # under the License.
 
 get_bucket_policy() {
+  log 6 "get_bucket_policy '$1' '$2'"
   record_command "get-bucket-policy" "client:$1"
-  if [[ $# -ne 2 ]]; then
-    log 2 "get bucket policy command requires command type, bucket"
+  if ! check_param_count "get_bucket_policy" "command type, bucket" 2 $#; then
     return 1
   fi
   local get_bucket_policy_result=0
   if [[ $1 == 's3api' ]]; then
-    get_bucket_policy_aws "$2" || get_bucket_policy_result=$?
+    get_bucket_policy_s3api "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
     get_bucket_policy_mc "$2" || get_bucket_policy_result=$?
+  elif [ "$1" == 'rest' ]; then
+    get_bucket_policy_rest "$2" || get_bucket_policy_result=$?
   else
     log 2 "command 'get bucket policy' not implemented for '$1'"
     return 1
@@ -38,10 +40,10 @@ get_bucket_policy() {
   return 0
 }
 
-get_bucket_policy_aws() {
+get_bucket_policy_s3api() {
+  log 6 "get_bucket_policy_s3api '$1'"
   record_command "get-bucket-policy" "client:s3api"
-  if [[ $# -ne 1 ]]; then
-    log 2 "aws 'get bucket policy' command requires bucket"
+  if ! check_param_count "get_bucket_policy_s3api" "bucket" 1 $#; then
     return 1
   fi
   policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
@@ -62,8 +64,7 @@ get_bucket_policy_aws() {
 
 get_bucket_policy_with_user() {
   record_command "get-bucket-policy" "client:s3api"
-  if [[ $# -ne 3 ]]; then
-    log 2 "'get bucket policy with user' command requires bucket, username, password"
+  if ! check_param_count "get_bucket_policy_with_user" "bucket, username, password" 3 $#; then
     return 1
   fi
   if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
@@ -82,8 +83,7 @@ get_bucket_policy_with_user() {
 
 get_bucket_policy_s3cmd() {
   record_command "get-bucket-policy" "client:s3cmd"
-  if [[ $# -ne 1 ]]; then
-    log 2 "s3cmd 'get bucket policy' command requires bucket"
+  if ! check_param_count "get_bucket_policy_s3cmd" "bucket" 1 $#; then
     return 1
   fi
 
@@ -106,8 +106,7 @@ get_bucket_policy_s3cmd() {
 }
 
 get_bucket_policy_rest() {
-  if [[ $# -ne 1 ]]; then
-    log 2 "s3cmd 'get bucket policy' command requires bucket name"
+  if ! check_param_count "get_bucket_policy_rest" "bucket" 1 $#; then
     return 1
   fi
   if ! get_bucket_policy_rest_expect_code "$1" "200"; then
@@ -118,8 +117,7 @@ get_bucket_policy_rest() {
 }
 
 get_bucket_policy_rest_expect_code() {
-  if [[ $# -ne 2 ]]; then
-    log 2 "s3cmd 'get bucket policy' command requires bucket name, expected code"
+  if ! check_param_count "get_bucket_policy_rest_expect_code" "bucket, code" 2 $#; then
     return 1
   fi
   if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then
@@ -169,8 +167,7 @@ search_for_first_policy_line_or_full_policy() {
 
 get_bucket_policy_mc() {
   record_command "get-bucket-policy" "client:mc"
-  if [[ $# -ne 1 ]]; then
-    log 2 "aws 'get bucket policy' command requires bucket"
+  if ! check_param_count "get_bucket_policy_mc" "bucket" 1 $#; then
     return 1
   fi
   bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?

tests/commands/get_object_legal_hold.sh

@@ -57,3 +57,19 @@ get_object_legal_hold_version_id() {
   echo "$legal_hold"
   return 0
 }
+
+get_object_legal_hold_rest_version_id() {
+  if ! check_param_count "get_object_legal_hold_rest_version_id" "bucket, key, version ID" 3 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/legal_hold.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then
+    log 2 "error getting object legal hold: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")"
+    return 1
+  fi
+  legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt")
+  return 0
+}

tests/commands/get_object_lock_configuration.sh

@@ -14,17 +14,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
+source ./tests/drivers/drivers.sh
+
 get_object_lock_configuration() {
   record_command "get-object-lock-configuration" "client:s3api"
-  if [[ $# -ne 1 ]]; then
-    log 2 "'get object lock configuration' command missing bucket name"
+  if ! check_param_count "get_object_lock_configuration" "client, bucket name" 2 $#; then
     return 1
   fi
-  if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
-    log 2 "error obtaining lock config: $lock_config"
-    # shellcheck disable=SC2034
-    get_object_lock_config_err=$lock_config
-    return 1
+  if [ "$1" == 'rest' ]; then
+    if ! get_object_lock_configuration_rest "$2"; then
+      log 2 "error getting REST object lock configuration"
+      get_object_lock_config_err=$(cat "$TEST_FILE_FOLDER/object-lock-config.txt")
+      return 1
+    fi
+  else
+    if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$2" 2>&1); then
+      log 2 "error obtaining lock config: $lock_config"
+      # shellcheck disable=SC2034
+      get_object_lock_config_err=$lock_config
+      return 1
+    fi
   fi
   lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning")
   return 0
@@ -32,8 +41,7 @@ get_object_lock_configuration() {
 
 get_object_lock_configuration_rest() {
   log 6 "get_object_lock_configuration_rest"
-  if [ $# -ne 1 ]; then
-    log 2 "'get_object_lock_configuration_rest' requires bucket name"
+  if ! check_param_count "get_object_lock_configuration_rest" "bucket name" 1 $#; then
     return 1
   fi
   if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object-lock-config.txt" ./tests/rest_scripts/get_object_lock_config.sh); then
@@ -44,5 +52,6 @@ get_object_lock_configuration_rest() {
     log 2 "expected '200', returned '$result': $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
     return 1
   fi
+  lock_config="$(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
   return 0
 }