Merge pull request #1348 from versity/ben/crc-mp-complete

feat: calculate full object crc for multi-part uploads for compatible checksums

.github/SECURITY.md

@@ -0,0 +1,25 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+If you discover a security vulnerability in `versitygw`, we strongly encourage you to report it privately and responsibly.
+
+Please do **not** create public issues or pull requests that contain details about the vulnerability.
+
+Instead, report the issue using GitHub's private **Security Advisories** feature:
+
+- Go to [versitygw's Security Advisories page](https://github.com/versity/versitygw/security/advisories)
+- Click on **"Report a vulnerability"**
+
+We aim to respond within **2 business days** and work with you to quickly resolve the issue.
+
+## Supported Versions
+
+| Version         | Supported |
+| --------------- | --------- |
+| Latest (v1.x.x) | ✅        |
+| Older versions  | ❌        |
+
+## Responsible Disclosure
+
+We appreciate responsible disclosures and are committed to fixing vulnerabilities in a timely manner. Thank you for helping keep `versitygw` secure.

.github/workflows/azurite.yml

@@ -1,5 +1,5 @@
 name: azurite functional tests
-
+permissions: {}
 on: pull_request
 
 jobs:

.github/workflows/betteralign.yml

@@ -1,23 +0,0 @@
-name: betteralign
-on: pull_request
-jobs:
-  build:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: "stable"
-        id: go
-
-      - name: Install betteralign
-        run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest
-
-      - name: Run betteralign
-        run: betteralign -test_files ./...

.github/workflows/docker-bats.yml

@@ -1,5 +1,5 @@
 name: docker bats tests
-
+permissions: {}
 on: pull_request
 
 jobs:
@@ -14,6 +14,7 @@ jobs:
         run: |
           cp tests/.env.docker.default tests/.env.docker
           cp tests/.secrets.default tests/.secrets
+          # see https://github.com/versity/versitygw/issues/1034
           docker build \
             --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
             --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \

.github/workflows/docker.yml

@@ -1,5 +1,4 @@
 name: Publish Docker image
-
 on:
   release:
     types: [published]

.github/workflows/functional.yml

@@ -1,5 +1,5 @@
 name: functional tests
-
+permissions: {}
 on: pull_request
 
 jobs:

.github/workflows/go.yml

@@ -1,9 +1,10 @@
 name: general
+permissions: {}
 on: pull_request
 jobs:
 
   build:
-    name: Build
+    name: Go Basic Checks
     runs-on: ubuntu-latest
     steps:
 
@@ -23,9 +24,6 @@ jobs:
       run: |
         go get -v -t -d ./...
 
-    - name: Build
-      run: make
-
     - name: Test
       run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
 
@@ -35,4 +33,26 @@ jobs:
 
     - name: Run govulncheck
       run: govulncheck ./...
-      shell: bash
+      shell: bash
+
+  verify-build:
+    name: Verify Build Targets
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [darwin, freebsd, linux]
+        arch: [amd64, arm64]
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+
+      - name: Build for ${{ matrix.os }}/${{ matrix.arch }}
+        run: |
+          GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o versitygw-${{ matrix.os }}-${{ matrix.arch }} cmd/versitygw/*.go

.github/workflows/goreleaser.yml

@@ -1,16 +1,12 @@
 name: goreleaser
-
+permissions:
+  contents: write
 on:
   push:
     # run only against tags
     tags:
       - '*'
 
-permissions:
-  contents: write
-  # packages: write
-  # issues: write
-
 jobs:
   goreleaser:
     runs-on: ubuntu-latest
@@ -29,10 +25,10 @@ jobs:
           go-version: stable
 
       - name: Run Releaser
-        uses: goreleaser/goreleaser-action@v5
+        uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser
-          version: latest
+          version: '~> v2'
           args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/host-style-tests.yml

@@ -0,0 +1,13 @@
+name: host style tests
+permissions: {}
+on: pull_request
+
+jobs:
+  build-and-run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: run host-style tests
+        run: make test-host-style

.github/workflows/shellcheck.yml

@@ -1,4 +1,5 @@
 name: shellcheck
+permissions: {}
 on: pull_request
 jobs:
 

.github/workflows/static.yml

@@ -1,4 +1,5 @@
 name: staticcheck
+permissions: {}
 on: pull_request
 jobs: 
 

.github/workflows/system.yml

@@ -1,4 +1,5 @@
 name: system tests
+permissions: {}
 on: pull_request
 jobs:
   build:
@@ -12,91 +13,114 @@ jobs:
             IAM_TYPE: folder
             RUN_SET: "mc-non-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "mc, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "mc-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
-          - set: "REST, posix, non-static, all, folder IAM"
+          - set: "REST, posix, non-static, base|acl|multipart, folder IAM"
             IAM_TYPE: folder
-            RUN_SET: "rest"
+            RUN_SET: "rest-base,rest-acl,rest-multipart"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "REST, posix, non-static, chunked|checksum|versioning|bucket, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3-non-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, policy, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, user, non-static, s3 IAM"
             IAM_TYPE: s3
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, bucket, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-bucket"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, multipart, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-multipart"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, object, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-object"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, policy, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, user, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
-          - set: "s3api, s3, multipart|object, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
-          - set: "s3api, s3, policy|user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy,s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
+          # TODO fix/debug s3 gateway
+          #- set: "s3api, s3, multipart|object, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
+          #- set: "s3api, s3, policy|user, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-policy,s3api-user"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
           - set: "s3cmd, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3cmd, posix, non-user, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-non-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3cmd, posix, user, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
@@ -105,7 +129,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 'stable'
+          go-version: "stable"
         id: go
 
       - name: Get Dependencies
@@ -121,6 +145,7 @@ jobs:
 
       - name: Install s3cmd
         run: |
+          sudo apt-get update
           sudo apt-get install s3cmd
 
       - name: Install mc
@@ -128,9 +153,18 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Install xmllint (for rest)
+      - name: Install xml libraries (for rest)
         run: |
-          sudo apt-get install libxml2-utils
+          sudo apt-get update
+          sudo apt-get install libxml2-utils xmlstarlet
+
+      # see https://github.com/versity/versitygw/issues/1034
+      - name: Install AWS cli
+        run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" 
+          unzip -o awscliv2.zip 
+          ./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
 
       - name: Build and run
         env:
@@ -141,6 +175,7 @@ jobs:
           RUN_VERSITYGW: true
           BACKEND: ${{ matrix.BACKEND }}
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
+          DELETE_BUCKETS_AFTER_TEST: ${{ matrix.DELETE_BUCKETS_AFTER_TEST }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
           LOCAL_FOLDER: /tmp/gw
@@ -163,6 +198,9 @@ jobs:
           VERSIONING_DIR: ${{ github.workspace }}/versioning
           COMMAND_LOG: command.log
           TIME_LOG: time.log
+          PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
+          AUTOGENERATE_USERS: true
+          USER_AUTOGENERATION_PREFIX: github-actions-test-
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -170,6 +208,7 @@ jobs:
           export AWS_REGION=us-east-1
           export AWS_ACCESS_KEY_ID_TWO=user
           export AWS_SECRET_ACCESS_KEY_TWO=pass
+          export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity

.goreleaser.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 before:
   hooks:
     - go mod tidy
@@ -23,7 +25,7 @@ builds:
       - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
 
 archives:
-  - format: tar.gz
+  - formats: [ 'tar.gz' ]
     # this name template makes the OS and Arch compatible with the results of uname.
     name_template: >-
       {{ .ProjectName }}_v{{ .Version }}_
@@ -43,7 +45,7 @@ archives:
     # use zip for windows archives
     format_overrides:
     - goos: windows
-      format: zip
+      formats: [ 'zip' ]
 
     # Additional files/globs you want to add to the archive.
     #
@@ -58,7 +60,7 @@ checksum:
   name_template: 'checksums.txt'
 
 snapshot:
-  name_template: "{{ incpatch .Version }}-next"
+  version_template: "{{ incpatch .Version }}-{{.ShortCommit}}"
 
 changelog:
   sort: asc
@@ -86,7 +88,7 @@ nfpms:
 
     license: Apache 2.0
 
-    builds:
+    ids:
       - versitygw
 
     formats:

Makefile

@@ -72,6 +72,11 @@ dist:
 	rm -f VERSION
 	gzip -f $(TARFILE)
 
+.PHONY: snapshot
+snapshot:
+# brew install goreleaser/tap/goreleaser
+	goreleaser release --snapshot --skip publish --clean
+
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
@@ -91,3 +96,9 @@ up-azurite:
 .PHONY: up-app
 up-app:
 	$(DOCKERCOMPOSE) up
+
+# Run the host-style tests in docker containers
+.PHONY: test-host-style
+test-host-style:
+	docker compose -f tests/host-style-tests/docker-compose.yml up --build --abort-on-container-exit --exit-code-from test
+

README.md

@@ -40,7 +40,7 @@ Versity Gateway, a simple to use tool for seamless inline translation between AW
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
 
-The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versity’s open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.  
 
 The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
 

auth/access-control.go

@@ -0,0 +1,201 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+)
+
+func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	// Verify destination bucket access
+	if err := VerifyAccess(ctx, be, opts); err != nil {
+		return err
+	}
+	// Verify source bucket access
+	srcBucket, srcObject, found := strings.Cut(copySource, "/")
+	if !found {
+		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	// Get source bucket ACL
+	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+	if err != nil {
+		return err
+	}
+
+	var srcBucketAcl ACL
+	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+		return err
+	}
+
+	if err := VerifyAccess(ctx, be, AccessOptions{
+		Acl:           srcBucketAcl,
+		AclPermission: PermissionRead,
+		IsRoot:        opts.IsRoot,
+		Acc:           opts.Acc,
+		Bucket:        srcBucket,
+		Object:        srcObject,
+		Action:        GetObjectAction,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type AccessOptions struct {
+	Acl            ACL
+	AclPermission  Permission
+	IsRoot         bool
+	Acc            Account
+	Bucket         string
+	Object         string
+	Action         Action
+	Readonly       bool
+	IsBucketPublic bool
+}
+
+func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	// Skip the access check for public buckets
+	if opts.IsBucketPublic {
+		return nil
+	}
+	if opts.Readonly {
+		if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+	}
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
+	}
+
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Detects if the action is policy related
+// e.g.
+// 'GetBucketPolicy', 'PutBucketPolicy'
+func isPolicyAction(action Action) bool {
+	return action == GetBucketPolicyAction || action == PutBucketPolicyAction
+}
+
+// VerifyPublicAccess checks if the bucket is publically accessible by ACL or Policy
+func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, permission Permission, bucket, object string) error {
+	// ACL disabled
+	policy, err := be.GetBucketPolicy(ctx, bucket)
+	if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+		return err
+	}
+	if err == nil {
+		err = VerifyPublicBucketPolicy(policy, bucket, object, action)
+		if err == nil {
+			// if ACLs are disabled, and the bucket grants public access,
+			// policy actions should return 'MethodNotAllowed'
+			if isPolicyAction(action) {
+				return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+			}
+
+			return nil
+		}
+	}
+
+	// if the action is not in the ACL whitelist the access is denied
+	_, ok := publicACLAllowedActions[action]
+	if !ok {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	err = VerifyPublicBucketACL(ctx, be, bucket, action, permission)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+func MayCreateBucket(acct Account, isRoot bool) error {
+	if isRoot {
+		return nil
+	}
+
+	if acct.Role == RoleUser {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
+	// Owner check
+	if acct.Access == acl.Owner {
+		return nil
+	}
+
+	// Root user has access over almost everything
+	if isRoot {
+		return nil
+	}
+
+	// Admin user case
+	if acct.Role == RoleAdmin {
+		return nil
+	}
+
+	// Return access denied in all other cases
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}
+
+type PublicACLAllowedActions map[Action]struct{}
+
+var publicACLAllowedActions PublicACLAllowedActions = PublicACLAllowedActions{
+	ListBucketAction:                 struct{}{},
+	PutObjectAction:                  struct{}{},
+	ListBucketMultipartUploadsAction: struct{}{},
+	DeleteObjectAction:               struct{}{},
+	ListBucketVersionsAction:         struct{}{},
+	GetObjectAction:                  struct{}{},
+	GetObjectAttributesAction:        struct{}{},
+	GetObjectAclAction:               struct{}{},
+}

auth/acl.go

@@ -17,6 +17,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"strings"
@@ -32,47 +33,163 @@ type ACL struct {
 	Grantees []Grantee
 }
 
+// IsPublic specifies if the acl grants public read access
+func (acl *ACL) IsPublic(permission Permission) bool {
+	for _, grt := range acl.Grantees {
+		if grt.Permission == permission && grt.Type == types.TypeGroup && grt.Access == "all-users" {
+			return true
+		}
+	}
+
+	return false
+}
+
 type Grantee struct {
-	Permission types.Permission
+	Permission Permission
 	Access     string
 	Type       types.Type
 }
 
 type GetBucketAclOutput struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
 	Owner             *types.Owner
 	AccessControlList AccessControlList
 }
 
 type PutBucketAclInput struct {
 	Bucket              *string
+	ACL                 types.BucketCannedACL
 	AccessControlPolicy *AccessControlPolicy
 	GrantFullControl    *string
 	GrantRead           *string
 	GrantReadACP        *string
 	GrantWrite          *string
 	GrantWriteACP       *string
-	ACL                 types.BucketCannedACL
 }
 
 type AccessControlPolicy struct {
-	Owner             *types.Owner
 	AccessControlList AccessControlList `xml:"AccessControlList"`
+	Owner             *types.Owner
+}
+
+func (acp *AccessControlPolicy) Validate() error {
+	if !acp.AccessControlList.isValid() {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner can't be nil
+	if acp.Owner == nil {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner ID can't be empty
+	if acp.Owner.ID == nil || *acp.Owner.ID == "" {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	return nil
 }
 
 type AccessControlList struct {
 	Grants []Grant `xml:"Grant"`
 }
 
+// Validates the AccessControlList
+func (acl *AccessControlList) isValid() bool {
+	for _, el := range acl.Grants {
+		if !el.isValid() {
+			return false
+		}
+	}
+
+	return true
+}
+
+type Permission string
+
+const (
+	PermissionFullControl Permission = "FULL_CONTROL"
+	PermissionWrite       Permission = "WRITE"
+	PermissionWriteAcp    Permission = "WRITE_ACP"
+	PermissionRead        Permission = "READ"
+	PermissionReadAcp     Permission = "READ_ACP"
+)
+
+// Check if the permission is valid
+func (p Permission) isValid() bool {
+	return p == PermissionFullControl ||
+		p == PermissionRead ||
+		p == PermissionReadAcp ||
+		p == PermissionWrite ||
+		p == PermissionWriteAcp
+}
+
 type Grant struct {
-	Grantee    *Grt
-	Permission types.Permission
+	Grantee    *Grt       `xml:"Grantee"`
+	Permission Permission `xml:"Permission"`
+}
+
+// Checks if Grant is valid
+func (g *Grant) isValid() bool {
+	return g.Permission.isValid() && g.Grantee.isValid()
 }
 
 type Grt struct {
-	XMLNS  string     `xml:"xmlns:xsi,attr"`
-	XMLXSI types.Type `xml:"xsi:type,attr"`
-	Type   types.Type `xml:"Type"`
-	ID     string     `xml:"ID"`
+	XMLNS string     `xml:"xmlns:xsi,attr"`
+	Type  types.Type `xml:"xsi:type,attr"`
+	ID    string     `xml:"ID"`
+}
+
+// Custom Unmarshalling for Grt to parse xsi:type properly
+func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Iterate through the XML tokens to process the attributes
+	for _, attr := range start.Attr {
+		// Check if the attribute is xsi:type and belongs to the xsi namespace
+		if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
+			g.Type = types.Type(attr.Value)
+		}
+		// Handle xmlns:xsi
+		if attr.Name.Local == "xmlns:xsi" {
+			g.XMLNS = attr.Value
+		}
+	}
+
+	// Decode the inner XML elements like ID
+	for {
+		t, err := d.Token()
+		if err != nil {
+			return err
+		}
+
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "ID" {
+				if err := d.DecodeElement(&g.ID, &se); err != nil {
+					return err
+				}
+			}
+		case xml.EndElement:
+			if se.Name.Local == start.Name.Local {
+				return nil
+			}
+		}
+	}
+}
+
+// Validates Grt
+func (g *Grt) isValid() bool {
+	// Validate the Type
+	// Only these 2 types are supported in the gateway
+	if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
+		return false
+	}
+
+	// The ID prop shouldn't be empty
+	if g.ID == "" {
+		return false
+	}
+
+	return true
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -87,22 +204,32 @@ func ParseACL(data []byte) (ACL, error) {
 	return acl, nil
 }
 
-func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
+func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
+	grants := []Grant{}
+
+	if len(data) == 0 {
+		return GetBucketAclOutput{
+			Owner: &types.Owner{
+				ID: &owner,
+			},
+			AccessControlList: AccessControlList{
+				Grants: grants,
+			},
+		}, nil
+	}
+
 	var acl ACL
 	if err := json.Unmarshal(data, &acl); err != nil {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
-	grants := []Grant{}
-
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
 		grants = append(grants, Grant{
 			Grantee: &Grt{
-				XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
-				XMLXSI: elem.Type,
-				ID:     acs,
-				Type:   elem.Type,
+				XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
+				ID:    acs,
+				Type:  elem.Type,
 			},
 			Permission: elem.Permission,
 		})
@@ -125,7 +252,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 
 	defaultGrantees := []Grantee{
 		{
-			Permission: types.PermissionFullControl,
+			Permission: PermissionFullControl,
 			Access:     acl.Owner,
 			Type:       types.TypeCanonicalUser,
 		},
@@ -136,19 +263,19 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 		switch input.ACL {
 		case types.BucketCannedACLPublicRead:
 			defaultGrantees = append(defaultGrantees, Grantee{
-				Permission: types.PermissionRead,
+				Permission: PermissionRead,
 				Access:     "all-users",
 				Type:       types.TypeGroup,
 			})
 		case types.BucketCannedACLPublicReadWrite:
 			defaultGrantees = append(defaultGrantees, []Grantee{
 				{
-					Permission: types.PermissionRead,
+					Permission: PermissionRead,
 					Access:     "all-users",
 					Type:       types.TypeGroup,
 				},
 				{
-					Permission: types.PermissionWrite,
+					Permission: PermissionWrite,
 					Access:     "all-users",
 					Type:       types.TypeGroup,
 				},
@@ -165,7 +292,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range fullControlList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionFullControl,
+						Permission: PermissionFullControl,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -175,7 +302,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range readList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionRead,
+						Permission: PermissionRead,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -185,7 +312,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range readACPList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionReadAcp,
+						Permission: PermissionReadAcp,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -195,7 +322,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range writeList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionWrite,
+						Permission: PermissionWrite,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -205,7 +332,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range writeACPList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionWriteAcp,
+						Permission: PermissionWriteAcp,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -262,8 +389,8 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
 				result = append(result, acc)
 				continue
 			}
-			if err == ErrNotSupported {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
+				return nil, err
 			}
 			return nil, fmt.Errorf("check user account: %w", err)
 		}
@@ -286,7 +413,7 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func verifyACL(acl ACL, access string, permission types.Permission) error {
+func verifyACL(acl ACL, access string, permission Permission) error {
 	grantee := Grantee{
 		Access:     access,
 		Permission: permission,
@@ -294,7 +421,7 @@ func verifyACL(acl ACL, access string, permission types.Permission) error {
 	}
 	granteeFullCtrl := Grantee{
 		Access:     access,
-		Permission: types.PermissionFullControl,
+		Permission: PermissionFullControl,
 		Type:       types.TypeCanonicalUser,
 	}
 	granteeAllUsers := Grantee{
@@ -319,117 +446,22 @@ func verifyACL(acl ACL, access string, permission types.Permission) error {
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
-func MayCreateBucket(acct Account, isRoot bool) error {
-	if isRoot {
-		return nil
-	}
-
-	if acct.Role == RoleUser {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	return nil
-}
-
-func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
-	// Owner check
-	if acct.Access == acl.Owner {
-		return nil
-	}
-
-	// Root user has access over almost everything
-	if isRoot {
-		return nil
-	}
-
-	// Admin user case
-	if acct.Role == RoleAdmin {
-		return nil
-	}
-
-	// Return access denied in all other cases
-	return s3err.GetAPIError(s3err.ErrAccessDenied)
-}
-
-type AccessOptions struct {
-	AclPermission types.Permission
-	Bucket        string
-	Object        string
-	Action        Action
-	Acl           ACL
-	Acc           Account
-	IsRoot        bool
-	Readonly      bool
-}
-
-func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
-	if opts.Readonly {
-		if opts.AclPermission == types.PermissionWrite || opts.AclPermission == types.PermissionWriteAcp {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-	}
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
-		return nil
-	}
-
-	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
-	if policyErr != nil {
-		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-			return policyErr
-		}
-	} else {
-		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
-	}
-
-	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
-		return nil
-	}
-
-	// Verify destination bucket access
-	if err := VerifyAccess(ctx, be, opts); err != nil {
-		return err
-	}
-	// Verify source bucket access
-	srcBucket, srcObject, found := strings.Cut(copySource, "/")
-	if !found {
-		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
-	}
-
-	// Get source bucket ACL
-	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+// Verifies if the bucket acl grants public access
+func VerifyPublicBucketACL(ctx context.Context, be backend.Backend, bucket string, action Action, permission Permission) error {
+	aclBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{
+		Bucket: &bucket,
+	})
 	if err != nil {
 		return err
 	}
 
-	var srcBucketAcl ACL
-	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+	acl, err := ParseACL(aclBytes)
+	if err != nil {
 		return err
 	}
 
-	if err := VerifyAccess(ctx, be, AccessOptions{
-		Acl:           srcBucketAcl,
-		AclPermission: types.PermissionRead,
-		IsRoot:        opts.IsRoot,
-		Acc:           opts.Acc,
-		Bucket:        srcBucket,
-		Object:        srcObject,
-		Action:        GetObjectAction,
-	}); err != nil {
-		return err
+	if !acl.IsPublic(permission) {
+		return ErrAccessDenied
 	}
 
 	return nil

auth/bucket_policy.go

@@ -22,20 +22,48 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
-var (
-	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidResource = errors.New("Policy has invalid resource")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidPrincipal = errors.New("Invalid principal in policy")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidAction = errors.New("Policy has invalid action")
+var ErrAccessDenied = errors.New("access denied")
+
+type policyErr string
+
+func (p policyErr) Error() string {
+	return string(p)
+}
+
+const (
+	policyErrResourceMismatch     = policyErr("Action does not apply to any resource(s) in statement")
+	policyErrInvalidResource      = policyErr("Policy has invalid resource")
+	policyErrInvalidPrincipal     = policyErr("Invalid principal in policy")
+	policyErrInvalidAction        = policyErr("Policy has invalid action")
+	policyErrInvalidPolicy        = policyErr("This policy contains invalid Json")
+	policyErrInvalidFirstChar     = policyErr("Policies must be valid JSON and the first byte must be '{'")
+	policyErrEmptyStatement       = policyErr("Could not parse the policy: Statement is empty!")
+	policyErrMissingStatmentField = policyErr("Missing required field Statement")
 )
 
 type BucketPolicy struct {
 	Statement []BucketPolicyItem `json:"Statement"`
 }
 
+func (bp *BucketPolicy) UnmarshalJSON(data []byte) error {
+	var tmp struct {
+		Statement *[]BucketPolicyItem `json:"Statement"`
+	}
+
+	if err := json.Unmarshal(data, &tmp); err != nil {
+		return err
+	}
+
+	// If Statement is nil (not present in JSON), return an error
+	if tmp.Statement == nil {
+		return policyErrMissingStatmentField
+	}
+
+	// Assign the parsed value to the actual struct
+	bp.Statement = *tmp.Statement
+	return nil
+}
+
 func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 	for _, statement := range bp.Statement {
 		err := statement.Validate(bucket, iam)
@@ -48,25 +76,44 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 }
 
 func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	var isAllowed bool
 	for _, statement := range bp.Statement {
 		if statement.findMatch(principal, action, resource) {
 			switch statement.Effect {
 			case BucketPolicyAccessTypeAllow:
-				return true
+				isAllowed = true
 			case BucketPolicyAccessTypeDeny:
 				return false
 			}
 		}
 	}
 
-	return false
+	return isAllowed
+}
+
+// isPublic checks if the bucket policy statements contain
+// an entity granting public access
+func (bp *BucketPolicy) isPublic(resource string, action Action) bool {
+	var isAllowed bool
+	for _, statement := range bp.Statement {
+		if statement.isPublic(resource, action) {
+			switch statement.Effect {
+			case BucketPolicyAccessTypeAllow:
+				isAllowed = true
+			case BucketPolicyAccessTypeDeny:
+				return false
+			}
+		}
+	}
+
+	return isAllowed
 }
 
 type BucketPolicyItem struct {
+	Effect     BucketPolicyAccessType `json:"Effect"`
 	Principals Principals             `json:"Principal"`
 	Actions    Actions                `json:"Action"`
 	Resources  Resources              `json:"Resource"`
-	Effect     BucketPolicyAccessType `json:"Effect"`
 }
 
 func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
@@ -89,10 +136,10 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 			break
 		}
 		if *isObjectAction && !containsObjectAction {
-			return errResourceMismatch
+			return policyErrResourceMismatch
 		}
 		if !*isObjectAction && !containsBucketAction {
-			return errResourceMismatch
+			return policyErrResourceMismatch
 		}
 	}
 
@@ -107,6 +154,11 @@ func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource
 	return false
 }
 
+// isPublic checks if the bucket policy statemant grants public access
+func (bpi *BucketPolicyItem) isPublic(resource string, action Action) bool {
+	return bpi.Principals.IsPublic() && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource)
+}
+
 func getMalformedPolicyError(err error) error {
 	return s3err.APIError{
 		Code:           "MalformedPolicy",
@@ -116,14 +168,20 @@ func getMalformedPolicyError(err error) error {
 }
 
 func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
+	if len(policyBin) == 0 || policyBin[0] != '{' {
+		return getMalformedPolicyError(policyErrInvalidFirstChar)
+	}
 	var policy BucketPolicy
 	if err := json.Unmarshal(policyBin, &policy); err != nil {
-		return getMalformedPolicyError(err)
+		var pe policyErr
+		if errors.As(err, &pe) {
+			return getMalformedPolicyError(err)
+		}
+		return getMalformedPolicyError(policyErrInvalidPolicy)
 	}
 
 	if len(policy.Statement) == 0 {
-		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
+		return getMalformedPolicyError(policyErrEmptyStatement)
 	}
 
 	if err := policy.Validate(bucket, iam); err != nil {
@@ -150,3 +208,22 @@ func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Act
 
 	return nil
 }
+
+// Checks if the bucket policy grants public access
+func VerifyPublicBucketPolicy(policy []byte, bucket, object string, action Action) error {
+	var bucketPolicy BucketPolicy
+	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
+		return err
+	}
+
+	resource := bucket
+	if object != "" {
+		resource += "/" + object
+	}
+
+	if !bucketPolicy.isPublic(resource, action) {
+		return ErrAccessDenied
+	}
+
+	return nil
+}

auth/bucket_policy_actions.go

@@ -58,6 +58,8 @@ const (
 	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
 	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
 	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
+	PutBucketCorsAction                    Action = "s3:PutBucketCORS"
+	GetBucketCorsAction                    Action = "s3:GetBucketCORS"
 	AllActions                             Action = "s3:*"
 )
 
@@ -89,6 +91,7 @@ var supportedActionList = map[Action]struct{}{
 	DeleteObjectTaggingAction:              {},
 	ListBucketVersionsAction:               {},
 	ListBucketAction:                       {},
+	GetBucketObjectLockConfigurationAction: {},
 	PutBucketObjectLockConfigurationAction: {},
 	GetObjectLegalHoldAction:               {},
 	PutObjectLegalHoldAction:               {},
@@ -97,6 +100,8 @@ var supportedActionList = map[Action]struct{}{
 	BypassGovernanceRetentionAction:        {},
 	PutBucketOwnershipControlsAction:       {},
 	GetBucketOwnershipControlsAction:       {},
+	PutBucketCorsAction:                    {},
+	GetBucketCorsAction:                    {},
 	AllActions:                             {},
 }
 
@@ -125,7 +130,7 @@ var supportedObjectActionList = map[Action]struct{}{
 // Validates Action: it should either wildcard match with supported actions list or be in it
 func (a Action) IsValid() error {
 	if !strings.HasPrefix(string(a), "s3:") {
-		return errInvalidAction
+		return policyErrInvalidAction
 	}
 
 	if a == AllActions {
@@ -140,12 +145,12 @@ func (a Action) IsValid() error {
 			}
 		}
 
-		return errInvalidAction
+		return policyErrInvalidAction
 	}
 
 	_, found := supportedActionList[a]
 	if !found {
-		return errInvalidAction
+		return policyErrInvalidAction
 	}
 	return nil
 }
@@ -191,7 +196,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return errInvalidAction
+			return policyErrInvalidAction
 		}
 		*a = make(Actions)
 		for _, s := range ss {
@@ -204,7 +209,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return errInvalidAction
+				return policyErrInvalidAction
 			}
 			*a = make(Actions)
 			err = a.Add(s)

auth/bucket_policy_principals.go

@@ -36,7 +36,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return errInvalidPrincipal
+			return policyErrInvalidPrincipal
 		}
 		*p = make(Principals)
 		for _, s := range ss {
@@ -45,7 +45,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &s); err == nil {
 		if s == "" {
-			return errInvalidPrincipal
+			return policyErrInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(s)
@@ -53,7 +53,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &k); err == nil {
 		if k.AWS == "" {
-			return errInvalidPrincipal
+			return policyErrInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(k.AWS)
@@ -65,7 +65,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		}
 		if err = json.Unmarshal(data, &sk); err == nil {
 			if len(sk.AWS) == 0 {
-				return errInvalidPrincipal
+				return policyErrInvalidPrincipal
 			}
 			*p = make(Principals)
 			for _, s := range sk.AWS {
@@ -97,7 +97,7 @@ func (p Principals) Validate(iam IAMService) error {
 		if len(p) == 1 {
 			return nil
 		}
-		return errInvalidPrincipal
+		return policyErrInvalidPrincipal
 	}
 
 	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -105,7 +105,7 @@ func (p Principals) Validate(iam IAMService) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return errInvalidPrincipal
+		return policyErrInvalidPrincipal
 	}
 
 	return nil
@@ -121,3 +121,10 @@ func (p Principals) Contains(userAccess string) bool {
 	_, found := p[userAccess]
 	return found
 }
+
+// Bucket policy grants public access, if it contains
+// a wildcard match to all the users
+func (p Principals) IsPublic() bool {
+	_, ok := p["*"]
+	return ok
+}

auth/bucket_policy_resources.go

@@ -29,7 +29,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return errInvalidResource
+			return policyErrInvalidResource
 		}
 		*r = make(Resources)
 		for _, s := range ss {
@@ -42,7 +42,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return errInvalidResource
+				return policyErrInvalidResource
 			}
 			*r = make(Resources)
 			err = r.Add(s)
@@ -59,7 +59,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 func (r Resources) Add(rc string) error {
 	ok, pattern := isValidResource(rc)
 	if !ok {
-		return errInvalidResource
+		return policyErrInvalidResource
 	}
 
 	r[pattern] = struct{}{}
@@ -93,7 +93,7 @@ func (r Resources) ContainsBucketPattern() bool {
 func (r Resources) Validate(bucket string) error {
 	for resource := range r {
 		if !strings.HasPrefix(resource, bucket) {
-			return errInvalidResource
+			return policyErrInvalidResource
 		}
 	}
 
@@ -102,21 +102,45 @@ func (r Resources) Validate(bucket string) error {
 
 func (r Resources) FindMatch(resource string) bool {
 	for res := range r {
-		if strings.HasSuffix(res, "*") {
-			pattern := strings.TrimSuffix(res, "*")
-			if strings.HasPrefix(resource, pattern) {
-				return true
-			}
-		} else {
-			if res == resource {
-				return true
-			}
+		if r.Match(res, resource) {
+			return true
 		}
 	}
 
 	return false
 }
 
+// Match checks if the input string matches the given pattern with wildcards (`*`, `?`).
+// - `?` matches exactly one occurrence of any character.
+// - `*` matches arbitrary many (including zero) occurrences of any character.
+func (r Resources) Match(pattern, input string) bool {
+	pIdx, sIdx := 0, 0
+	starIdx, matchIdx := -1, 0
+
+	for sIdx < len(input) {
+		if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
+			sIdx++
+			pIdx++
+		} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
+			starIdx = pIdx
+			matchIdx = sIdx
+			pIdx++
+		} else if starIdx != -1 {
+			pIdx = starIdx + 1
+			matchIdx++
+			sIdx = matchIdx
+		} else {
+			return false
+		}
+	}
+
+	for pIdx < len(pattern) && pattern[pIdx] == '*' {
+		pIdx++
+	}
+
+	return pIdx == len(pattern)
+}
+
 // Checks the resource to have arn prefix and not starting with /
 func isValidResource(rc string) (isValid bool, pattern string) {
 	if !strings.HasPrefix(rc, ResourceArnPrefix) {

auth/bucket_policy_resources_test.go

@@ -0,0 +1,182 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestUnmarshalJSON(t *testing.T) {
+	var r Resources
+
+	cases := []struct {
+		input    string
+		expected int
+		wantErr  bool
+	}{
+		{`"arn:aws:s3:::my-bucket/*"`, 1, false},
+		{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
+		{`""`, 0, true},
+		{`[]`, 0, true},
+		{`["invalid-bucket"]`, 0, true},
+	}
+
+	for _, tc := range cases {
+		r = Resources{}
+		err := json.Unmarshal([]byte(tc.input), &r)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+		if len(r) != tc.expected {
+			t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
+		}
+	}
+}
+
+func TestAdd(t *testing.T) {
+	r := Resources{}
+
+	cases := []struct {
+		input   string
+		wantErr bool
+	}{
+		{"arn:aws:s3:::valid-bucket/*", false},
+		{"arn:aws:s3:::valid-bucket/object", false},
+		{"invalid-bucket/*", true},
+		{"/invalid-start", true},
+	}
+
+	for _, tc := range cases {
+		err := r.Add(tc.input)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+	}
+}
+
+func TestContainsObjectPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, true},
+		{[]string{"arn:aws:s3:::my-bucket"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsObjectPattern() != tc.expected {
+			t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestContainsBucketPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsBucketPattern() != tc.expected {
+			t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestValidate(t *testing.T) {
+	cases := []struct {
+		resources []string
+		bucket    string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
+		{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
+		{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if (r.Validate(tc.bucket) == nil) != tc.expected {
+			t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
+		}
+	}
+}
+
+func TestFindMatch(t *testing.T) {
+	cases := []struct {
+		resources []string
+		input     string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
+		{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.FindMatch(tc.input) != tc.expected {
+			t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
+		}
+	}
+}
+
+func TestMatch(t *testing.T) {
+	r := Resources{}
+	cases := []struct {
+		pattern  string
+		input    string
+		expected bool
+	}{
+		{"my-bucket/*", "my-bucket/object", true},
+		{"my-bucket/?bject", "my-bucket/object", true},
+		{"my-bucket/*", "other-bucket/object", false},
+		{"*", "any-bucket/object", true},
+		{"my-bucket/*", "my-bucket/subdir/object", true},
+		{"my-bucket/*", "other-bucket", false},
+		{"my-bucket/*/*", "my-bucket/hello", false},
+		{"my-bucket/*/*", "my-bucket/hello/world", true},
+		{"foo/???/bar", "foo/qux/bar", true},
+		{"foo/???/bar", "foo/quxx/bar", false},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
+	}
+	for _, tc := range cases {
+		if r.Match(tc.pattern, tc.input) != tc.expected {
+			t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
+		}
+	}
+}

auth/iam.go

@@ -18,6 +18,8 @@ import (
 	"errors"
 	"fmt"
 	"time"
+
+	"github.com/versity/versitygw/s3err"
 )
 
 type Role string
@@ -57,10 +59,19 @@ type ListUserAccountsResult struct {
 // Mutable props, which could be changed when updating an IAM account
 type MutableProps struct {
 	Secret  *string `json:"secret"`
+	Role    Role    `json:"role"`
 	UserID  *int    `json:"userID"`
 	GroupID *int    `json:"groupID"`
 }
 
+func (m MutableProps) Validate() error {
+	if m.Role != "" && !m.Role.IsValid() {
+		return s3err.GetAPIError(s3err.ErrAdminInvalidUserRole)
+	}
+
+	return nil
+}
+
 func updateAcc(acc *Account, props MutableProps) {
 	if props.Secret != nil {
 		acc.Secret = *props.Secret
@@ -71,6 +82,9 @@ func updateAcc(acc *Account, props MutableProps) {
 	if props.UserID != nil {
 		acc.UserID = *props.UserID
 	}
+	if props.Role != "" {
+		acc.Role = props.Role
+	}
 }
 
 // IAMService is the interface for all IAM service implementations
@@ -93,6 +107,7 @@ var (
 )
 
 type Opts struct {
+	RootAccount            Account
 	Dir                    string
 	LDAPServerURL          string
 	LDAPBindDN             string
@@ -118,12 +133,17 @@ type Opts struct {
 	S3Region               string
 	S3Bucket               string
 	S3Endpoint             string
-	RootAccount            Account
-	CacheTTL               int
-	CachePrune             int
 	S3DisableSSlVerfiy     bool
 	S3Debug                bool
 	CacheDisable           bool
+	CacheTTL               int
+	CachePrune             int
+	IpaHost                string
+	IpaVaultName           string
+	IpaUser                string
+	IpaPassword            string
+	IpaInsecure            bool
+	IpaDebug               bool
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -149,10 +169,13 @@ func New(o *Opts) (IAMService, error) {
 			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
 			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
 		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
+	case o.IpaHost != "":
+		svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure, o.IpaDebug)
+		fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")
-		return IAMServiceSingle{}, nil
+		return NewIAMServiceSingle(o.RootAccount), nil
 	}
 
 	if err != nil {

auth/iam_cache.go

@@ -36,14 +36,14 @@ type IAMCache struct {
 var _ IAMService = &IAMCache{}
 
 type item struct {
-	exp   time.Time
 	value Account
+	exp   time.Time
 }
 
 type icache struct {
-	items  map[string]item
-	expire time.Duration
 	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
 }
 
 func (i *icache) set(k string, v Account) {

auth/iam_internal.go

@@ -33,8 +33,6 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	dir     string
-	rootAcc Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -42,6 +40,8 @@ type IAMServiceInternal struct {
 	// IAM service. All account updates should be sent to a single
 	// gateway instance if possible.
 	sync.RWMutex
+	dir     string
+	rootAcc Account
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -290,93 +290,49 @@ func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
 
 func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
 	// We are going to be racing with other running gateways without any
-	// coordination. So the strategy here is to read the current file data.
-	// If the file doesn't exist, then we assume someone else is currently
-	// updating the file. So we just need to keep retrying. We also need
-	// to make sure the data is consistent within a single update. So racing
-	// writes to a file would possibly leave this in some invalid state.
-	// We can get atomic updates with rename. If we read the data, update
-	// the data, write to a temp file, then rename the tempfile back to the
-	// data file. This should always result in a complete data image.
+	// coordination. So the strategy here is to read the current file data,
+	// update the data, write back out to a temp file, then rename the
+	// temp file to the original file. This rename will replace the
+	// original file with the new file. This is atomic and should always
+	// allow for a consistent view of the data. There is a small
+	// window where the file could be read and then updated by
+	// another process. In this case any updates the other process did
+	// will be lost. This is a limitation of the internal IAM service.
+	// This should be rare, and even when it does happen should result
+	// in a valid IAM file, just without the other process's updates.
 
-	// There is at least one unsolved failure mode here.
-	// If a gateway removes the data file and then crashes, all other
-	// gateways will retry forever thinking that the original will eventually
-	// write the file.
+	iamFname := filepath.Join(s.dir, iamFile)
+	backupFname := filepath.Join(s.dir, iamBackupFile)
 
-	retries := 0
-	fname := filepath.Join(s.dir, iamFile)
+	b, err := os.ReadFile(iamFname)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("read iam file: %w", err)
+	}
 
-	for {
-		b, err := os.ReadFile(fname)
-		if errors.Is(err, fs.ErrNotExist) {
-			// racing with someone else updating
-			// keep retrying after backoff
-			retries++
-			if retries < maxretry {
-				time.Sleep(backoff)
-				continue
-			}
+	// save copy of data
+	datacopy := make([]byte, len(b))
+	copy(datacopy, b)
 
-			// we have been unsuccessful trying to read the iam file
-			// so this must be the case where something happened and
-			// the file did not get updated successfully, and probably
-			// isn't going to be. The recovery procedure would be to
-			// copy the backup file into place of the original.
-			return fmt.Errorf("no iam file, needs backup recovery")
-		}
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("read iam file: %w", err)
-		}
+	// make a backup copy in case something happens
+	err = s.writeUsingTempFile(b, backupFname)
+	if err != nil {
+		return fmt.Errorf("write backup iam file: %w", err)
+	}
 
-		// reset retries on successful read
-		retries = 0
+	b, err = update(b)
+	if err != nil {
+		return fmt.Errorf("update iam data: %w", err)
+	}
 
-		err = os.Remove(fname)
-		if errors.Is(err, fs.ErrNotExist) {
-			// racing with someone else updating
-			// keep retrying after backoff
-			time.Sleep(backoff)
-			continue
-		}
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("remove old iam file: %w", err)
-		}
-
-		// save copy of data
-		datacopy := make([]byte, len(b))
-		copy(datacopy, b)
-
-		// make a backup copy in case we crash before update
-		// this is after remove, so there is a small window something
-		// can go wrong, but the remove should barrier other gateways
-		// from trying to write backup at the same time. Only one
-		// gateway will successfully remove the file.
-		os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
-
-		b, err = update(b)
-		if err != nil {
-			// update failed, try to write old data back out
-			os.WriteFile(fname, datacopy, iamMode)
-			return fmt.Errorf("update iam data: %w", err)
-		}
-
-		err = s.writeTempFile(b)
-		if err != nil {
-			// update failed, try to write old data back out
-			os.WriteFile(fname, datacopy, iamMode)
-			return err
-		}
-
-		break
+	err = s.writeUsingTempFile(b, iamFname)
+	if err != nil {
+		return fmt.Errorf("write iam file: %w", err)
 	}
 
 	return nil
 }
 
-func (s *IAMServiceInternal) writeTempFile(b []byte) error {
-	fname := filepath.Join(s.dir, iamFile)
-
+func (s *IAMServiceInternal) writeUsingTempFile(b []byte, fname string) error {
 	f, err := os.CreateTemp(s.dir, iamFile)
 	if err != nil {
 		return fmt.Errorf("create temp file: %w", err)
@@ -384,6 +340,7 @@ func (s *IAMServiceInternal) writeTempFile(b []byte) error {
 	defer os.Remove(f.Name())
 
 	_, err = f.Write(b)
+	f.Close()
 	if err != nil {
 		return fmt.Errorf("write temp file: %w", err)
 	}

auth/iam_ipa.go

@@ -0,0 +1,441 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+const IpaVersion = "2.254"
+
+type IpaIAMService struct {
+	client          http.Client
+	id              int
+	version         string
+	host            string
+	vaultName       string
+	username        string
+	password        string
+	kraTransportKey *rsa.PublicKey
+	debug           bool
+	rootAcc         Account
+}
+
+var _ IAMService = &IpaIAMService{}
+
+func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
+	ipa := IpaIAMService{
+		id:        0,
+		version:   IpaVersion,
+		host:      host,
+		vaultName: vaultName,
+		username:  username,
+		password:  password,
+		debug:     debug,
+		rootAcc:   rootAcc,
+	}
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		// this should never happen
+		return nil, fmt.Errorf("cookie jar creation: %w", err)
+	}
+
+	mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
+	tr := &http.Transport{
+		TLSClientConfig: mTLSConfig,
+		Proxy:           http.ProxyFromEnvironment,
+	}
+	ipa.client = http.Client{Jar: jar, Transport: tr}
+
+	err = ipa.login()
+	if err != nil {
+		return nil, fmt.Errorf("ipa login failed: %w", err)
+	}
+
+	req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
+	if err != nil {
+		return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
+	}
+	vaultConfig := struct {
+		Kra_Server_Server             []string
+		Transport_Cert                Base64EncodedWrapped
+		Wrapping_default_algorithm    string
+		Wrapping_supported_algorithms []string
+	}{}
+	err = ipa.rpc(req, &vaultConfig)
+	if err != nil {
+		return nil, fmt.Errorf("ipa vault config: %w", err)
+	}
+
+	cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
+	if err != nil {
+		return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
+	}
+
+	ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
+
+	isSupported := slices.Contains(vaultConfig.Wrapping_supported_algorithms, "aes-128-cbc")
+
+	if !isSupported {
+		return nil,
+			fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
+				vaultConfig.Wrapping_supported_algorithms)
+	}
+	return &ipa, nil
+}
+
+func (ipa *IpaIAMService) CreateAccount(account Account) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ipa.rootAcc.Access {
+		return ipa.rootAcc, nil
+	}
+
+	req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa user_show: %w", err)
+	}
+
+	userResult := struct {
+		Gidnumber []string
+		Uidnumber []string
+	}{}
+
+	err = ipa.rpc(req, &userResult)
+	if err != nil {
+		return Account{}, err
+	}
+
+	uid, err := strconv.Atoi(userResult.Uidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
+	}
+	gid, err := strconv.Atoi(userResult.Gidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
+	}
+
+	account := Account{
+		Access:  access,
+		Role:    RoleUser,
+		UserID:  uid,
+		GroupID: gid,
+	}
+
+	session_key := make([]byte, 16)
+
+	_, err = rand.Read(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot generate session key: %w", err)
+	}
+
+	encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
+	}
+
+	req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
+		map[string]any{"username": access,
+			"session_key":   Base64EncodedWrapped(encryptedKey),
+			"wrapping_algo": "aes-128-cbc"})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
+	}
+
+	data := struct {
+		Vault_data Base64EncodedWrapped
+		Nonce      Base64EncodedWrapped
+	}{}
+
+	err = ipa.rpc(req, &data)
+	if err != nil {
+		return account, err
+	}
+
+	aes, err := aes.NewCipher(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
+	}
+	cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
+	cbc.CryptBlocks(data.Vault_data, data.Vault_data)
+	secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
+	}
+
+	secret := struct {
+		Data Base64Encoded
+	}{}
+	json.Unmarshal(secretUnpaddedJson, &secret)
+	account.Secret = string(secret.Data)
+
+	return account, nil
+}
+
+func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
+	return []Account{}, fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) Shutdown() error {
+	return nil
+}
+
+// Implementation
+
+func (ipa *IpaIAMService) login() error {
+	form := url.Values{}
+	form.Set("user", ipa.username)
+	form.Set("password", ipa.password)
+
+	req, err := http.NewRequest(
+		"POST",
+		fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
+		strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := ipa.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode == 401 {
+		return errors.New("cannot login to FreeIPA: invalid credentials")
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+type rpcRequest = string
+
+type rpcResponse struct {
+	Result    json.RawMessage
+	Principal string
+	Id        int
+	Version   string
+}
+
+func (p rpcResponse) String() string {
+	return string(p.Result)
+}
+
+var errRpc = errors.New("IPA RPC error")
+
+func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
+	err := ipa.login()
+	if err != nil {
+		return err
+	}
+
+	res, err := ipa.rpcInternal(req)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(res.Result, value)
+}
+
+func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
+	httpReq, err := http.NewRequest("POST",
+		fmt.Sprintf("%s/ipa/session/json", ipa.host),
+		strings.NewReader(req))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	ipa.log(fmt.Sprintf("%v", req))
+	httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	httpResp, err := ipa.client.Do(httpReq)
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	bytes, err := io.ReadAll(httpResp.Body)
+	ipa.log(string(bytes))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	result := struct {
+		Result struct {
+			Json    json.RawMessage `json:"result"`
+			Value   string          `json:"value"`
+			Summary any             `json:"summary"`
+		} `json:"result"`
+		Error     json.RawMessage `json:"error"`
+		Id        int             `json:"id"`
+		Principal string          `json:"principal"`
+		Version   string          `json:"version"`
+	}{}
+
+	err = json.Unmarshal(bytes, &result)
+	if err != nil {
+		return rpcResponse{}, err
+	}
+	if string(result.Error) != "null" {
+		return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
+	}
+
+	return rpcResponse{
+		Result:    result.Result.Json,
+		Principal: result.Principal,
+		Id:        result.Id,
+		Version:   result.Version,
+	}, nil
+}
+
+func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
+
+	id := ipa.id
+	ipa.id++
+
+	dict["version"] = ipa.version
+
+	jmethod, errMethod := json.Marshal(method)
+	jargs, errArgs := json.Marshal(args)
+	jdict, errDict := json.Marshal(dict)
+
+	err := errors.Join(errMethod, errArgs, errDict)
+	if err != nil {
+		return "", fmt.Errorf("ipa request invalid: %w", err)
+	}
+
+	request := map[string]interface{}{
+		"id":     id,
+		"method": json.RawMessage(jmethod),
+		"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
+	}
+
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	return string(requestJSON), nil
+}
+
+// pkcs7Unpad validates and unpads data from the given bytes slice.
+// The returned value will be 1 to n bytes smaller depending on the
+// amount of padding, where n is the block size.
+func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
+	if blocksize <= 0 {
+		return nil, errors.New("invalid blocksize")
+	}
+	if len(b) == 0 {
+		return nil, errors.New("invalid PKCS7 data (empty or not padded)")
+	}
+	if len(b)%blocksize != 0 {
+		return nil, errors.New("invalid padding on input")
+	}
+	c := b[len(b)-1]
+	n := int(c)
+	if n == 0 || n > len(b) {
+		return nil, errors.New("invalid padding on input")
+	}
+	for i := 0; i < n; i++ {
+		if b[len(b)-n+i] != c {
+			return nil, errors.New("invalid padding on input")
+		}
+	}
+	return b[:len(b)-n], nil
+}
+
+/*
+e.g.
+
+	"value" {
+		"__base64__": "aGVsbG93b3JsZAo="
+	 }
+*/
+type Base64EncodedWrapped []byte
+
+func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{}
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
+	return err
+}
+
+func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{Base64: base64.StdEncoding.EncodeToString(*b)}
+	return json.Marshal(intermediate)
+}
+
+/*
+e.g.
+
+	"value": "aGVsbG93b3JsZAo="
+*/
+type Base64Encoded []byte
+
+func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
+	var intermediate string
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate)
+	return err
+}
+
+func (ipa *IpaIAMService) log(msg string) {
+	if ipa.debug {
+		log.Println(msg)
+	}
+}

auth/iam_ldap.go

@@ -111,11 +111,13 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 	entry := result.Entries[0]
 	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
 	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
+		return Account{}, fmt.Errorf("invalid entry value for group-id %q: %w",
+			entry.GetAttributeValue(ld.groupIdAtr), err)
 	}
 	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
 	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
+		return Account{}, fmt.Errorf("invalid entry value for user-id %q: %w",
+			entry.GetAttributeValue(ld.userIdAtr), err)
 	}
 	return Account{
 		Access:  entry.GetAttributeValue(ld.accessAtr),
@@ -137,6 +139,9 @@ func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) e
 	if props.UserID != nil {
 		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
 	}
+	if props.Role != "" {
+		req.Replace(ld.roleAtr, []string{string(props.Role)})
+	}
 
 	err := ld.conn.Modify(req)
 	//TODO: Handle non existing user case
@@ -183,11 +188,13 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 	for _, el := range resp.Entries {
 		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
 		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
+			return nil, fmt.Errorf("invalid entry value for group-id %q: %w",
+				el.GetAttributeValue(ld.groupIdAtr), err)
 		}
 		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
 		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
+			return nil, fmt.Errorf("invalid entry value for user-id %q: %w",
+				el.GetAttributeValue(ld.userIdAtr), err)
 		}
 		result = append(result, Account{
 			Access:  el.GetAttributeValue(ld.accessAtr),

auth/iam_s3_object.go

@@ -42,14 +42,6 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
-	client *s3.Client
-
-	access   string
-	secret   string
-	region   string
-	bucket   string
-	endpoint string
-	rootAcc  Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -58,8 +50,15 @@ type IAMServiceS3 struct {
 	// gateway instance if possible.
 	sync.RWMutex
 
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
 	sslSkipVerify bool
 	debug         bool
+	rootAcc       Account
+	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}

auth/iam_single.go

@@ -15,39 +15,49 @@
 package auth
 
 import (
-	"errors"
+	"github.com/versity/versitygw/s3err"
 )
 
 // IAMServiceSingle manages the single tenant (root-only) IAM service
-type IAMServiceSingle struct{}
+type IAMServiceSingle struct {
+	root Account
+}
 
 var _ IAMService = &IAMServiceSingle{}
 
-var ErrNotSupported = errors.New("method is not supported")
+func NewIAMServiceSingle(r Account) IAMService {
+	return &IAMServiceSingle{
+		root: r,
+	}
+}
 
 // CreateAccount not valid in single tenant mode
 func (IAMServiceSingle) CreateAccount(account Account) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
-// GetUserAccount no accounts in single tenant mode
-func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	return Account{}, ErrNoSuchUser
+// GetUserAccount returns root account, if the root access key
+// is provided and "ErrAdminUserNotFound" otherwise
+func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	if access == s.root.Access {
+		return s.root, nil
+	}
+	return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound)
 }
 
 // UpdateUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // ListUserAccounts no accounts in single tenant mode
 func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
-	return []Account{}, nil
+	return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // Shutdown graceful termination of service

auth/iam_vault.go

@@ -47,7 +47,7 @@ func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath,
 		tls.ServerCertificate.FromBytes = []byte(serverCert)
 		if clientCert != "" {
 			if clientCertKey == "" {
-				return nil, fmt.Errorf("client certificate and client certificate should both be specified")
+				return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
 			}
 
 			tls.ClientCertificate.FromBytes = []byte(clientCert)

auth/object_lock.go

@@ -29,9 +29,9 @@ import (
 )
 
 type BucketLockConfig struct {
+	Enabled          bool
 	DefaultRetention *types.DefaultRetention
 	CreatedAt        *time.Time
-	Enabled          bool
 }
 
 func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
@@ -95,7 +95,7 @@ func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfigur
 func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
 	var retention s3response.PutObjectRetentionInput
 	if err := xml.Unmarshal(input, &retention); err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
 	}
 
 	if retention.RetainUntilDate.Before(time.Now()) {
@@ -120,23 +120,23 @@ func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, e
 	return &retention, nil
 }
 
-func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
+func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResult {
 	if status == nil {
 		return nil
 	}
 
 	if *status {
-		return &types.ObjectLockLegalHold{
+		return &s3response.GetObjectLegalHoldResult{
 			Status: types.ObjectLockLegalHoldStatusOn,
 		}
 	}
 
-	return &types.ObjectLockLegalHold{
+	return &s3response.GetObjectLegalHoldResult{
 		Status: types.ObjectLockLegalHoldStatusOff,
 	}
 }
 
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass, isBucketPublic bool, be backend.Backend) error {
 	data, err := be.GetObjectLockConfiguration(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -211,7 +211,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 							if err != nil {
 								return err
 							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+							if isBucketPublic {
+								err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
+							} else {
+								err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+							}
 							if err != nil {
 								return s3err.GetAPIError(s3err.ErrObjectLocked)
 							}
@@ -254,7 +258,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 					if err != nil {
 						return err
 					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+					if isBucketPublic {
+						err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
+					} else {
+						err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+					}
 					if err != nil {
 						return s3err.GetAPIError(s3err.ErrObjectLocked)
 					}

aws/signer/v4/v4.go

@@ -85,10 +85,6 @@ type keyDerivator interface {
 
 // SignerOptions is the SigV4 Signer options.
 type SignerOptions struct {
-
-	// The logger to send log messages to.
-	Logger logging.Logger
-
 	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
 	// request header to the request's query string. This is most commonly used
 	// with pre-signed requests preventing headers from being added to the
@@ -104,6 +100,9 @@ type SignerOptions struct {
 	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
 	DisableURIPathEscaping bool
 
+	// The logger to send log messages to.
+	Logger logging.Logger
+
 	// Enable logging of signed requests.
 	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
 	// presigned URL.
@@ -118,8 +117,8 @@ type SignerOptions struct {
 // Signer applies AWS v4 signing to given request. Use this to sign requests
 // that need to be signed with AWS V4 Signatures.
 type Signer struct {
-	keyDerivator keyDerivator
 	options      SignerOptions
+	keyDerivator keyDerivator
 }
 
 // NewSigner returns a new SigV4 Signer
@@ -134,19 +133,17 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
 }
 
 type httpSigner struct {
-	KeyDerivator keyDerivator
 	Request      *http.Request
-	Credentials  aws.Credentials
-	Time         v4Internal.SigningTime
 	ServiceName  string
 	Region       string
+	Time         v4Internal.SigningTime
+	Credentials  aws.Credentials
+	KeyDerivator keyDerivator
+	IsPreSign    bool
+	SignedHdrs   []string
 
 	PayloadHash string
 
-	SignedHdrs []string
-
-	IsPreSign bool
-
 	DisableHeaderHoisting  bool
 	DisableURIPathEscaping bool
 	DisableSessionToken    bool

backend/azure/azure.go

@@ -27,7 +27,6 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -64,6 +63,7 @@ const (
 	keyBucketLock          key = "Bucketlock"
 	keyObjRetention        key = "Objectretention"
 	keyObjLegalHold        key = "Objectlegalhold"
+	keyExpires             key = "Vgwexpires"
 	onameAttr              key = "Objname"
 	onameAttrLower         key = "objname"
 	metaTmpMultipartPrefix key = ".sgwtmp" + "/multipart"
@@ -77,6 +77,7 @@ func (key) Table() map[string]struct{} {
 		"policy":            {},
 		"bucketlock":        {},
 		"objectretention":   {},
+		"vgwexpires":        {},
 		"objectlegalhold":   {},
 		"objname":           {},
 		".sgwtmp/multipart": {},
@@ -180,11 +181,9 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 			return err
 		}
 
-		var acl auth.ACL
-		if len(aclBytes) > 0 {
-			if err := json.Unmarshal(aclBytes, &acl); err != nil {
-				return fmt.Errorf("unmarshal acl: %w", err)
-			}
+		acl, err := auth.ParseACL(aclBytes)
+		if err != nil {
+			return err
 		}
 
 		if acl.Owner == acct.Access {
@@ -196,7 +195,6 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 }
 
 func (az *Azure) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
-	fmt.Printf("%+v\n", input)
 	pager := az.client.NewListContainersPager(
 		&service.ListContainersOptions{
 			Include: service.ListContainersInclude{
@@ -294,14 +292,27 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin
 	return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership))
 }
 
-func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
-	tags, err := parseTags(po.Tagging)
+func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
+	tags, err := backend.ParseObjectTags(getString(po.Tagging))
 	if err != nil {
 		return s3response.PutObjectOutput{}, err
 	}
 
+	metadata := parseMetadata(po.Metadata)
+
+	// Store the "Expires" property in the object metadata
+	if getString(po.Expires) != "" {
+		if metadata == nil {
+			metadata = map[string]*string{
+				string(keyExpires): po.Expires,
+			}
+		} else {
+			metadata[string(keyExpires)] = po.Expires
+		}
+	}
+
 	opts := &blockblob.UploadStreamOptions{
-		Metadata: parseMetadata(po.Metadata),
+		Metadata: metadata,
 		Tags:     tags,
 	}
 
@@ -309,6 +320,8 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respon
 	opts.HTTPHeaders.BlobContentEncoding = po.ContentEncoding
 	opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage
 	opts.HTTPHeaders.BlobContentDisposition = po.ContentDisposition
+	opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage
+	opts.HTTPHeaders.BlobCacheControl = po.CacheControl
 	if strings.HasSuffix(*po.Key, "/") {
 		// Hardcode "application/x-directory" for direcoty objects
 		opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DirContentType)
@@ -391,17 +404,29 @@ func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error {
 }
 
 func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	client, err := az.getBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
 	var opts *azblob.DownloadStreamOptions
 	if *input.Range != "" {
-		offset, count, err := backend.ParseRange(0, *input.Range)
+		offset, count, isValid, err := backend.ParseObjectRange(*resp.ContentLength, *input.Range)
 		if err != nil {
 			return nil, err
 		}
-		opts = &azblob.DownloadStreamOptions{
-			Range: blob.HTTPRange{
-				Count:  count,
-				Offset: offset,
-			},
+		if isValid {
+			opts = &azblob.DownloadStreamOptions{
+				Range: blob.HTTPRange{
+					Count:  count,
+					Offset: offset,
+				},
+			}
 		}
 	}
 	blobDownloadResponse, err := az.client.DownloadStream(ctx, *input.Bucket, *input.Key, opts)
@@ -420,17 +445,21 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G
 	}
 
 	return &s3.GetObjectOutput{
-		AcceptRanges:    input.Range,
-		ContentLength:   blobDownloadResponse.ContentLength,
-		ContentEncoding: blobDownloadResponse.ContentEncoding,
-		ContentType:     contentType,
-		ETag:            (*string)(blobDownloadResponse.ETag),
-		LastModified:    blobDownloadResponse.LastModified,
-		Metadata:        parseAzMetadata(blobDownloadResponse.Metadata),
-		TagCount:        &tagcount,
-		ContentRange:    blobDownloadResponse.ContentRange,
-		Body:            blobDownloadResponse.Body,
-		StorageClass:    types.StorageClassStandard,
+		AcceptRanges:       backend.GetPtrFromString("bytes"),
+		ContentLength:      blobDownloadResponse.ContentLength,
+		ContentEncoding:    blobDownloadResponse.ContentEncoding,
+		ContentType:        contentType,
+		ContentDisposition: blobDownloadResponse.ContentDisposition,
+		ContentLanguage:    blobDownloadResponse.ContentLanguage,
+		CacheControl:       blobDownloadResponse.CacheControl,
+		ExpiresString:      blobDownloadResponse.Metadata[string(keyExpires)],
+		ETag:               (*string)(blobDownloadResponse.ETag),
+		LastModified:       blobDownloadResponse.LastModified,
+		Metadata:           parseAndFilterAzMetadata(blobDownloadResponse.Metadata),
+		TagCount:           &tagcount,
+		ContentRange:       blobDownloadResponse.ContentRange,
+		Body:               blobDownloadResponse.Body,
+		StorageClass:       types.StorageClassStandard,
 	}, nil
 }
 
@@ -476,18 +505,35 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 	if err != nil {
 		return nil, azureErrToS3Err(err)
 	}
+	var size int64
+	if resp.ContentLength != nil {
+		size = *resp.ContentLength
+	}
+
+	startOffset, length, isValid, err := backend.ParseObjectRange(size, getString(input.Range))
+	if err != nil {
+		return nil, err
+	}
+
+	var contentRange string
+	if isValid {
+		contentRange = fmt.Sprintf("bytes %v-%v/%v",
+			startOffset, startOffset+length-1, size)
+	}
 
 	result := &s3.HeadObjectOutput{
-		AcceptRanges:       resp.AcceptRanges,
-		ContentLength:      resp.ContentLength,
+		ContentRange:       &contentRange,
+		AcceptRanges:       backend.GetPtrFromString("bytes"),
+		ContentLength:      &length,
 		ContentType:        resp.ContentType,
 		ContentEncoding:    resp.ContentEncoding,
 		ContentLanguage:    resp.ContentLanguage,
 		ContentDisposition: resp.ContentDisposition,
+		CacheControl:       resp.CacheControl,
+		ExpiresString:      resp.Metadata[string(keyExpires)],
 		ETag:               (*string)(resp.ETag),
 		LastModified:       resp.LastModified,
-		Metadata:           parseAzMetadata(resp.Metadata),
-		Expires:            resp.ExpiresOn,
+		Metadata:           parseAndFilterAzMetadata(resp.Metadata),
 		StorageClass:       types.StorageClassStandard,
 	}
 
@@ -522,7 +568,7 @@ func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAtt
 	}
 
 	return s3response.GetObjectAttributesResponse{
-		ETag:         data.ETag,
+		ETag:         backend.TrimEtag(data.ETag),
 		ObjectSize:   data.ContentLength,
 		StorageClass: data.StorageClass,
 		LastModified: data.LastModified,
@@ -552,6 +598,18 @@ func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s
 		maxKeys = *input.MaxKeys
 	}
 
+	// Retrieve the bucket acl to get the bucket owner
+	// All the objects in the bucket are owner by the bucket owner
+	aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
+	if err != nil {
+		return s3response.ListObjectsResult{}, azureErrToS3Err(err)
+	}
+
+	acl, err := auth.ParseACL(aclBytes)
+	if err != nil {
+		return s3response.ListObjectsResult{}, err
+	}
+
 Pager:
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
@@ -565,11 +623,14 @@ Pager:
 				break Pager
 			}
 			objects = append(objects, s3response.Object{
-				ETag:         (*string)(v.Properties.ETag),
+				ETag:         backend.GetPtrFromString(fmt.Sprintf("%q", *v.Properties.ETag)),
 				Key:          v.Name,
 				LastModified: v.Properties.LastModified,
 				Size:         v.Properties.ContentLength,
 				StorageClass: types.ObjectStorageClassStandard,
+				Owner: &types.Owner{
+					ID: &acl.Owner,
+				},
 			})
 		}
 		for _, v := range resp.Segment.BlobPrefixes {
@@ -629,10 +690,29 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input
 	var nextMarker *string
 	var isTruncated bool
 	var maxKeys int32 = math.MaxInt32
+	var fetchOwner bool
 
 	if input.MaxKeys != nil {
 		maxKeys = *input.MaxKeys
 	}
+	if input.FetchOwner != nil {
+		fetchOwner = *input.FetchOwner
+	}
+
+	// Retrieve the bucket acl to get the bucket owner, if "fetchOwner" is true
+	// All the objects in the bucket are owner by the bucket owner
+	var acl auth.ACL
+	if fetchOwner {
+		aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
+		if err != nil {
+			return s3response.ListObjectsV2Result{}, azureErrToS3Err(err)
+		}
+
+		acl, err = auth.ParseACL(aclBytes)
+		if err != nil {
+			return s3response.ListObjectsV2Result{}, err
+		}
+	}
 
 Pager:
 	for pager.More() {
@@ -646,13 +726,20 @@ Pager:
 				isTruncated = true
 				break Pager
 			}
-			objects = append(objects, s3response.Object{
-				ETag:         (*string)(v.Properties.ETag),
+
+			obj := s3response.Object{
+				ETag:         backend.GetPtrFromString(fmt.Sprintf("%q", *v.Properties.ETag)),
 				Key:          v.Name,
 				LastModified: v.Properties.LastModified,
 				Size:         v.Properties.ContentLength,
 				StorageClass: types.ObjectStorageClassStandard,
-			})
+			}
+			if fetchOwner {
+				obj.Owner = &types.Owner{
+					ID: &acl.Owner,
+				}
+			}
+			objects = append(objects, obj)
 		}
 		for _, v := range resp.Segment.BlobPrefixes {
 			if *v.Name <= marker {
@@ -735,41 +822,158 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
 	}, nil
 }
 
-func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	bclient, err := az.getBlobClient(*input.Bucket, *input.Key)
+func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
+	dstClient, err := az.getBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return nil, err
+		return s3response.CopyObjectOutput{}, err
 	}
-
 	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource {
-		props, err := bclient.GetProperties(ctx, nil)
+		if input.MetadataDirective != types.MetadataDirectiveReplace {
+			return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+		}
+
+		// Set object meta http headers
+		res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{
+			BlobCacheControl:       input.CacheControl,
+			BlobContentDisposition: input.ContentDisposition,
+			BlobContentEncoding:    input.ContentEncoding,
+			BlobContentLanguage:    input.ContentLanguage,
+			BlobContentType:        input.ContentType,
+		}, nil)
 		if err != nil {
-			return nil, azureErrToS3Err(err)
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 		}
 
-		mdmap := props.Metadata
-		if isMetaSame(mdmap, input.Metadata) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+		meta := input.Metadata
+		if meta == nil {
+			meta = make(map[string]string)
+		}
+
+		// Embed "Expires" in object metadata
+		if getString(input.Expires) != "" {
+			meta[string(keyExpires)] = *input.Expires
+		}
+		// Set object metadata
+		_, err = dstClient.SetMetadata(ctx, parseMetadata(meta), nil)
+		if err != nil {
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
+		}
+
+		// Set object legal hold
+		if input.ObjectLockLegalHoldStatus != "" {
+			err = az.PutObjectLegalHold(ctx, *input.Bucket, *input.Key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn)
+			if err != nil {
+				return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
+			}
+		}
+		// Set object retention
+		if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil {
+			retention := s3response.PutObjectRetentionInput{
+				Mode: types.ObjectLockRetentionMode(input.ObjectLockMode),
+				RetainUntilDate: s3response.AmzDate{
+					Time: *input.ObjectLockRetainUntilDate,
+				},
+			}
+
+			retParsed, err := json.Marshal(retention)
+			if err != nil {
+				return s3response.CopyObjectOutput{}, fmt.Errorf("parse object retention: %w", err)
+			}
+			err = az.PutObjectRetention(ctx, *input.Bucket, *input.Key, "", true, retParsed)
+			if err != nil {
+				return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
+			}
+		}
+
+		// Set object Tagging, if tagging directive is "REPLACE"
+		if input.TaggingDirective == types.TaggingDirectiveReplace {
+			tags, err := backend.ParseObjectTags(getString(input.Tagging))
+			if err != nil {
+				return s3response.CopyObjectOutput{}, err
+			}
+			_, err = dstClient.SetTags(ctx, tags, nil)
+			if err != nil {
+				return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
+			}
+		}
+
+		return s3response.CopyObjectOutput{
+			CopyObjectResult: &s3response.CopyObjectResult{
+				LastModified: res.LastModified,
+				ETag:         (*string)(res.ETag),
+			},
+		}, nil
+	}
+
+	srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
+	if err != nil {
+		return s3response.CopyObjectOutput{}, err
+	}
+
+	// Get the source object
+	downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
+	if err != nil {
+		return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
+	}
+
+	pInput := s3response.PutObjectInput{
+		Body:                      downloadResp.Body,
+		Bucket:                    input.Bucket,
+		Key:                       input.Key,
+		ContentLength:             downloadResp.ContentLength,
+		ContentType:               input.ContentType,
+		ContentEncoding:           input.ContentEncoding,
+		ContentDisposition:        input.ContentDisposition,
+		ContentLanguage:           input.ContentLanguage,
+		CacheControl:              input.CacheControl,
+		Expires:                   input.Expires,
+		Metadata:                  input.Metadata,
+		ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate,
+		ObjectLockMode:            input.ObjectLockMode,
+		ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus,
+	}
+
+	if input.MetadataDirective == types.MetadataDirectiveCopy {
+		// Expires is in downloadResp.Metadata
+		pInput.Expires = nil
+		pInput.CacheControl = downloadResp.CacheControl
+		pInput.ContentDisposition = downloadResp.ContentDisposition
+		pInput.ContentEncoding = downloadResp.ContentEncoding
+		pInput.ContentLanguage = downloadResp.ContentLanguage
+		pInput.ContentType = downloadResp.ContentType
+		pInput.Metadata = parseAzMetadata(downloadResp.Metadata)
+	}
+
+	if input.TaggingDirective == types.TaggingDirectiveReplace {
+		pInput.Tagging = input.Tagging
+	}
+
+	// Create the destination object
+	resp, err := az.PutObject(ctx, pInput)
+	if err != nil {
+		return s3response.CopyObjectOutput{}, err
+	}
+
+	// Copy the object tagging, if tagging directive is "COPY"
+	if input.TaggingDirective == types.TaggingDirectiveCopy {
+		srcClient, err := az.getBlobClient(srcBucket, srcObj)
+		if err != nil {
+			return s3response.CopyObjectOutput{}, err
+		}
+		res, err := srcClient.GetTags(ctx, nil)
+		if err != nil {
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
+		}
+
+		_, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil)
+		if err != nil {
+			return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
 		}
 	}
 
-	tags, err := parseTags(input.Tagging)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := bclient.CopyFromURL(ctx, az.serviceURL+"/"+*input.CopySource, &blob.CopyFromURLOptions{
-		BlobTags: tags,
-		Metadata: parseMetadata(input.Metadata),
-	})
-	if err != nil {
-		return nil, azureErrToS3Err(err)
-	}
-
-	return &s3.CopyObjectOutput{
-		CopyObjectResult: &types.CopyObjectResult{
-			ETag:         (*string)(resp.ETag),
-			LastModified: resp.LastModified,
+	return s3response.CopyObjectOutput{
+		CopyObjectResult: &s3response.CopyObjectResult{
+			ETag: &resp.ETag,
 		},
 	}, nil
 }
@@ -816,7 +1020,7 @@ func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object string)
 	return nil
 }
 
-func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	if input.ObjectLockLegalHoldStatus != "" || input.ObjectLockMode != "" {
 		bucketLock, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyBucketLock))
 		if err != nil {
@@ -840,21 +1044,14 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 	meta := parseMetadata(input.Metadata)
 	meta[string(onameAttr)] = input.Key
 
+	if getString(input.Expires) != "" {
+		meta[string(keyExpires)] = input.Expires
+	}
+
 	// parse object tags
-	tagsStr := getString(input.Tagging)
-	tags := map[string]string{}
-	if tagsStr != "" {
-		tagParts := strings.Split(tagsStr, "&")
-		for _, prt := range tagParts {
-			p := strings.Split(prt, "=")
-			if len(p) != 2 {
-				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTag)
-			}
-			if len(p[0]) > 128 || len(p[1]) > 256 {
-				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTag)
-			}
-			tags[p[0]] = p[1]
-		}
+	tags, err := backend.ParseObjectTags(getString(input.Tagging))
+	if err != nil {
+		return s3response.InitiateMultipartUploadResult{}, err
 	}
 
 	// set blob legal hold status in metadata
@@ -882,18 +1079,19 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 	opts := &blockblob.UploadBufferOptions{
 		Metadata: meta,
 		Tags:     tags,
-	}
-	if getString(input.ContentType) != "" {
-		opts.HTTPHeaders = &blob.HTTPHeaders{
-			BlobContentType:     input.ContentType,
-			BlobContentEncoding: input.ContentEncoding,
-		}
+		HTTPHeaders: &blob.HTTPHeaders{
+			BlobContentType:        input.ContentType,
+			BlobContentEncoding:    input.ContentEncoding,
+			BlobCacheControl:       input.CacheControl,
+			BlobContentDisposition: input.ContentDisposition,
+			BlobContentLanguage:    input.ContentLanguage,
+		},
 	}
 
 	// Create and empty blob in .sgwtmp/multipart/<uploadId>/<object hash>
 	// The blob indicates multipart upload initialization and holds the mp metadata
 	// e.g tagging, content-type, metadata, object lock status ...
-	_, err := az.client.UploadBuffer(ctx, *input.Bucket, tmpPath, []byte{}, opts)
+	_, err = az.client.UploadBuffer(ctx, *input.Bucket, tmpPath, []byte{}, opts)
 	if err != nil {
 		return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err)
 	}
@@ -906,9 +1104,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 }
 
 // Each part is translated into an uncommitted block in a newly created blob in staging area
-func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 	if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
-		return "", err
+		return nil, err
 	}
 
 	// TODO: request streamable version of StageBlock()
@@ -917,32 +1115,34 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta
 	// the body in memory to create an io.ReadSeekCloser
 	rdr, err := getReadSeekCloser(input.Body)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	// block id serves as etag here
-	etag = blockIDInt32ToBase64(*input.PartNumber)
+	etag := blockIDInt32ToBase64(*input.PartNumber)
 	_, err = client.StageBlock(ctx, etag, rdr, nil)
 	if err != nil {
-		return "", parseMpError(err)
+		return nil, parseMpError(err)
 	}
 
-	return etag, nil
+	return &s3.UploadPartOutput{
+		ETag: &etag,
+	}, nil
 }
 
-func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return s3response.CopyObjectResult{}, nil
+		return s3response.CopyPartResult{}, nil
 	}
 
 	if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
-		return s3response.CopyObjectResult{}, err
+		return s3response.CopyPartResult{}, err
 	}
 
 	eTag := blockIDInt32ToBase64(*input.PartNumber)
@@ -950,10 +1150,10 @@ func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInp
 	//TODO: the action returns not implemented on azurite, maybe in production this will work?
 	_, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil)
 	if err != nil {
-		return s3response.CopyObjectResult{}, parseMpError(err)
+		return s3response.CopyPartResult{}, parseMpError(err)
 	}
 
-	return s3response.CopyObjectResult{}, nil
+	return s3response.CopyPartResult{}, nil
 }
 
 // Lists all uncommitted parts from the blob
@@ -1165,83 +1365,114 @@ func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultip
 // Copeies the multipart metadata from .sgwtmp namespace into the newly created blob
 // Deletes the multipart upload 'blob' from .sgwtmp namespace
 // It indicates the end of the multipart upload
-func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+	var res s3response.CompleteMultipartUploadResult
+
 	tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
 	blobClient, err := az.getBlobClient(*input.Bucket, tmpPath)
 	if err != nil {
-		return nil, err
+		return res, "", err
 	}
 
 	props, err := blobClient.GetProperties(ctx, nil)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 	tags, err := blobClient.GetTags(ctx, nil)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return nil, err
+		return res, "", err
 	}
 	blockIds := []string{}
 
 	blockList, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
 	if err != nil {
-		return nil, azureErrToS3Err(err)
+		return res, "", azureErrToS3Err(err)
 	}
 
 	if len(blockList.UncommittedBlocks) != len(input.MultipartUpload.Parts) {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 	}
 
-	slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int {
-		ptNumber, _ := decodeBlockId(*a.Name)
-		nextPtNumber, _ := decodeBlockId(*b.Name)
-		return ptNumber - nextPtNumber
-	})
-
-	for i, block := range blockList.UncommittedBlocks {
-		ptNumber, err := decodeBlockId(*block.Name)
+	uncommittedBlocks := map[int32]*blockblob.Block{}
+	for _, el := range blockList.UncommittedBlocks {
+		ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name))
 		if err != nil {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+			return res, "", fmt.Errorf("invalid block name: %w", err)
 		}
 
-		if *input.MultipartUpload.Parts[i].ETag != *block.Name {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		uncommittedBlocks[int32(ptNumber)] = el
+	}
+
+	// The initialie values is the lower limit of partNumber: 0
+	var totalSize int64
+	var partNumber int32
+	last := len(blockList.UncommittedBlocks) - 1
+	for i, part := range input.MultipartUpload.Parts {
+		if part.PartNumber == nil {
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		if *part.PartNumber < 1 {
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
 		}
+		if *part.PartNumber <= partNumber {
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder)
+		}
+		partNumber = *part.PartNumber
+
+		block, ok := uncommittedBlocks[*part.PartNumber]
+		if !ok {
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		if *part.ETag != *block.Name {
+			return res, "", s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		// all parts except the last need to be greater, than
+		// the minimum allowed size (5 Mib)
+		if i < last && *block.Size < backend.MinPartSize {
+			return res, "", s3err.GetAPIError(s3err.ErrEntityTooSmall)
+		}
+		totalSize += *block.Size
 		blockIds = append(blockIds, *block.Name)
 	}
 
+	if input.MpuObjectSize != nil && totalSize != *input.MpuObjectSize {
+		return res, "", s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize)
+	}
+
 	opts := &blockblob.CommitBlockListOptions{
 		Metadata: props.Metadata,
 		Tags:     parseAzTags(tags.BlobTagSet),
 	}
 	opts.HTTPHeaders = &blob.HTTPHeaders{
-		BlobContentType:     props.ContentType,
-		BlobContentEncoding: props.ContentEncoding,
+		BlobContentType:        props.ContentType,
+		BlobContentEncoding:    props.ContentEncoding,
+		BlobContentDisposition: props.ContentDisposition,
+		BlobContentLanguage:    props.ContentLanguage,
+		BlobCacheControl:       props.CacheControl,
 	}
 
 	resp, err := client.CommitBlockList(ctx, blockIds, opts)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 
 	// cleanup the multipart upload
 	_, err = blobClient.Delete(ctx, nil)
 	if err != nil {
-		return nil, parseMpError(err)
+		return res, "", parseMpError(err)
 	}
 
-	return &s3.CompleteMultipartUploadOutput{
+	return s3response.CompleteMultipartUploadResult{
 		Bucket: input.Bucket,
 		Key:    input.Key,
 		ETag:   (*string)(resp.ETag),
-	}, nil
+	}, "", nil
 }
 
 func (az *Azure) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
@@ -1459,7 +1690,10 @@ func (az *Azure) ChangeBucketOwner(ctx context.Context, bucket string, acl []byt
 // The action actually returns the containers owned by the user, who initialized the gateway
 // TODO: Not sure if there's a way to list all the containers and owners?
 func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
-	pager := az.client.NewListContainersPager(nil)
+	opts := &service.ListContainersOptions{
+		Include: service.ListContainersInclude{Metadata: true},
+	}
+	pager := az.client.NewListContainersPager(opts)
 
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
@@ -1558,7 +1792,7 @@ func parseMetadata(m map[string]string) map[string]*string {
 	return meta
 }
 
-func parseAzMetadata(m map[string]*string) map[string]string {
+func parseAndFilterAzMetadata(m map[string]*string) map[string]string {
 	if m == nil {
 		return nil
 	}
@@ -1577,22 +1811,17 @@ func parseAzMetadata(m map[string]*string) map[string]string {
 	return meta
 }
 
-func parseTags(tagstr *string) (map[string]string, error) {
-	tagsStr := getString(tagstr)
-	tags := make(map[string]string)
-
-	if tagsStr != "" {
-		tagParts := strings.Split(tagsStr, "&")
-		for _, prt := range tagParts {
-			p := strings.Split(prt, "=")
-			if len(p) != 2 {
-				return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
-			}
-			tags[p[0]] = p[1]
-		}
+func parseAzMetadata(m map[string]*string) map[string]string {
+	if m == nil {
+		return nil
 	}
 
-	return tags, nil
+	meta := make(map[string]string)
+
+	for k, v := range m {
+		meta[k] = *v
+	}
+	return meta
 }
 
 func parseAzTags(tagSet []*blob.Tags) map[string]string {
@@ -1737,7 +1966,7 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string
 func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 	data, ok := meta[string(key)]
 	if !ok {
-		return nil, s3err.GetAPIError(s3err.ErrInternalError)
+		return &auth.ACL{}, nil
 	}
 
 	value, err := decodeString(*data)
@@ -1745,37 +1974,14 @@ func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 		return nil, err
 	}
 
-	var acl auth.ACL
-	if len(value) == 0 {
-		return &acl, nil
-	}
-
-	err = json.Unmarshal(value, &acl)
+	acl, err := auth.ParseACL(value)
 	if err != nil {
-		return nil, fmt.Errorf("unmarshal acl: %w", err)
+		return nil, err
 	}
 
 	return &acl, nil
 }
 
-func isMetaSame(azMeta map[string]*string, awsMeta map[string]string) bool {
-	if len(azMeta) != len(awsMeta) {
-		return false
-	}
-
-	for key, val := range azMeta {
-		if key == string(keyAclCapital) || key == string(keyAclLower) {
-			continue
-		}
-		awsVal, ok := awsMeta[key]
-		if !ok || awsVal != *val {
-			return false
-		}
-	}
-
-	return true
-}
-
 func createMetaTmpPath(obj, uploadId string) string {
 	objNameSum := sha256.Sum256([]byte(obj))
 	return filepath.Join(string(metaTmpMultipartPrefix), uploadId, fmt.Sprintf("%x", objNameSum))

backend/azure/err.go

@@ -40,7 +40,7 @@ func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
 	case "BlobNotFound":
 		return s3err.GetAPIError(s3err.ErrNoSuchKey)
 	case "TagsTooLarge":
-		return s3err.GetAPIError(s3err.ErrInvalidTag)
+		return s3err.GetAPIError(s3err.ErrInvalidTagValue)
 	case "Requested Range Not Satisfiable":
 		return s3err.GetAPIError(s3err.ErrInvalidRange)
 	}

backend/backend.go

@@ -46,23 +46,26 @@ type Backend interface {
 	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
 	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
 	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
+	PutBucketCors(context.Context, []byte) error
+	GetBucketCors(_ context.Context, bucket string) ([]byte, error)
+	DeleteBucketCors(_ context.Context, bucket string) error
 
 	// multipart operations
-	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
-	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
+	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (_ s3response.CompleteMultipartUploadResult, versionid string, _ error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
-	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
-	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// standard object operations
-	PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
+	PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
-	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error)
 	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
 	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
@@ -150,12 +153,21 @@ func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket s
 func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketCors(context.Context, []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+	return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -166,14 +178,14 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
 func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
 	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+	return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
@@ -188,8 +200,8 @@ func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (
 func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
+	return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)

backend/common.go

@@ -17,12 +17,17 @@ package backend
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
+	"hash"
 	"io"
-	"net/http"
+	"io/fs"
+	"net/url"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -34,6 +39,9 @@ const (
 	// this is the media type for directories in AWS and Nextcloud
 	DirContentType     = "application/x-directory"
 	DefaultContentType = "binary/octet-stream"
+
+	// this is the minimum allowed size for mp parts
+	MinPartSize = 5 * 1024 * 1024
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -68,45 +76,130 @@ func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
 
+func TrimEtag(etag *string) *string {
+	if etag == nil {
+		return nil
+	}
+
+	return GetPtrFromString(strings.Trim(*etag, "\""))
+}
+
 var (
-	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidRange           = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
 )
 
-// ParseRange parses input range header and returns startoffset, length, and
-// error. If no endoffset specified, then length is set to -1.
-func ParseRange(size int64, acceptRange string) (int64, int64, error) {
+// ParseObjectRange parses input range header and returns startoffset, length, isValid
+// and error. If no endoffset specified, then length is set to the object size
+// for invalid inputs, it returns no error, but isValid=false
+// `InvalidRange` error is returnd, only if startoffset is greater than the object size
+func ParseObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
+	if acceptRange == "" {
+		return 0, size, false, nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) != 2 {
+		return 0, size, false, nil
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, size, false, nil
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) != 2 {
+		return 0, size, false, nil
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil && bRange[0] != "" {
+		return 0, size, false, nil
+	}
+
+	if bRange[1] == "" {
+		if bRange[0] == "" {
+			return 0, size, false, nil
+		}
+		if startOffset >= size {
+			return 0, 0, false, errInvalidRange
+		}
+		return startOffset, size - startOffset, true, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, size, false, nil
+	}
+
+	if startOffset > endOffset {
+		return 0, size, false, nil
+	}
+
+	// for ranges like 'bytes=-100' return the last bytes specified with 'endOffset'
+	if bRange[0] == "" {
+		endOffset = min(endOffset, size)
+		return size - endOffset, endOffset, true, nil
+	}
+
+	if startOffset >= size {
+		return 0, 0, false, errInvalidRange
+	}
+
+	if endOffset >= size {
+		endOffset = size - 1
+	}
+
+	return startOffset, endOffset - startOffset + 1, true, nil
+}
+
+// ParseCopySourceRange parses input range header and returns startoffset, length
+// and error. If no endoffset specified, then length is set to the object size
+func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
 		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
 
-	if len(rangeKv) < 2 {
-		return 0, 0, errInvalidRange
+	if len(rangeKv) != 2 {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 1 || len(bRange) > 2 {
-		return 0, 0, errInvalidRange
+	if len(bRange) != 2 {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
-	endOffset := int64(-1)
-	if len(bRange) == 1 || bRange[1] == "" {
-		return startOffset, endOffset, nil
+	if startOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
-	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
+	if bRange[1] == "" {
+		return startOffset, size - startOffset + 1, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	if endOffset < startOffset {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if endOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
@@ -136,12 +229,82 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
 	return srcBucket, srcObject, versionId, nil
 }
 
-func CreateExceedingRangeErr(objSize int64) s3err.APIError {
-	return s3err.APIError{
-		Code:           "InvalidArgument",
-		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
-		HTTPStatusCode: http.StatusBadRequest,
+// ParseObjectTags parses the url encoded input string into
+// map[string]string with unescaped key/value pair
+func ParseObjectTags(tagging string) (map[string]string, error) {
+	if tagging == "" {
+		return nil, nil
 	}
+
+	tagSet := make(map[string]string)
+
+	for tagging != "" {
+		var tag string
+		tag, tagging, _ = strings.Cut(tagging, "&")
+		// if 'tag' before the first appearance of '&' is empty continue
+		if tag == "" {
+			continue
+		}
+
+		key, value, found := strings.Cut(tag, "=")
+		// if key is empty, but "=" is present, return invalid url ecnoding err
+		if found && key == "" {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// return invalid tag key, if the key is longer than 128
+		if len(key) > 128 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// return invalid tag value, if tag value is longer than 256
+		if len(value) > 256 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// query unescape tag key
+		key, err := url.QueryUnescape(key)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// query unescape tag value
+		value, err = url.QueryUnescape(value)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// check tag key to be valid
+		if !isValidTagComponent(key) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// check tag value to be valid
+		if !isValidTagComponent(value) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// duplicate keys are not allowed: return invalid url encoding err
+		_, ok := tagSet[key]
+		if ok {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		tagSet[key] = value
+	}
+
+	return tagSet, nil
+}
+
+var validTagComponent = regexp.MustCompile(`^[a-zA-Z0-9:/_.\-+ ]+$`)
+
+// isValidTagComponent matches strings which contain letters, decimal digits,
+// and special chars: '/', '_', '-', '+', '.', ' ' (space)
+func isValidTagComponent(str string) bool {
+	if str == "" {
+		return true
+	}
+	return validTagComponent.Match([]byte(str))
 }
 
 func GetMultipartMD5(parts []types.CompletedPart) string {
@@ -149,8 +312,8 @@ func GetMultipartMD5(parts []types.CompletedPart) string {
 	for _, part := range parts {
 		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
 	}
-	s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
-	return s3MD5
+
+	return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
 }
 
 func getEtagBytes(etag string) []byte {
@@ -178,3 +341,65 @@ func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
 func (f *FileSectionReadCloser) Close() error {
 	return f.F.Close()
 }
+
+// MoveFile moves a file from source to destination.
+func MoveFile(source, destination string, perm os.FileMode) error {
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	err := os.Rename(source, destination)
+	if err == nil || !errors.Is(err, syscall.EXDEV) {
+		return err
+	}
+
+	// Rename can fail if the source and destination are not on the same
+	// filesystem. The fallback is to copy the file and then remove the source.
+	// We need to be careful that the desination does not exist before copying
+	// to prevent any other simultaneous writes to the file.
+	sourceFile, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("open source: %w", err)
+	}
+	defer sourceFile.Close()
+
+	var destFile *os.File
+	for {
+		destFile, err = os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+		if err != nil {
+			if errors.Is(err, fs.ErrExist) {
+				if removeErr := os.Remove(destination); removeErr != nil {
+					return fmt.Errorf("remove existing destination: %w", removeErr)
+				}
+				continue
+			}
+			return fmt.Errorf("create destination: %w", err)
+		}
+		break
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return fmt.Errorf("copy data: %w", err)
+	}
+
+	err = os.Remove(source)
+	if err != nil {
+		return fmt.Errorf("remove source: %w", err)
+	}
+
+	return nil
+}
+
+// GenerateEtag generates a new quoted etag from the provided hash.Hash
+func GenerateEtag(h hash.Hash) string {
+	dataSum := h.Sum(nil)
+	return fmt.Sprintf("\"%s\"", hex.EncodeToString(dataSum[:]))
+}
+
+// AreEtagsSame compares 2 etags by ignoring quotes
+func AreEtagsSame(e1, e2 string) bool {
+	return strings.Trim(e1, `"`) == strings.Trim(e2, `"`)
+}

backend/meta/none.go

@@ -0,0 +1,54 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"os"
+)
+
+// NoMeta is a metadata storer that does not store metadata.
+// This can be useful for read only mounts where attempting to store metadata
+// would fail.
+type NoMeta struct{}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+// always returns ErrNoSuchKey
+func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
+	return nil, ErrNoSuchKey
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+// always returns nil without storing the attribute
+func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+// always returns nil without deleting the attribute
+func (NoMeta) DeleteAttribute(_, _, _ string) error {
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+// always returns an empty list of attributes
+func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
+	return []string{}, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+// always returns nil without deleting any attributes
+func (NoMeta) DeleteAttributes(bucket, object string) error {
+	return nil
+}

backend/meta/sidecar.go

@@ -0,0 +1,139 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// SideCar is a metadata storer that uses sidecar files to store metadata.
+type SideCar struct {
+	dir string
+}
+
+const (
+	sidecarmeta = "meta"
+)
+
+// NewSideCar creates a new SideCar metadata storer.
+func NewSideCar(dir string) (SideCar, error) {
+	fi, err := os.Lstat(dir)
+	if err != nil {
+		return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
+	}
+	if !fi.IsDir() {
+		return SideCar{}, fmt.Errorf("not a directory")
+	}
+
+	return SideCar{dir: dir}, nil
+}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	value, err := os.ReadFile(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, ErrNoSuchKey
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attribute: %v", err)
+	}
+
+	return value, nil
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	err := os.MkdirAll(metadir, 0777)
+	if err != nil {
+		return fmt.Errorf("failed to create metadata directory: %v", err)
+	}
+
+	attr := filepath.Join(metadir, attribute)
+	err = os.WriteFile(attr, value, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to write attribute: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	err := os.Remove(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return ErrNoSuchKey
+	}
+	if err != nil {
+		return fmt.Errorf("failed to remove attribute: %v", err)
+	}
+
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	ents, err := os.ReadDir(metadir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []string{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to list attributes: %v", err)
+	}
+
+	var attrs []string
+	for _, ent := range ents {
+		attrs = append(attrs, ent.Name())
+	}
+
+	return attrs, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+func (s SideCar) DeleteAttributes(bucket, object string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	err := os.RemoveAll(metadir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove attributes: %v", err)
+	}
+	return nil
+}

backend/meta/xattr.go

@@ -23,6 +23,7 @@ import (
 	"syscall"
 
 	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/s3err"
 )
 
 const (
@@ -56,10 +57,18 @@ func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute strin
 // StoreAttribute stores the value of a specific attribute for an object in a bucket.
 func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
 	if f != nil {
-		return xattr.FSet(f, xattrPrefix+attribute, value)
+		err := xattr.FSet(f, xattrPrefix+attribute, value)
+		if errors.Is(err, syscall.EROFS) {
+			return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return err
 	}
 
-	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
+	return err
 }
 
 // DeleteAttribute removes the value of a specific attribute for an object in a bucket.
@@ -68,6 +77,9 @@ func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
 	if errors.Is(err, xattr.ENOATTR) {
 		return ErrNoSuchKey
 	}
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
 	return err
 }
 

backend/posix/with_otmpfile.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 	"golang.org/x/sys/unix"
 )
 
@@ -38,12 +39,12 @@ type tmpfile struct {
 	f          *os.File
 	bucket     string
 	objname    string
+	isOTmp     bool
 	size       int64
+	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
-	isOTmp     bool
-	needsChown bool
 }
 
 var (
@@ -51,9 +52,13 @@ var (
 	defaultFilePerm uint32 = 0644
 )
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool, forceNoTmpFile bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
+	if forceNoTmpFile {
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
+	}
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
@@ -62,38 +67,12 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	// this is not supported.
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
-		if err != nil {
-			return nil, fmt.Errorf("make temp dir: %w", err)
-		}
-		f, err := os.CreateTemp(dir,
-			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-		if err != nil {
-			return nil, err
-		}
-		tmp := &tmpfile{
-			f:          f,
-			bucket:     bucket,
-			objname:    obj,
-			size:       size,
-			needsChown: doChown,
-			uid:        uid,
-			gid:        gid,
-		}
-		// falloc is best effort, its fine if this fails
-		if size > 0 && dofalloc {
-			tmp.falloc()
-		}
-
-		if doChown {
-			err := f.Chown(uid, gid)
-			if err != nil {
-				return nil, fmt.Errorf("set temp file ownership: %w", err)
-			}
-		}
-
-		return tmp, nil
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
@@ -127,6 +106,46 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	return tmp, nil
 }
 
+func (p *Posix) openMkTemp(dir, bucket, obj string, size int64, dofalloc bool, uid, gid int, doChown bool) (*tmpfile, error) {
+	err := backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, err
+	}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+	// falloc is best effort, its fine if this fails
+	if size > 0 && dofalloc {
+		tmp.falloc()
+	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
+	return tmp, nil
+}
+
 func (tmp *tmpfile) falloc() error {
 	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
 	if err != nil {
@@ -217,7 +236,9 @@ func (tmp *tmpfile) fallbackLink() error {
 	objPath := filepath.Join(tmp.bucket, tmp.objname)
 	err = os.Rename(tempname, objPath)
 	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
+		// rename only works for files within the same filesystem
+		// if this fails fallback to copy
+		return backend.MoveFile(tempname, objPath, fs.FileMode(defaultFilePerm))
 	}
 
 	return nil

backend/posix/without_otmpfile.go

@@ -24,9 +24,11 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"syscall"
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 )
 
 type tmpfile struct {
@@ -36,18 +38,24 @@ type tmpfile struct {
 	size    int64
 }
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool, _ bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
 	// Create a temp file for upload while in progress (see link comments below).
 	var err error
 	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("create temp file: %w", err)
 	}
 
@@ -72,31 +80,17 @@ func (tmp *tmpfile) link() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
-	// We use Rename as the atomic operation for object puts. The upload is
-	// written to a temp file to not conflict with any other simultaneous
-	// uploads. The final operation is to move the temp file into place for
-	// the object. This ensures the object semantics of last upload completed
-	// wins and is not some combination of writes from simultaneous uploads.
 	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err := os.Remove(objPath)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove stale path: %w", err)
-	}
 
 	// reset default file mode because CreateTemp uses 0600
 	tmp.f.Chmod(defaultFilePerm)
 
-	err = tmp.f.Close()
+	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)
 	}
 
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
+	return backend.MoveFile(tempname, objPath, defaultFilePerm)
 }
 
 func (tmp *tmpfile) Write(b []byte) (int, error) {

backend/s3proxy/client.go

@@ -36,6 +36,11 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
 	if s.endpoint != "" {
 		return s3.NewFromConfig(cfg, func(o *s3.Options) {
 			o.BaseEndpoint = &s.endpoint
+			o.UsePathStyle = s.usePathStyle
+			// The http body stream is not seekable, so most operations cannot
+			// be retried. The error returned to the original client may be
+			// retried by the client.
+			o.Retryer = aws.NopRetryer{}
 		}), nil
 	}
 

backend/scoutfs/scoutfs_compat.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 
 	"golang.org/x/sys/unix"
 
@@ -31,6 +32,7 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/s3err"
 )
 
 func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
@@ -52,14 +54,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:       p,
-		rootfd:      f,
-		rootdir:     rootdir,
-		meta:        metastore,
-		chownuid:    opts.ChownUID,
-		chowngid:    opts.ChownGID,
-		glaciermode: opts.GlacierMode,
-		newDirPerm:  opts.NewDirPerm,
+		Posix:            p,
+		rootfd:           f,
+		rootdir:          rootdir,
+		meta:             metastore,
+		chownuid:         opts.ChownUID,
+		chowngid:         opts.ChownGID,
+		glaciermode:      opts.GlacierMode,
+		newDirPerm:       opts.NewDirPerm,
+		disableNoArchive: opts.DisableNoArchive,
 	}, nil
 }
 
@@ -70,10 +73,10 @@ type tmpfile struct {
 	bucket     string
 	objname    string
 	size       int64
+	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
-	needsChown bool
 }
 
 var (
@@ -89,6 +92,9 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 	// file descriptor into the namespace.
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, err
 	}
 
@@ -149,10 +155,20 @@ func (tmp *tmpfile) link() error {
 	}
 	defer dirf.Close()
 
-	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-	if err != nil {
-		return fmt.Errorf("link tmpfile: %w", err)
+	for {
+		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		if errors.Is(err, fs.ErrExist) {
+			err := os.Remove(objPath)
+			if err != nil && !errors.Is(err, fs.ErrNotExist) {
+				return fmt.Errorf("remove stale path: %w", err)
+			}
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("link tmpfile: %w", err)
+		}
+		break
 	}
 
 	err = tmp.f.Close()

backend/walk.go

@@ -19,36 +19,71 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
-	"sort"
 	"strings"
+	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3response"
 )
 
 type WalkResults struct {
-	NextMarker     string
 	CommonPrefixes []types.CommonPrefix
 	Objects        []s3response.Object
 	Truncated      bool
+	NextMarker     string
 }
 
 type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
 
 var ErrSkipObj = errors.New("skip this object")
 
+// map to store object common prefixes
+type cpMap map[string]int
+
+func (c cpMap) Add(key string) {
+	_, ok := c[key]
+	if !ok {
+		c[key] = len(c)
+	}
+}
+
+// Len returns the length of the map
+func (c cpMap) Len() int {
+	return len(c)
+}
+
+// CpArray converts the map into a sorted []types.CommonPrefixes array
+func (c cpMap) CpArray() []types.CommonPrefix {
+	commonPrefixes := make([]types.CommonPrefix, c.Len())
+	for cp, i := range c {
+		pfx := cp
+		commonPrefixes[i] = types.CommonPrefix{
+			Prefix: &pfx,
+		}
+	}
+
+	return commonPrefixes
+}
+
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
 func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
-	cpmap := make(map[string]struct{})
+	cpmap := cpMap{}
 	var objects []s3response.Object
 
+	// if max is 0, it should return empty non-truncated result
+	if max == 0 {
+		return WalkResults{
+			Truncated: false,
+		}, nil
+	}
+
 	var pastMarker bool
 	if marker == "" {
 		pastMarker = true
 	}
 
-	pastMax := max == 0
+	var pastMax bool
 	var newMarker string
 	var truncated bool
 
@@ -75,14 +110,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipDir
 		}
 
-		if pastMax {
-			if len(objects) != 0 {
-				newMarker = *objects[len(objects)-1].Key
-				truncated = true
-			}
-			return fs.SkipAll
-		}
-
 		// After this point, return skipflag instead of nil
 		// so we can skip a directory without an early return
 		var skipflag error
@@ -107,13 +134,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
 				skipflag = fs.SkipDir
 			} else {
-				// TODO: can we do better here rather than a second readdir
-				// per directory?
-				ents, err := fs.ReadDir(fileSystem, path)
-				if err != nil {
-					return fmt.Errorf("readdir %q: %w", path, err)
-				}
-				if len(ents) == 0 && delimiter == "" {
+				if delimiter == "" {
 					dirobj, err := getObj(path+"/", d)
 					if err == ErrSkipObj {
 						return skipflag
@@ -121,11 +142,26 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 					if err != nil {
 						return fmt.Errorf("directory to object %q: %w", path, err)
 					}
+					if pastMax {
+						truncated = true
+						return fs.SkipAll
+					}
 					objects = append(objects, dirobj)
+					if (len(objects) + cpmap.Len()) == int(max) {
+						newMarker = path
+						pastMax = true
+					}
 
 					return skipflag
 				}
 
+				// TODO: can we do better here rather than a second readdir
+				// per directory?
+				ents, err := fs.ReadDir(fileSystem, path)
+				if err != nil {
+					return fmt.Errorf("readdir %q: %w", path, err)
+				}
+
 				if len(ents) != 0 {
 					return skipflag
 				}
@@ -158,9 +194,15 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
 			}
+			if pastMax {
+				truncated = true
+				return fs.SkipAll
+			}
+
 			objects = append(objects, obj)
 
-			if max > 0 && (len(objects)+len(cpmap)) == int(max) {
+			if (len(objects) + cpmap.Len()) == int(max) {
+				newMarker = path
 				pastMax = true
 			}
 
@@ -198,8 +240,13 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
 			}
+			if pastMax {
+				truncated = true
+				return fs.SkipAll
+			}
 			objects = append(objects, obj)
-			if (len(objects) + len(cpmap)) == int(max) {
+			if (len(objects) + cpmap.Len()) == int(max) {
+				newMarker = path
 				pastMax = true
 			}
 			return skipflag
@@ -220,38 +267,32 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return skipflag
 		}
 
-		cpmap[cpref] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
-			newMarker = cpref
+		if pastMax {
 			truncated = true
 			return fs.SkipAll
 		}
+		cpmap.Add(cpref)
+		if (len(objects) + cpmap.Len()) == int(max) {
+			newMarker = cpref
+			pastMax = true
+		}
 
 		return skipflag
 	})
 	if err != nil {
 		// suppress file not found caused by user's prefix
-		if errors.Is(err, fs.ErrNotExist) {
+		if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
 			return WalkResults{}, nil
 		}
 		return WalkResults{}, err
 	}
 
-	var commonPrefixStrings []string
-	for k := range cpmap {
-		commonPrefixStrings = append(commonPrefixStrings, k)
-	}
-	sort.Strings(commonPrefixStrings)
-	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
-	for _, cp := range commonPrefixStrings {
-		pfx := cp
-		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
-			Prefix: &pfx,
-		})
+	if !truncated {
+		newMarker = ""
 	}
 
 	return WalkResults{
-		CommonPrefixes: commonPrefixes,
+		CommonPrefixes: cpmap.CpArray(),
 		Objects:        objects,
 		Truncated:      truncated,
 		NextMarker:     newMarker,
@@ -268,18 +309,18 @@ func contains(a string, strs []string) bool {
 }
 
 type WalkVersioningResults struct {
-	NextMarker          string
-	NextVersionIdMarker string
 	CommonPrefixes      []types.CommonPrefix
-	ObjectVersions      []types.ObjectVersion
+	ObjectVersions      []s3response.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
 	Truncated           bool
+	NextMarker          string
+	NextVersionIdMarker string
 }
 
 type ObjVersionFuncResult struct {
-	NextVersionIdMarker string
-	ObjectVersions      []types.ObjectVersion
+	ObjectVersions      []s3response.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
+	NextVersionIdMarker string
 	Truncated           bool
 }
 
@@ -288,8 +329,8 @@ type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *boo
 // WalkVersions walks the supplied fs.FS and returns results compatible with
 // ListObjectVersions action response
 func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
-	cpmap := make(map[string]struct{})
-	var objects []types.ObjectVersion
+	cpmap := cpMap{}
+	var objects []s3response.ObjectVersion
 	var delMarkers []types.DeleteMarkerEntry
 
 	var pastMarker bool
@@ -344,11 +385,11 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 			if delimiter == "/" &&
 				prefix != path+"/" &&
 				strings.HasPrefix(path+"/", prefix) {
-				cpmap[path+"/"] = struct{}{}
+				cpmap.Add(path + "/")
 				return fs.SkipDir
 			}
 
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-cpmap.Len(), d)
 			if err == ErrSkipObj {
 				return nil
 			}
@@ -375,7 +416,7 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		if delimiter == "" {
 			// If no delimiter specified, then all files with matching
 			// prefix are included in results
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-cpmap.Len(), d)
 			if err == ErrSkipObj {
 				return nil
 			}
@@ -418,7 +459,7 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		suffix := strings.TrimPrefix(path, prefix)
 		before, _, found := strings.Cut(suffix, delimiter)
 		if !found {
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-cpmap.Len(), d)
 			if err == ErrSkipObj {
 				return nil
 			}
@@ -440,8 +481,8 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
 		// delimiter at the end.
-		cpmap[prefix+before+delimiter] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
+		cpmap.Add(prefix + before + delimiter)
+		if (len(objects) + cpmap.Len()) == int(max) {
 			nextMarker = path
 			truncated = true
 
@@ -454,21 +495,8 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM
 		return WalkVersioningResults{}, err
 	}
 
-	var commonPrefixStrings []string
-	for k := range cpmap {
-		commonPrefixStrings = append(commonPrefixStrings, k)
-	}
-	sort.Strings(commonPrefixStrings)
-	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
-	for _, cp := range commonPrefixStrings {
-		pfx := cp
-		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
-			Prefix: &pfx,
-		})
-	}
-
 	return WalkVersioningResults{
-		CommonPrefixes:      commonPrefixes,
+		CommonPrefixes:      cpmap.CpArray(),
 		ObjectVersions:      objects,
 		DelMarkers:          delMarkers,
 		Truncated:           truncated,

backend/walk_test.go

@@ -41,8 +41,8 @@ type testcase struct {
 	prefix    string
 	delimiter string
 	marker    string
-	expected  backend.WalkResults
 	maxObjs   int32
+	expected  backend.WalkResults
 }
 
 func getObj(path string, d fs.DirEntry) (s3response.Object, error) {

cmd/versitygw/admin.go

@@ -100,6 +100,11 @@ func adminCommand() *cli.Command {
 						Usage:   "secret access key for the new user",
 						Aliases: []string{"s"},
 					},
+					&cli.StringFlag{
+						Name:    "role",
+						Usage:   "the new user role",
+						Aliases: []string{"r"},
+					},
 					&cli.IntFlag{
 						Name:    "user-id",
 						Usage:   "userID for the new user",
@@ -311,8 +316,14 @@ func deleteUser(ctx *cli.Context) error {
 }
 
 func updateUser(ctx *cli.Context) error {
-	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
+	access, secret, userId, groupId, role := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id"), auth.Role(ctx.String("role"))
 	props := auth.MutableProps{}
+	if ctx.IsSet("role") {
+		if !role.IsValid() {
+			return fmt.Errorf("invalid user role: %v", role)
+		}
+		props.Role = role
+	}
 	if ctx.IsSet("secret") {
 		props.Secret = &secret
 	}

cmd/versitygw/main.go

@@ -50,6 +50,7 @@ var (
 	logWebhookURL, accessLog                 string
 	adminLogFile                             string
 	healthPath                               string
+	virtualDomain                            string
 	debug                                    bool
 	pprof                                    string
 	quiet                                    bool
@@ -74,6 +75,9 @@ var (
 	metricsService                           string
 	statsdServers                            string
 	dogstatsServers                          string
+	ipaHost, ipaVaultName                    string
+	ipaUser, ipaPassword                     string
+	ipaInsecure, ipaDebug                    bool
 )
 
 var (
@@ -95,6 +99,7 @@ func main() {
 		scoutfsCommand(),
 		s3Command(),
 		azureCommand(),
+		pluginCommand(),
 		adminCommand(),
 		testCommand(),
 		utilsCommand(),
@@ -206,6 +211,7 @@ func initFlags() []cli.Flag {
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
+			Value:       false,
 			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
@@ -222,6 +228,13 @@ func initFlags() []cli.Flag {
 			Destination: &quiet,
 			Aliases:     []string{"q"},
 		},
+		&cli.StringFlag{
+			Name:        "virtual-domain",
+			Usage:       "enables the virtual host style bucket addressing with the specified arg as the base domain",
+			EnvVars:     []string{"VGW_VIRTUAL_DOMAIN"},
+			Destination: &virtualDomain,
+			Aliases:     []string{"vd"},
+		},
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
@@ -506,6 +519,42 @@ func initFlags() []cli.Flag {
 			Aliases:     []string{"mds"},
 			Destination: &dogstatsServers,
 		},
+		&cli.StringFlag{
+			Name:        "ipa-host",
+			Usage:       "FreeIPA server url e.g. https://ipa.example.test",
+			EnvVars:     []string{"VGW_IPA_HOST"},
+			Destination: &ipaHost,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-vault-name",
+			Usage:       "A name of the user vault containing their secret",
+			EnvVars:     []string{"VGW_IPA_VAULT_NAME"},
+			Destination: &ipaVaultName,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-user",
+			Usage:       "Username used to connect to FreeIPA (requires permissions to read user vault contents)",
+			EnvVars:     []string{"VGW_IPA_USER"},
+			Destination: &ipaUser,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-password",
+			Usage:       "Password of the user used to connect to FreeIPA",
+			EnvVars:     []string{"VGW_IPA_PASSWORD"},
+			Destination: &ipaPassword,
+		},
+		&cli.BoolFlag{
+			Name:        "ipa-insecure",
+			Usage:       "Disable verify TLS certificate of FreeIPA server",
+			EnvVars:     []string{"VGW_IPA_INSECURE"},
+			Destination: &ipaInsecure,
+		},
+		&cli.BoolFlag{
+			Name:        "ipa-debug",
+			Usage:       "FreeIPA IAM debug output",
+			EnvVars:     []string{"VGW_IPA_DEBUG"},
+			Destination: &ipaDebug,
+		},
 	}
 }
 
@@ -562,6 +611,9 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if readonly {
 		opts = append(opts, s3api.WithReadOnly())
 	}
+	if virtualDomain != "" {
+		opts = append(opts, s3api.WithHostStyle(virtualDomain))
+	}
 
 	admApp := fiber.New(fiber.Config{
 		AppName:               "versitygw",
@@ -623,6 +675,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		CacheDisable:           iamCacheDisable,
 		CacheTTL:               iamCacheTTL,
 		CachePrune:             iamCachePrune,
+		IpaHost:                ipaHost,
+		IpaVaultName:           ipaVaultName,
+		IpaUser:                ipaUser,
+		IpaPassword:            ipaPassword,
+		IpaInsecure:            ipaInsecure,
+		IpaDebug:               ipaDebug,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)

cmd/versitygw/plugin.go

@@ -0,0 +1,74 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"plugin"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/plugins"
+)
+
+func pluginCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "plugin",
+		Usage:       "load a backend from a plugin",
+		Description: "Runs a s3 gateway and redirects the requests to the backend defined in the plugin",
+		Action:      runPluginBackend,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "config",
+				Usage:   "location of the config file",
+				Aliases: []string{"c"},
+			},
+		},
+	}
+}
+
+func runPluginBackend(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no plugin file provided to be loaded")
+	}
+
+	pluginPath := ctx.Args().Get(0)
+	config := ctx.String("config")
+
+	p, err := plugin.Open(pluginPath)
+	if err != nil {
+		return err
+	}
+
+	backendSymbol, err := p.Lookup("Backend")
+	if err != nil {
+		return err
+	}
+	backendPluginPtr, ok := backendSymbol.(*plugins.BackendPlugin)
+	if !ok {
+		return errors.New("plugin is not of type *plugins.BackendPlugin")
+	}
+
+	if backendPluginPtr == nil {
+		return errors.New("variable Backend is nil")
+	}
+
+	be, err := (*backendPluginPtr).New(config)
+	if err != nil {
+		return err
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/posix.go

@@ -29,6 +29,9 @@ var (
 	bucketlinks        bool
 	versioningDir      string
 	dirPerms           uint
+	sidecar            string
+	nometa             bool
+	forceNoTmpFile     bool
 )
 
 func posixCommand() *cli.Command {
@@ -79,6 +82,24 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.StringFlag{
+				Name:        "sidecar",
+				Usage:       "use provided sidecar directory to store metadata",
+				EnvVars:     []string{"VGW_META_SIDECAR"},
+				Destination: &sidecar,
+			},
+			&cli.BoolFlag{
+				Name:        "nometa",
+				Usage:       "disable metadata storage",
+				EnvVars:     []string{"VGW_META_NONE"},
+				Destination: &nometa,
+			},
+			&cli.BoolFlag{
+				Name:        "disableotmp",
+				Usage:       "disable O_TMPFILE support for new objects",
+				EnvVars:     []string{"VGW_DISABLE_OTMP"},
+				Destination: &forceNoTmpFile,
+			},
 		},
 	}
 }
@@ -89,24 +110,46 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	gwroot := (ctx.Args().Get(0))
-	err := meta.XattrMeta{}.Test(gwroot)
-	if err != nil {
-		return fmt.Errorf("posix xattr check: %v", err)
-	}
 
 	if dirPerms > math.MaxUint32 {
 		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
 	}
 
-	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
-		ChownUID:      chownuid,
-		ChownGID:      chowngid,
-		BucketLinks:   bucketlinks,
-		VersioningDir: versioningDir,
-		NewDirPerm:    fs.FileMode(dirPerms),
-	})
+	if nometa && sidecar != "" {
+		return fmt.Errorf("cannot use both nometa and sidecar metadata")
+	}
+
+	opts := posix.PosixOpts{
+		ChownUID:       chownuid,
+		ChownGID:       chowngid,
+		BucketLinks:    bucketlinks,
+		VersioningDir:  versioningDir,
+		NewDirPerm:     fs.FileMode(dirPerms),
+		ForceNoTmpFile: forceNoTmpFile,
+	}
+
+	var ms meta.MetadataStorer
+	switch {
+	case sidecar != "":
+		sc, err := meta.NewSideCar(sidecar)
+		if err != nil {
+			return fmt.Errorf("failed to init sidecar metadata: %w", err)
+		}
+		ms = sc
+		opts.SideCarDir = sidecar
+	case nometa:
+		ms = meta.NoMeta{}
+	default:
+		ms = meta.XattrMeta{}
+		err := meta.XattrMeta{}.Test(gwroot)
+		if err != nil {
+			return fmt.Errorf("xattr check failed: %w", err)
+		}
+	}
+
+	be, err := posix.New(gwroot, ms, opts)
 	if err != nil {
-		return fmt.Errorf("init posix: %v", err)
+		return fmt.Errorf("failed to init posix backend: %w", err)
 	}
 
 	return runGateway(ctx.Context, be)

cmd/versitygw/s3.go

@@ -26,8 +26,10 @@ var (
 	s3proxySecret          string
 	s3proxyEndpoint        string
 	s3proxyRegion          string
+	s3proxyMetaBucket      string
 	s3proxyDisableChecksum bool
 	s3proxySslSkipVerify   bool
+	s3proxyUsePathStyle    bool
 	s3proxyDebug           bool
 )
 
@@ -71,6 +73,12 @@ to an s3 storage backend service.`,
 				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
+			&cli.StringFlag{
+				Name:        "meta-bucket",
+				Usage:       "s3 service meta bucket to store buckets acl/policy",
+				EnvVars:     []string{"VGW_S3_META_BUCKET"},
+				Destination: &s3proxyMetaBucket,
+			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
@@ -85,6 +93,13 @@ to an s3 storage backend service.`,
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
+			&cli.BoolFlag{
+				Name:        "use-path-style",
+				Usage:       "use path style addressing for s3 proxy",
+				EnvVars:     []string{"VGW_S3_USE_PATH_STYLE"},
+				Value:       false,
+				Destination: &s3proxyUsePathStyle,
+			},
 			&cli.BoolFlag{
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
@@ -97,8 +112,8 @@ to an s3 storage backend service.`,
 }
 
 func runS3(ctx *cli.Context) error {
-	be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
-		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
 	if err != nil {
 		return fmt.Errorf("init s3 backend: %w", err)
 	}

cmd/versitygw/scoutfs.go

@@ -24,7 +24,8 @@ import (
 )
 
 var (
-	glacier bool
+	glacier          bool
+	disableNoArchive bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -79,6 +80,12 @@ move interfaces as well as support for tiered filesystems.`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.BoolFlag{
+				Name:        "disable-noarchive",
+				Usage:       "disable setting noarchive for multipart part uploads",
+				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
+				Destination: &disableNoArchive,
+			},
 		},
 	}
 }
@@ -98,6 +105,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
 	opts.NewDirPerm = fs.FileMode(dirPerms)
+	opts.DisableNoArchive = disableNoArchive
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

cmd/versitygw/test.go

@@ -34,10 +34,11 @@ var (
 	totalReqs         int
 	upload            bool
 	download          bool
-	pathStyle         bool
+	hostStyle         bool
 	checksumDisable   bool
 	versioningEnabled bool
 	azureTests        bool
+	tlsStatus         bool
 )
 
 func testCommand() *cli.Command {
@@ -73,12 +74,24 @@ func initTestFlags() []cli.Flag {
 			Destination: &endpoint,
 			Aliases:     []string{"e"},
 		},
+		&cli.BoolFlag{
+			Name:        "host-style",
+			Usage:       "Use host-style bucket addressing",
+			Value:       false,
+			Destination: &hostStyle,
+		},
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug mode",
 			Aliases:     []string{"d"},
 			Destination: &debug,
 		},
+		&cli.BoolFlag{
+			Name:        "allow-insecure",
+			Usage:       "skip tls verification",
+			Aliases:     []string{"ai"},
+			Destination: &tlsStatus,
+		},
 	}
 }
 
@@ -117,6 +130,11 @@ func initTestCommands() []*cli.Command {
 				},
 			},
 		},
+		{
+			Name:   "scoutfs",
+			Usage:  "Tests scoutfs full flow",
+			Action: getAction(integration.TestScoutfs),
+		},
 		{
 			Name:   "iam",
 			Usage:  "Tests iam service",
@@ -179,12 +197,6 @@ func initTestCommands() []*cli.Command {
 					Value:       1,
 					Destination: &concurrency,
 				},
-				&cli.BoolFlag{
-					Name:        "pathStyle",
-					Usage:       "Use Pathstyle bucket addressing",
-					Value:       false,
-					Destination: &pathStyle,
-				},
 				&cli.BoolFlag{
 					Name:        "checksumDis",
 					Usage:       "Disable server checksum",
@@ -211,12 +223,13 @@ func initTestCommands() []*cli.Command {
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
 					integration.WithPartSize(partSize),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
-				if pathStyle {
-					opts = append(opts, integration.WithPathStyle())
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
 				}
 				if checksumDisable {
 					opts = append(opts, integration.WithDisableChecksum())
@@ -271,6 +284,7 @@ func initTestCommands() []*cli.Command {
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -278,6 +292,9 @@ func initTestCommands() []*cli.Command {
 				if checksumDisable {
 					opts = append(opts, integration.WithDisableChecksum())
 				}
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
+				}
 
 				s3conf := integration.NewS3Conf(opts...)
 
@@ -296,6 +313,7 @@ func getAction(tf testFunc) func(*cli.Context) error {
 			integration.WithSecret(awsSecret),
 			integration.WithRegion(region),
 			integration.WithEndpoint(endpoint),
+			integration.WithTLSStatus(tlsStatus),
 		}
 		if debug {
 			opts = append(opts, integration.WithDebug())
@@ -306,6 +324,9 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if azureTests {
 			opts = append(opts, integration.WithAzureMode())
 		}
+		if hostStyle {
+			opts = append(opts, integration.WithHostStyle())
+		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -333,6 +354,7 @@ func extractIntTests() (commands []*cli.Command) {
 					integration.WithSecret(awsSecret),
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -340,6 +362,9 @@ func extractIntTests() (commands []*cli.Command) {
 				if versioningEnabled {
 					opts = append(opts, integration.WithVersioningEnabled())
 				}
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
+				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)

extra/example.conf

@@ -99,6 +99,26 @@ ROOT_SECRET_ACCESS_KEY=
 # endpoint is unauthenticated, and returns a 200 status for GET.
 #VGW_HEALTH=
 
+# Enable VGW_READ_ONLY to only allow read operations to the S3 server. No write
+# operations will be allowed.
+#VGW_READ_ONLY=false
+
+# The VGW_VIRTUAL_DOMAIN option enables the virtual host style bucket
+# addressing. The path style addressing is the default, and remains enabled
+# even when virtual host style is enabled. The VGW_VIRTUAL_DOMAIN option
+# specifies the domain name that will be used for the virtual host style
+# addressing. For virtual addressing, access to a bucket is in the request
+# form:
+# https://<bucket>.<VGW_VIRTUAL_DOMAIN>/
+# for example: https://mybucket.example.com/ where
+# VGW_VIRTUAL_DOMAIN=example.com
+# and all subdomains of VGW_VIRTUAL_DOMAIN should be reserved for buckets.
+# This means that virtual host addressing will generally require a DNS
+# entry for each bucket that needs to be accessed.
+# The default path style request is of the form:
+# https://<VGW_ENDPOINT>/<bucket>
+#VGW_VIRTUAL_DOMAIN=
+
 ###############
 # Access Logs #
 ###############
@@ -240,6 +260,24 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_LDAP_USER_ID_ATR=
 #VGW_IAM_LDAP_GROUP_ID_ATR=
 
+# The FreeIPA options will enable the FreeIPA IAM service with accounts stored
+# in an external FreeIPA service. Currently the FreeIPA IAM service only
+# supports account retrieval. Creating and modifying accounts must be done
+# outside of the versitygw service.
+# FreeIPA server url e.g. https://ipa.example.test
+#VGW_IPA_HOST=
+# A name of the user vault containing their secret
+#VGW_IPA_VAULT_NAME=
+# Username used to connect to FreeIPA (requires permissions to read user vault
+# contents)
+#VGW_IPA_USER=
+# Password of the user used to connect to FreeIPA
+#VGW_IPA_PASSWORD=
+# Disable verify TLS certificate of FreeIPA server
+#VGW_IPA_INSECURE=false
+# FreeIPA IAM debug output
+#VGW_IPA_DEBUG=false
+
 ###############
 # IAM caching #
 ###############
@@ -317,6 +355,40 @@ ROOT_SECRET_ACCESS_KEY=
 # as any parent directories automatically created with object uploads.
 #VGW_DIR_PERMS=0755
 
+# To enable object versions, the VGW_VERSIONING_DIR option must be set to the
+# directory that will be used to store the object versions. The version
+# directory must NOT be a subdirectory of the VGW_BACKEND_ARG directory.
+#VGW_VERSIONING_DIR=
+
+# The gateway uses xattrs to store metadata for objects by default. For systems
+# that do not support xattrs, the VGW_META_SIDECAR option can be set to a
+# directory that will be used to store the metadata for objects. This is
+# currently experimental, and may have issues for some edge cases.
+#VGW_META_SIDECAR=
+
+# The VGW_META_NONE option will disable the metadata functionality for the
+# gateway. This will cause the gateway to not store any metadata for objects
+# or buckets. This include bucket ACLs and Policy. This may be useful for
+# read only access to pre-existing data where the gateway should not modify
+# the data. It is recommened to enable VGW_READ_ONLY (Global Options) along
+# with this.
+#VGW_META_NONE=false
+
+# The gateway will use O_TMPFILE for writing objects while uploading and
+# link the file to the final object name when the upload is complete if the
+# filesystem supports O_TMPFILE. This creates an atomic object creation
+# that is not visible to other clients or racing uploads until the upload
+# is complete. This will not work if there is a different filesystem mounted
+# below the bucket level than where the bucket resides. The VGW_DISABLE_OTMP
+# option can be set to true to disable this functionality and force the fallback
+# mode when O_TMPFILE is not available. This fallback will create a temporary
+# file in the bucket directory and rename it to the final object name when
+# the upload is complete if the final location is in the same filesystem, or
+# copy the file to the final location if the final location is in a different
+# filesystem. This fallback mode is still atomic, but may be less efficient
+# than O_TMPFILE when the data needs to be copied into the final location.
+#VGW_DISABLE_OTMP=false
+
 ###########
 # scoutfs #
 ###########
@@ -358,6 +430,13 @@ ROOT_SECRET_ACCESS_KEY=
 # as any parent directories automatically created with object uploads.
 #VGW_DIR_PERMS=0755
 
+# The default behavior of the gateway is to automatically set the noarchive
+# flag on the multipart upload parts while the multipart upload is in progress.
+# This is to prevent the parts from being archived since they are temporary
+# and will be deleted after the multipart upload is completed or aborted. The
+# VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
+#VGW_DISABLE_NOARCHIVE=false
+
 ######
 # s3 #
 ######

go.mod

@@ -1,82 +1,83 @@
 module github.com/versity/versitygw
 
-go 1.21.0
+go 1.23.0
+
+toolchain go1.24.1
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
-	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.32.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
-	github.com/aws/smithy-go v1.22.0
-	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/gofiber/fiber/v2 v2.52.5
-	github.com/google/go-cmp v0.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
+	github.com/DataDog/datadog-go/v5 v5.6.0
+	github.com/aws/aws-sdk-go-v2 v1.36.5
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0
+	github.com/aws/smithy-go v1.22.4
+	github.com/go-ldap/ldap/v3 v3.4.11
+	github.com/gofiber/fiber/v2 v2.52.8
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.37.0
-	github.com/oklog/ulid/v2 v2.1.0
-	github.com/pkg/xattr v0.4.10
-	github.com/segmentio/kafka-go v0.4.47
+	github.com/nats-io/nats.go v1.43.0
+	github.com/oklog/ulid/v2 v2.1.1
+	github.com/pkg/xattr v0.4.12
+	github.com/segmentio/kafka-go v0.4.48
 	github.com/smira/go-statsd v1.3.4
-	github.com/urfave/cli/v2 v2.27.5
-	github.com/valyala/fasthttp v1.57.0
+	github.com/urfave/cli/v2 v2.27.7
+	github.com/valyala/fasthttp v1.62.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.26.0
+	golang.org/x/sync v0.15.0
+	golang.org/x/sys v0.33.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nkeys v0.4.11 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.28.1
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.42
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.17
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.70
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/tcplisten v1.0.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 )

go.sum

@@ -1,72 +1,72 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
-github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
+github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
-github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
-github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw=
-github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35 h1:ihPPdcCVSN0IvBByXwqVp28/l4VosBZ6sDulcvU2J7w=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35/go.mod h1:JkgEhs3SVF51Dj3m1Bj+yL8IznpxzkwlA3jLg3x7Kls=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58=
-github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE=
-github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
-github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0=
+github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY=
+github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0=
+github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 h1:EO13QJTCD1Ig2IrQnoHTRrn981H9mB7afXsZ89WptI4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82/go.mod h1:AGh1NCg0SH+uyJamiJA5tTQcql4MMRDXGRdMmCxCXzY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 h1:JubM8CGDDFaAOmBrd8CRYNr49ZNgEAiLwGwgNMdS0nw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E=
+github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w=
+github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw=
+github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
-github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -74,33 +74,29 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
-github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
-github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
-github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU=
+github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM=
+github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4=
+github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
+github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
+github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
-github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
-github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
+github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
+github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
@@ -117,43 +113,42 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
+github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
+github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
-github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
-github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
-github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
+github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
+github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
+github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
-github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
+github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
+github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
 github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
-github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
-github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
+github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
+github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
+github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -161,8 +156,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
-github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
+github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs=
+github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
 github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
@@ -171,20 +166,17 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
-github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
-github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE=
-github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
-github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/valyala/fasthttp v1.62.0 h1:8dKRBX/y2rCzyc6903Zu1+3qN0H/d2MsxPPmVNamiH0=
+github.com/valyala/fasthttp v1.62.0/go.mod h1:FCINgr4GKdKqV8Q0xv8b+UxPV+H/O5nNFo3D+r54Htg=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -202,35 +194,28 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -242,23 +227,18 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -266,11 +246,10 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
-golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
@@ -280,7 +259,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

metrics/actions.go

@@ -72,6 +72,9 @@ var (
 	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
 	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
 	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
+	ActionPutBucketCors                 = "s3_PutBucketCors"
+	ActionGetBucketCors                 = "s3_GetBucketCors"
+	ActionDeleteBucketCors              = "s3_DeleteBucketCors"
 
 	// Admin actions
 	ActionAdminCreateUser        = "admin_CreateUser"
@@ -266,4 +269,16 @@ func init() {
 		Name:    "UploadPartCopy",
 		Service: "s3",
 	}
+	ActionMap[ActionPutBucketCors] = Action{
+		Name:    "PutBucketCors",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketCors] = Action{
+		Name:    "GetBucketCors",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucketCors] = Action{
+		Name:    "DeleteBucketCors",
+		Service: "s3",
+	}
 }

metrics/metrics.go

@@ -43,14 +43,13 @@ type Tag struct {
 
 // Manager is a manager of metrics plugins
 type Manager struct {
+	wg  sync.WaitGroup
 	ctx context.Context
 
-	addDataChan chan datapoint
-
 	config Config
 
-	publishers []publisher
-	wg         sync.WaitGroup
+	publishers  []publisher
+	addDataChan chan datapoint
 }
 
 type Config struct {
@@ -221,6 +220,6 @@ func (m *Manager) addForwarder(addChan <-chan datapoint) {
 
 type datapoint struct {
 	key   string
-	tags  []Tag
 	value int64
+	tags  []Tag
 }

plugins/plugins.go

@@ -0,0 +1,35 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package plugins
+
+import "github.com/versity/versitygw/backend"
+
+// BackendPlugin defines an interface for creating backend
+// implementation instances.
+// Plugins implementing this interface can be built as shared
+// libraries using Go's plugin system (to build use `go build -buildmode=plugin`).
+// The shared library should export an instance of
+// this interface in a variable named `Backend`.
+type BackendPlugin interface {
+	// New creates and initializes a new backend.Backend instance.
+	// The config parameter specifies the path of the file containing
+	// the configuration for the backend.
+	//
+	// Implementations of this method should perform the necessary steps to
+	// establish a connection to the underlying storage system or service
+	// (e.g., network storage system, distributed storage system, cloud storage)
+	//  and configure it according to the provided configuration.
+	New(config string) (backend.Backend, error)
+}

runtests.sh

@@ -10,6 +10,14 @@ mkdir /tmp/versioning.covdata
 rm -rf /tmp/versioningdir
 mkdir /tmp/versioningdir
 
+# setup tls certificate and key
+ECHO "Generating TLS certificate and key in the cert.pem and key.pem files"
+
+openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
+openssl req -new -x509 -key key.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+
+ECHO "Running the sdk test over http"
 # run server in background not versioning-enabled
 # port: 7070(default)
 GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
@@ -17,7 +25,7 @@ GW_PID=$!
 
 sleep 1
 
-# check if versioning-enabled gateway process is still running
+# check if gateway process is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -45,9 +53,48 @@ fi
 
 kill $GW_PID
 
+ECHO "Running the sdk test over https"
+
+# run server in background with TLS certificate
+# port: 7071(default)
+GOCOVERDIR=/tmp/https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7071 -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
+GW_HTTPS_PID=$!
+
+sleep 1
+
+# check if https gateway process is still running
+if ! kill -0 $GW_HTTPS_PID; then
+	echo "server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow; then
+	echo "full flow tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 posix; then
+	echo "posix tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+# iam tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 iam; then
+	echo "iam tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+
+kill $GW_HTTPS_PID
+
+
+ECHO "Running the sdk test over http against the versioning-enabled gateway"
 # run server in background versioning-enabled
-# port: 7071
-GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7071 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+# port: 7072
+GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7072 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
 GW_VS_PID=$!
 
 # wait a second for server to start up
@@ -61,13 +108,13 @@ fi
 
 # run tests
 # full flow tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 full-flow -vs; then
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs; then
 	echo "versioning-enabled full-flow tests failed"
 	kill $GW_VS_PID
 	exit 1
 fi
 # posix tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 posix -vs; then
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 posix -vs; then
 	echo "versiongin-enabled posix tests failed"
 	kill $GW_VS_PID
 	exit 1
@@ -76,6 +123,38 @@ fi
 # kill off server
 kill $GW_VS_PID
 
+ECHO "Running the sdk test over https against the versioning-enabled gateway"
+# run server in background versioning-enabled
+# port: 7073
+GOCOVERDIR=/tmp/versioning.https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7073 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+GW_VS_HTTPS_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if versioning-enabled gateway process is still running
+if ! kill -0 $GW_VS_HTTPS_PID; then
+	echo "versioning-enabled server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs; then
+	echo "versioning-enabled full-flow tests failed"
+	kill $GW_VS_HTTPS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 posix -vs; then
+	echo "versiongin-enabled posix tests failed"
+	kill $GW_VS_HTTPS_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_VS_HTTPS_PID
+
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-server.go

@@ -26,11 +26,11 @@ import (
 )
 
 type S3AdminServer struct {
-	backend backend.Backend
 	app     *fiber.App
+	backend backend.Backend
 	router  *S3AdminRouter
-	cert    *tls.Certificate
 	port    string
+	cert    *tls.Certificate
 }
 
 func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {

s3api/controllers/admin.go

@@ -100,7 +100,16 @@ func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := c.iam.UpdateUserAccount(access, props)
+	err := props.Validate()
+	if err != nil {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
+	}
+
+	err = c.iam.UpdateUserAccount(access, props)
 	if err != nil {
 		if strings.Contains(err.Error(), "user not found") {
 			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
@@ -167,7 +176,7 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		Owner: owner,
 		Grantees: []auth.Grantee{
 			{
-				Permission: types.PermissionFullControl,
+				Permission: auth.PermissionFullControl,
 				Access:     owner,
 				Type:       types.TypeCanonicalUser,
 			},

s3api/controllers/admin_test.go

@@ -64,11 +64,11 @@ func TestAdminController_CreateUser(t *testing.T) {
 	`
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-create-user-malformed-body",
@@ -149,11 +149,11 @@ func TestAdminController_UpdateUser(t *testing.T) {
 	`
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-update-user-success",
@@ -223,11 +223,11 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	app.Patch("/delete-user", adminController.DeleteUser)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-delete-user-success",
@@ -280,11 +280,11 @@ func TestAdminController_ListUsers(t *testing.T) {
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-list-users-iam-error",
@@ -361,11 +361,11 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -424,11 +424,11 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "List-buckets-success",

s3api/controllers/backend_moq_test.go

@@ -29,21 +29,24 @@ var _ backend.Backend = &BackendMock{}
 //			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
 //				panic("mock out the ChangeBucketOwner method")
 //			},
-//			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+//			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
 //				panic("mock out the CompleteMultipartUpload method")
 //			},
-//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
 //				panic("mock out the CopyObject method")
 //			},
 //			CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
 //				panic("mock out the CreateBucket method")
 //			},
-//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 //				panic("mock out the CreateMultipartUpload method")
 //			},
 //			DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketCorsFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketCors method")
+//			},
 //			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketOwnershipControls method")
 //			},
@@ -65,6 +68,9 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketCorsFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+//				panic("mock out the GetBucketCors method")
+//			},
 //			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
 //				panic("mock out the GetBucketOwnershipControls method")
 //			},
@@ -128,6 +134,9 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketCorsFunc: func(contextMoqParam context.Context, bytes []byte) error {
+//				panic("mock out the PutBucketCors method")
+//			},
 //			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
 //				panic("mock out the PutBucketOwnershipControls method")
 //			},
@@ -140,7 +149,7 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 //				panic("mock out the PutBucketVersioning method")
 //			},
-//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 //				panic("mock out the PutObject method")
 //			},
 //			PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error {
@@ -170,10 +179,10 @@ var _ backend.Backend = &BackendMock{}
 //			StringFunc: func() string {
 //				panic("mock out the String method")
 //			},
-//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 //				panic("mock out the UploadPart method")
 //			},
-//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 //				panic("mock out the UploadPartCopy method")
 //			},
 //		}
@@ -190,20 +199,23 @@ type BackendMock struct {
 	ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, acl []byte) error
 
 	// CompleteMultipartUploadFunc mocks the CompleteMultipartUpload method.
-	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error)
 
 	// CopyObjectFunc mocks the CopyObject method.
-	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (s3response.CopyObjectOutput, error)
 
 	// CreateBucketFunc mocks the CreateBucket method.
 	CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error
 
 	// CreateMultipartUploadFunc mocks the CreateMultipartUpload method.
-	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
+	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error
 
+	// DeleteBucketCorsFunc mocks the DeleteBucketCors method.
+	DeleteBucketCorsFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
 	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -225,6 +237,9 @@ type BackendMock struct {
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketCorsFunc mocks the GetBucketCors method.
+	GetBucketCorsFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
+
 	// GetBucketOwnershipControlsFunc mocks the GetBucketOwnershipControls method.
 	GetBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error)
 
@@ -288,6 +303,9 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketCorsFunc mocks the PutBucketCors method.
+	PutBucketCorsFunc func(contextMoqParam context.Context, bytes []byte) error
+
 	// PutBucketOwnershipControlsFunc mocks the PutBucketOwnershipControls method.
 	PutBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error
 
@@ -301,7 +319,7 @@ type BackendMock struct {
 	PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error
 
 	// PutObjectFunc mocks the PutObject method.
-	PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error)
+	PutObjectFunc func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error)
 
 	// PutObjectAclFunc mocks the PutObjectAcl method.
 	PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error
@@ -331,10 +349,10 @@ type BackendMock struct {
 	StringFunc func() string
 
 	// UploadPartFunc mocks the UploadPart method.
-	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error)
+	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error)
 
 	// UploadPartCopyFunc mocks the UploadPartCopy method.
-	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// calls tracks calls to the methods.
 	calls struct {
@@ -366,7 +384,7 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// CopyObjectInput is the copyObjectInput argument value.
-			CopyObjectInput *s3.CopyObjectInput
+			CopyObjectInput s3response.CopyObjectInput
 		}
 		// CreateBucket holds details about calls to the CreateBucket method.
 		CreateBucket []struct {
@@ -382,7 +400,7 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// CreateMultipartUploadInput is the createMultipartUploadInput argument value.
-			CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+			CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 		}
 		// DeleteBucket holds details about calls to the DeleteBucket method.
 		DeleteBucket []struct {
@@ -391,6 +409,13 @@ type BackendMock struct {
 			// Bucket is the bucket argument value.
 			Bucket string
 		}
+		// DeleteBucketCors holds details about calls to the DeleteBucketCors method.
+		DeleteBucketCors []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
 		DeleteBucketOwnershipControls []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -442,6 +467,13 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketCors holds details about calls to the GetBucketCors method.
+		GetBucketCors []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetBucketOwnershipControls holds details about calls to the GetBucketOwnershipControls method.
 		GetBucketOwnershipControls []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -599,6 +631,13 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketCors holds details about calls to the PutBucketCors method.
+		PutBucketCors []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bytes is the bytes argument value.
+			Bytes []byte
+		}
 		// PutBucketOwnershipControls holds details about calls to the PutBucketOwnershipControls method.
 		PutBucketOwnershipControls []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -640,7 +679,7 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// PutObjectInput is the putObjectInput argument value.
-			PutObjectInput *s3.PutObjectInput
+			PutObjectInput s3response.PutObjectInput
 		}
 		// PutObjectAcl holds details about calls to the PutObjectAcl method.
 		PutObjectAcl []struct {
@@ -739,6 +778,7 @@ type BackendMock struct {
 	lockCreateBucket                  sync.RWMutex
 	lockCreateMultipartUpload         sync.RWMutex
 	lockDeleteBucket                  sync.RWMutex
+	lockDeleteBucketCors              sync.RWMutex
 	lockDeleteBucketOwnershipControls sync.RWMutex
 	lockDeleteBucketPolicy            sync.RWMutex
 	lockDeleteBucketTagging           sync.RWMutex
@@ -746,6 +786,7 @@ type BackendMock struct {
 	lockDeleteObjectTagging           sync.RWMutex
 	lockDeleteObjects                 sync.RWMutex
 	lockGetBucketAcl                  sync.RWMutex
+	lockGetBucketCors                 sync.RWMutex
 	lockGetBucketOwnershipControls    sync.RWMutex
 	lockGetBucketPolicy               sync.RWMutex
 	lockGetBucketTagging              sync.RWMutex
@@ -767,6 +808,7 @@ type BackendMock struct {
 	lockListObjectsV2                 sync.RWMutex
 	lockListParts                     sync.RWMutex
 	lockPutBucketAcl                  sync.RWMutex
+	lockPutBucketCors                 sync.RWMutex
 	lockPutBucketOwnershipControls    sync.RWMutex
 	lockPutBucketPolicy               sync.RWMutex
 	lockPutBucketTagging              sync.RWMutex
@@ -862,7 +904,7 @@ func (mock *BackendMock) ChangeBucketOwnerCalls() []struct {
 }
 
 // CompleteMultipartUpload calls CompleteMultipartUploadFunc.
-func (mock *BackendMock) CompleteMultipartUpload(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (mock *BackendMock) CompleteMultipartUpload(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
 	if mock.CompleteMultipartUploadFunc == nil {
 		panic("BackendMock.CompleteMultipartUploadFunc: method is nil but Backend.CompleteMultipartUpload was just called")
 	}
@@ -898,13 +940,13 @@ func (mock *BackendMock) CompleteMultipartUploadCalls() []struct {
 }
 
 // CopyObject calls CopyObjectFunc.
-func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
 	if mock.CopyObjectFunc == nil {
 		panic("BackendMock.CopyObjectFunc: method is nil but Backend.CopyObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
-		CopyObjectInput *s3.CopyObjectInput
+		CopyObjectInput s3response.CopyObjectInput
 	}{
 		ContextMoqParam: contextMoqParam,
 		CopyObjectInput: copyObjectInput,
@@ -921,11 +963,11 @@ func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectI
 //	len(mockedBackend.CopyObjectCalls())
 func (mock *BackendMock) CopyObjectCalls() []struct {
 	ContextMoqParam context.Context
-	CopyObjectInput *s3.CopyObjectInput
+	CopyObjectInput s3response.CopyObjectInput
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
-		CopyObjectInput *s3.CopyObjectInput
+		CopyObjectInput s3response.CopyObjectInput
 	}
 	mock.lockCopyObject.RLock()
 	calls = mock.calls.CopyObject
@@ -974,13 +1016,13 @@ func (mock *BackendMock) CreateBucketCalls() []struct {
 }
 
 // CreateMultipartUpload calls CreateMultipartUploadFunc.
-func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	if mock.CreateMultipartUploadFunc == nil {
 		panic("BackendMock.CreateMultipartUploadFunc: method is nil but Backend.CreateMultipartUpload was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam            context.Context
-		CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+		CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 	}{
 		ContextMoqParam:            contextMoqParam,
 		CreateMultipartUploadInput: createMultipartUploadInput,
@@ -997,11 +1039,11 @@ func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context,
 //	len(mockedBackend.CreateMultipartUploadCalls())
 func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
 	ContextMoqParam            context.Context
-	CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+	CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 } {
 	var calls []struct {
 		ContextMoqParam            context.Context
-		CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+		CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 	}
 	mock.lockCreateMultipartUpload.RLock()
 	calls = mock.calls.CreateMultipartUpload
@@ -1045,6 +1087,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketCors calls DeleteBucketCorsFunc.
+func (mock *BackendMock) DeleteBucketCors(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketCorsFunc == nil {
+		panic("BackendMock.DeleteBucketCorsFunc: method is nil but Backend.DeleteBucketCors was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketCors.Lock()
+	mock.calls.DeleteBucketCors = append(mock.calls.DeleteBucketCors, callInfo)
+	mock.lockDeleteBucketCors.Unlock()
+	return mock.DeleteBucketCorsFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketCorsCalls gets all the calls that were made to DeleteBucketCors.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketCorsCalls())
+func (mock *BackendMock) DeleteBucketCorsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketCors.RLock()
+	calls = mock.calls.DeleteBucketCors
+	mock.lockDeleteBucketCors.RUnlock()
+	return calls
+}
+
 // DeleteBucketOwnershipControls calls DeleteBucketOwnershipControlsFunc.
 func (mock *BackendMock) DeleteBucketOwnershipControls(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketOwnershipControlsFunc == nil {
@@ -1301,6 +1379,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketCors calls GetBucketCorsFunc.
+func (mock *BackendMock) GetBucketCors(contextMoqParam context.Context, bucket string) ([]byte, error) {
+	if mock.GetBucketCorsFunc == nil {
+		panic("BackendMock.GetBucketCorsFunc: method is nil but Backend.GetBucketCors was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketCors.Lock()
+	mock.calls.GetBucketCors = append(mock.calls.GetBucketCors, callInfo)
+	mock.lockGetBucketCors.Unlock()
+	return mock.GetBucketCorsFunc(contextMoqParam, bucket)
+}
+
+// GetBucketCorsCalls gets all the calls that were made to GetBucketCors.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketCorsCalls())
+func (mock *BackendMock) GetBucketCorsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketCors.RLock()
+	calls = mock.calls.GetBucketCors
+	mock.lockGetBucketCors.RUnlock()
+	return calls
+}
+
 // GetBucketOwnershipControls calls GetBucketOwnershipControlsFunc.
 func (mock *BackendMock) GetBucketOwnershipControls(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
 	if mock.GetBucketOwnershipControlsFunc == nil {
@@ -2077,6 +2191,42 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketCors calls PutBucketCorsFunc.
+func (mock *BackendMock) PutBucketCors(contextMoqParam context.Context, bytes []byte) error {
+	if mock.PutBucketCorsFunc == nil {
+		panic("BackendMock.PutBucketCorsFunc: method is nil but Backend.PutBucketCors was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bytes           []byte
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bytes:           bytes,
+	}
+	mock.lockPutBucketCors.Lock()
+	mock.calls.PutBucketCors = append(mock.calls.PutBucketCors, callInfo)
+	mock.lockPutBucketCors.Unlock()
+	return mock.PutBucketCorsFunc(contextMoqParam, bytes)
+}
+
+// PutBucketCorsCalls gets all the calls that were made to PutBucketCors.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketCorsCalls())
+func (mock *BackendMock) PutBucketCorsCalls() []struct {
+	ContextMoqParam context.Context
+	Bytes           []byte
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bytes           []byte
+	}
+	mock.lockPutBucketCors.RLock()
+	calls = mock.calls.PutBucketCors
+	mock.lockPutBucketCors.RUnlock()
+	return calls
+}
+
 // PutBucketOwnershipControls calls PutBucketOwnershipControlsFunc.
 func (mock *BackendMock) PutBucketOwnershipControls(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
 	if mock.PutBucketOwnershipControlsFunc == nil {
@@ -2238,13 +2388,13 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct {
 }
 
 // PutObject calls PutObjectFunc.
-func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 	if mock.PutObjectFunc == nil {
 		panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
-		PutObjectInput  *s3.PutObjectInput
+		PutObjectInput  s3response.PutObjectInput
 	}{
 		ContextMoqParam: contextMoqParam,
 		PutObjectInput:  putObjectInput,
@@ -2261,11 +2411,11 @@ func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInp
 //	len(mockedBackend.PutObjectCalls())
 func (mock *BackendMock) PutObjectCalls() []struct {
 	ContextMoqParam context.Context
-	PutObjectInput  *s3.PutObjectInput
+	PutObjectInput  s3response.PutObjectInput
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
-		PutObjectInput  *s3.PutObjectInput
+		PutObjectInput  s3response.PutObjectInput
 	}
 	mock.lockPutObject.RLock()
 	calls = mock.calls.PutObject
@@ -2620,7 +2770,7 @@ func (mock *BackendMock) StringCalls() []struct {
 }
 
 // UploadPart calls UploadPartFunc.
-func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 	if mock.UploadPartFunc == nil {
 		panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called")
 	}
@@ -2656,7 +2806,7 @@ func (mock *BackendMock) UploadPartCalls() []struct {
 }
 
 // UploadPartCopy calls UploadPartCopyFunc.
-func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 	if mock.UploadPartCopyFunc == nil {
 		panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called")
 	}

s3api/controllers/base_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
@@ -99,8 +100,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
-		ctx.Locals("isDebug", false)
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"})
 		return ctx.Next()
 	})
 	app.Get("/", s3ApiController.ListBuckets)
@@ -116,18 +116,17 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
-		ctx.Locals("isDebug", false)
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"})
 		return ctx.Next()
 	})
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
 
 	tests := []struct {
+		name       string
 		args       args
 		app        *fiber.App
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "List-bucket-method-not-allowed",
@@ -220,10 +219,9 @@ func TestS3ApiController_GetActions(t *testing.T) {
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
@@ -232,12 +230,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
+	invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
+	invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Get-actions-get-tags-success",
@@ -329,6 +330,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Get-actions-get-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invalidChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Get-actions-get-object-success",
 			app:  app,
@@ -401,10 +411,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 
@@ -426,20 +435,19 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	}
 	appError := fiber.New()
 	appError.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Get-bucket-tagging-non-existing-bucket",
@@ -622,8 +630,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</VersioningConfiguration>
 	`
 
-	policyBody := `
-	{
+	policyBody := `{
 		"Statement": [
 			{
 				"Effect": "Allow",
@@ -696,10 +703,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	}
 	// Mock ctx.Locals
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{Owner: "valid access"})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{Owner: "valid access"})
 		return ctx.Next()
 	})
 	app.Put("/:bucket", s3ApiController.PutBucketActions)
@@ -728,11 +734,11 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Put-bucket-tagging-invalid-body",
@@ -878,15 +884,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "Create-bucket-invalid-bucket-name",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
-			},
-			wantErr:    false,
-			statusCode: 400,
-		},
 		{
 			name: "Create-bucket-success",
 			app:  app,
@@ -941,12 +938,12 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	</Tagging>
 	`
 
-	retentionBody := `
-	<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-		<Mode>GOVERNANCE</Mode>
-		<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
-	</Retention>
-	`
+	//retentionBody := `
+	//<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	//	<Mode>GOVERNANCE</Mode>
+	//	<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
+	//</Retention>
+	//`
 
 	legalHoldBody := `
 	<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
@@ -963,22 +960,22 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
 				return nil
 			},
-			CopyObjectFunc: func(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-				return &s3.CopyObjectOutput{
-					CopyObjectResult: &types.CopyObjectResult{},
+			CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
+				return s3response.CopyObjectOutput{
+					CopyObjectResult: &s3response.CopyObjectResult{},
 				}, nil
 			},
-			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+			PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 				return s3response.PutObjectOutput{}, nil
 			},
-			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
-				return "hello", nil
+			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+				return &s3.UploadPartOutput{}, nil
 			},
 			PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
 				return nil
 			},
-			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-				return s3response.CopyObjectResult{}, nil
+			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+				return s3response.CopyPartResult{}, nil
 			},
 			PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
 				return nil
@@ -992,10 +989,9 @@ func TestS3ApiController_PutActions(t *testing.T) {
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
@@ -1012,6 +1008,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
 
+	// CopyObject invalid checksum algorithm
+	cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	// PutObjectAcl success
 	aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	aclReq.Header.Set("X-Amz-Acl", "private")
@@ -1033,12 +1034,46 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	// PutObject invalid checksum algorithm
+	invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
+	// PutObject invalid base64 checksum
+	invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
+
+	// PutObject invalid crc32
+	invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
+
+	// PutObject invalid crc32c
+	invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
+
+	// PutObject invalid sha1
+	invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
+
+	// PutObject invalid sha256
+	invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
+
+	// PutObject multiple checksum headers
+	mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
+	// PutObject checksum algorithm and header mismatch
+	checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Put-object-part-error-case",
@@ -1076,15 +1111,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "put-object-retention-success",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
+		//{
+		//	name: "put-object-retention-success",
+		//	app:  app,
+		//	args: args{
+		//		req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
+		//	},
+		//	wantErr:    false,
+		//	statusCode: 200,
+		//},
 		{
 			name: "put-legal-hold-invalid-request",
 			app:  app,
@@ -1175,6 +1210,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Copy-object-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: cpyInvChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Copy-object-success",
 			app:  app,
@@ -1233,21 +1277,20 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 
 	app.Delete("/:bucket", s3ApiController.DeleteBucket)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Delete-bucket-success",
@@ -1319,10 +1362,9 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Post("/:bucket", s3ApiController.DeleteObjects)
@@ -1334,11 +1376,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	request.Header.Set("Content-Type", "application/xml")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Delete-Objects-success",
@@ -1399,10 +1441,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
@@ -1423,20 +1464,19 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	}}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Abort-multipart-upload-success",
@@ -1506,11 +1546,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
-		ctx.Locals("region", "us-east-1")
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
+		utils.ContextKeyRegion.Set(ctx, "us-east-1")
 		return ctx.Next()
 	})
 
@@ -1524,28 +1563,27 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 			return acldata, nil
 		},
 		HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
-			return nil, s3err.GetAPIError(3)
+			return nil, s3err.GetAPIError(s3err.ErrBucketNotEmpty)
 		},
 	},
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
-		ctx.Locals("region", "us-east-1")
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
+		utils.ContextKeyRegion.Set(ctx, "us-east-1")
 		return ctx.Next()
 	})
 
 	appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Head-bucket-success",
@@ -1611,10 +1649,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
@@ -1634,20 +1671,22 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
+	invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
+	invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Head-object-success",
@@ -1658,6 +1697,15 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Head-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Head-object-error",
 			app:  appErr,
@@ -1694,10 +1742,10 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			RestoreObjectFunc: func(context.Context, *s3.RestoreObjectInput) error {
 				return nil
 			},
-			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-				return &s3.CompleteMultipartUploadOutput{}, nil
+			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+				return s3response.CompleteMultipartUploadResult{}, "", nil
 			},
-			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+			CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 				return s3response.InitiateMultipartUploadResult{}, nil
 			},
 			SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
@@ -1713,21 +1761,36 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 		</SelectObjectContentRequest>
 	`
 
+	completMpBody := `
+		<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Part>
+				<ETag>etag</ETag>
+				<PartNumber>1</PartNumber>
+			</Part>
+		</CompleteMultipartUpload>
+	`
+
+	completMpEmptyBody := `
+		<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></CompleteMultipartUpload>
+	`
+
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "valid access"})
-		ctx.Locals("isRoot", true)
-		ctx.Locals("isDebug", false)
-		ctx.Locals("parsedAcl", auth.ACL{})
+		utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
+		utils.ContextKeyIsRoot.Set(ctx, true)
+		utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
 		return ctx.Next()
 	})
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
+	invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Restore-object-success",
@@ -1765,15 +1828,33 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
+		{
+			name: "Complete-multipart-upload-empty-parts",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpEmptyBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Complete-multipart-upload-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(`<root><key>body</key></root>`)),
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpBody)),
 			},
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Create-multipart-upload-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: invChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Create-multipart-upload-success",
 			app:  app,
@@ -1808,10 +1889,10 @@ func Test_XMLresponse(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		args       args
 		name       string
-		statusCode int
+		args       args
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Internal-server-error",
@@ -1883,10 +1964,10 @@ func Test_response(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		args       args
 		name       string
-		statusCode int
+		args       args
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Internal-server-error",

s3api/debuglogger/logger.go

@@ -0,0 +1,226 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package debuglogger
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"sync/atomic"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type Color string
+
+const (
+	green  Color = "\033[32m"
+	yellow Color = "\033[33m"
+	blue   Color = "\033[34m"
+	Purple Color = "\033[0;35m"
+
+	reset      = "\033[0m"
+	borderChar = "─"
+	boxWidth   = 120
+)
+
+// Logs http request details: headers, body, params, query args
+func LogFiberRequestDetails(ctx *fiber.Ctx) {
+	// Log the full request url
+	fullURL := ctx.Protocol() + "://" + ctx.Hostname() + ctx.OriginalURL()
+	fmt.Printf("%s[URL]: %s%s\n", green, fullURL, reset)
+
+	// log request headers
+	wrapInBox(green, "REQUEST HEADERS", boxWidth, func() {
+		ctx.Request().Header.VisitAll(func(key, value []byte) {
+			printWrappedLine(yellow, string(key), string(value))
+		})
+	})
+	// skip request body log for PutObject and UploadPart
+	skipBodyLog := isLargeDataAction(ctx)
+	if !skipBodyLog {
+		body := ctx.Request().Body()
+		if len(body) != 0 {
+			printBoxTitleLine(blue, "REQUEST BODY", boxWidth, false)
+			fmt.Printf("%s%s%s\n", blue, body, reset)
+			printHorizontalBorder(blue, boxWidth, false)
+		}
+	}
+
+	if ctx.Request().URI().QueryArgs().Len() != 0 {
+		ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
+			log.Printf("%s: %s", key, val)
+		})
+	}
+}
+
+// Logs http response details: body, headers
+func LogFiberResponseDetails(ctx *fiber.Ctx) {
+	wrapInBox(green, "RESPONSE HEADERS", boxWidth, func() {
+		ctx.Response().Header.VisitAll(func(key, value []byte) {
+			printWrappedLine(yellow, string(key), string(value))
+		})
+	})
+
+	_, ok := ctx.Locals("skip-res-body-log").(bool)
+	if !ok {
+		body := ctx.Response().Body()
+		if len(body) != 0 {
+			PrintInsideHorizontalBorders(blue, "RESPONSE BODY", string(body), boxWidth)
+		}
+	}
+}
+
+var debugEnabled atomic.Bool
+
+// SetDebugEnabled sets the debug mode
+func SetDebugEnabled() {
+	debugEnabled.Store(true)
+}
+
+// Logf is the same as 'fmt.Printf' with debug prefix,
+// a color added and '\n' at the end
+func Logf(format string, v ...any) {
+	if !debugEnabled.Load() {
+		return
+	}
+	debugPrefix := "[DEBUG]: "
+	fmt.Printf(string(yellow)+debugPrefix+format+reset+"\n", v...)
+}
+
+// Infof prints out green info block with [INFO]: prefix
+func Infof(format string, v ...any) {
+	if !debugEnabled.Load() {
+		return
+	}
+	debugPrefix := "[INFO]: "
+	fmt.Printf(string(green)+debugPrefix+format+reset+"\n", v...)
+}
+
+// PrintInsideHorizontalBorders prints the text inside horizontal
+// border and title in the center of upper border
+func PrintInsideHorizontalBorders(color Color, title, text string, width int) {
+	if !debugEnabled.Load() {
+		return
+	}
+	printBoxTitleLine(color, title, width, false)
+	fmt.Printf("%s%s%s\n", color, text, reset)
+	printHorizontalBorder(color, width, false)
+}
+
+// Prints out box title either with closing characters or not:  "┌", "┐"
+// e.g ┌────────────────[ RESPONSE HEADERS ]────────────────┐
+func printBoxTitleLine(color Color, title string, length int, closing bool) {
+	leftCorner, rightCorner := "┌", "┐"
+
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	// Calculate how many border characters are needed
+	titleFormatted := fmt.Sprintf("[ %s ]", title)
+	borderSpace := length - len(titleFormatted) - 2 // 2 for corners
+	leftLen := borderSpace / 2
+	rightLen := borderSpace - leftLen
+
+	// Build the line
+	line := leftCorner +
+		strings.Repeat(borderChar, leftLen) +
+		titleFormatted +
+		strings.Repeat(borderChar, rightLen) +
+		rightCorner
+
+	fmt.Println(string(color) + line + reset)
+}
+
+// Prints out a horizontal line either with closing characters or not: "└", "┘"
+func printHorizontalBorder(color Color, length int, closing bool) {
+	leftCorner, rightCorner := "└", "┘"
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	line := leftCorner + strings.Repeat(borderChar, length-2) + rightCorner + reset
+	fmt.Println(string(color) + line)
+}
+
+// wrapInBox wraps the output of a function call (fn) inside a styled box with a title.
+func wrapInBox(color Color, title string, length int, fn func()) {
+	printBoxTitleLine(color, title, length, true)
+	fn()
+	printHorizontalBorder(color, length, true)
+}
+
+// returns the provided string length
+// defaulting to 13 for exceeding lengths
+func getLen(str string) int {
+	if len(str) < 13 {
+		return 13
+	}
+
+	return len(str)
+}
+
+// prints a formatted key-value pair within a box layout,
+// wrapping the value text if it exceeds the allowed width.
+func printWrappedLine(keyColor Color, key, value string) {
+	prefix := fmt.Sprintf("%s│%s %s%-13s%s : ", green, reset, keyColor, key, reset)
+	prefixLen := len(prefix) - len(green) - len(reset) - len(keyColor) - len(reset)
+	// the actual prefix size without colors
+	actualPrefixLen := getLen(key) + 5
+
+	lineWidth := boxWidth - prefixLen
+	valueLines := wrapText(value, lineWidth)
+
+	for i, line := range valueLines {
+		if i == 0 {
+			if len(line) < lineWidth {
+				line += strings.Repeat(" ", lineWidth-len(line))
+			}
+			fmt.Printf("%s%s%s %s│%s\n", prefix, reset, line, green, reset)
+		} else {
+			line = strings.Repeat(" ", actualPrefixLen-2) + line
+			if len(line) < boxWidth-4 {
+				line += strings.Repeat(" ", boxWidth-len(line)-4)
+			}
+			fmt.Printf("%s│ %s%s %s│%s\n", green, reset, line, green, reset)
+		}
+	}
+}
+
+// wrapText splits the input text into lines of at most `width` characters each.
+func wrapText(text string, width int) []string {
+	var lines []string
+	for len(text) > width {
+		lines = append(lines, text[:width])
+		text = text[width:]
+	}
+	if text != "" {
+		lines = append(lines, text)
+	}
+	return lines
+}
+
+// TODO: remove this and use utils.IsBidDataAction after refactoring
+// and creating 'internal' package
+func isLargeDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}

s3api/middlewares/acl-parser.go

@@ -24,6 +24,7 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
@@ -34,7 +35,6 @@ var (
 
 func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
 		path := ctx.Path()
 		pathParts := strings.Split(path, "/")
 		bucket := pathParts[1]
@@ -51,7 +51,9 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			!ctx.Request().URI().QueryArgs().Has("versioning") &&
 			!ctx.Request().URI().QueryArgs().Has("policy") &&
 			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
-			!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+			!ctx.Request().URI().QueryArgs().Has("ownershipControls") &&
+			!ctx.Request().URI().QueryArgs().Has("cors") {
+			isRoot, acct := utils.ContextKeyIsRoot.Get(ctx).(bool), utils.ContextKeyAccount.Get(ctx).(auth.Account)
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
@@ -74,7 +76,12 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 		}
 
-		ctx.Locals("parsedAcl", parsedAcl)
+		// if owner is not set, set default owner to root account
+		if parsedAcl.Owner == "" {
+			parsedAcl.Owner = utils.ContextKeyRootAccessKey.Get(ctx).(string)
+		}
+
+		utils.ContextKeyParsedAcl.Set(ctx, parsedAcl)
 		return ctx.Next()
 	}
 }

s3api/middlewares/admin.go

@@ -21,13 +21,14 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
 func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		acct := ctx.Locals("account").(auth.Account)
+		acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
 		if acct.Role != auth.RoleAdmin {
 			path := ctx.Path()
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),

s3api/middlewares/authentication.go

@@ -33,7 +33,8 @@ import (
 )
 
 const (
-	iso8601Format = "20060102T150405Z"
+	iso8601Format   = "20060102T150405Z"
+	maxObjSizeLimit = 5 * 1024 * 1024 * 1024 // 5gb
 )
 
 type RootUserConfig struct {
@@ -45,14 +46,15 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
-		// If account is set in context locals, it means it was presigned url case
-		_, ok := ctx.Locals("account").(auth.Account)
-		if ok {
+		// The bucket is public, no need to check this signature
+		if utils.ContextKeyPublicBucket.IsSet(ctx) {
+			return ctx.Next()
+		}
+		// If ContextKeyAuthenticated is set in context locals, it means it was presigned url case
+		if utils.ContextKeyAuthenticated.IsSet(ctx) {
 			return ctx.Next()
 		}
 
-		ctx.Locals("region", region)
-		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
 		if authorization == "" {
 			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
@@ -63,10 +65,6 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return sendResponse(ctx, err, logger, mm)
 		}
 
-		if authData.Algorithm != "AWS4-HMAC-SHA256" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger, mm)
-		}
-
 		if authData.Region != region {
 			return sendResponse(ctx, s3err.APIError{
 				Code:           "SignatureDoesNotMatch",
@@ -75,7 +73,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			}, logger, mm)
 		}
 
-		ctx.Locals("isRoot", authData.Access == root.Access)
+		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
@@ -84,7 +82,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)
 		}
-		ctx.Locals("account", account)
+
+		utils.ContextKeyAccount.Set(ctx, account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
@@ -108,6 +107,17 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return sendResponse(ctx, err, logger, mm)
 		}
 
+		var contentLength int64
+		contentLengthStr := ctx.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+			//TODO: not sure if InvalidRequest should be returned in this case
+			if err != nil {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
+			}
+		}
+
+		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if utils.IsBigDataAction(ctx) {
 			// for streaming PUT actions, authorization is deferred
 			// until end of stream due to need to get length and
@@ -115,10 +125,36 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
 				return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
 			})
+
+			// wrap the io.Reader with ChunkReader if x-amz-content-sha256
+			// provide chunk encoding value
+			if utils.IsStreamingPayload(hashPayload) {
+				var err error
+				wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+					var cr io.Reader
+					cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate)
+					return cr
+				})
+				if err != nil {
+					return sendResponse(ctx, err, logger, mm)
+				}
+
+				return ctx.Next()
+			}
+
+			// Content-Length has to be set for data uploads: PutObject, UploadPart
+			if contentLengthStr == "" {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingContentLength), logger, mm)
+			}
+			// the upload limit for big data actions: PutObject, UploadPart
+			// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
+			if contentLength > maxObjSizeLimit {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrEntityTooLarge), logger, mm)
+			}
+
 			return ctx.Next()
 		}
 
-		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if !utils.IsSpecialPayload(hashPayload) {
 			// Calculate the hash of the request payload
 			hashedPayload := sha256.Sum256(ctx.Body())
@@ -130,15 +166,6 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			}
 		}
 
-		var contentLength int64
-		contentLengthStr := ctx.Get("Content-Length")
-		if contentLengthStr != "" {
-			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
-			if err != nil {
-				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
-			}
-		}
-
 		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)
@@ -149,8 +176,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 }
 
 type accounts struct {
-	iam  auth.IAMService
 	root RootUserConfig
+	iam  auth.IAMService
 }
 
 func (a accounts) getAccount(access string) (auth.Account, error) {

s3api/middlewares/body-reader.go

@@ -18,14 +18,15 @@ import (
 	"io"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/utils"
 )
 
 func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
-	r, ok := ctx.Locals("body-reader").(io.Reader)
+	r, ok := utils.ContextKeyBodyReader.Get(ctx).(io.Reader)
 	if !ok {
 		r = ctx.Request().BodyStream()
 	}
 
 	r = wr(r)
-	ctx.Locals("body-reader", r)
+	utils.ContextKeyBodyReader.Set(ctx, r)
 }

s3api/middlewares/bucket-object-name-validator.go

@@ -0,0 +1,58 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/http"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+// BucketObjectNameValidator extracts and validates
+// the bucket and object names from the request URI.
+func BucketObjectNameValidator(l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		// skip the check for admin apis
+		if ctx.Method() == http.MethodPatch {
+			return ctx.Next()
+		}
+
+		path := ctx.Path()
+		// skip the check if the operation isn't bucket/object scoped
+		// e.g ListBuckets
+		if path == "/" {
+			return ctx.Next()
+		}
+
+		bucket, object := parsePath(path)
+
+		// check if the provided bucket name is valid
+		if !utils.IsValidBucketName(bucket) {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketName), l, mm)
+		}
+
+		// check if the provided object name is valid
+		// skip for empty objects: e.g bucket operations: HeadBucket...
+		if object != "" && !utils.IsObjectNameValid(object) {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrBadRequest), l, mm)
+		}
+
+		return ctx.Next()
+	}
+}

s3api/middlewares/chunk.go

@@ -1,62 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package middlewares
-
-import (
-	"io"
-	"time"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/metrics"
-	"github.com/versity/versitygw/s3api/utils"
-	"github.com/versity/versitygw/s3log"
-)
-
-// ProcessChunkedBody initializes the chunked upload stream if the
-// request appears to be a chunked upload
-func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
-	return func(ctx *fiber.Ctx) error {
-		decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
-		if decodedLength == "" {
-			return ctx.Next()
-		}
-		// TODO: validate content length
-
-		authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
-		if err != nil {
-			return sendResponse(ctx, err, logger, mm)
-		}
-
-		acct := ctx.Locals("account").(auth.Account)
-		amzdate := ctx.Get("X-Amz-Date")
-		date, _ := time.Parse(iso8601Format, amzdate)
-
-		if utils.IsBigDataAction(ctx) {
-			var err error
-			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
-				var cr *utils.ChunkReader
-				cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
-				return cr
-			})
-			if err != nil {
-				return sendResponse(ctx, err, logger, mm)
-			}
-			return ctx.Next()
-		}
-
-		return ctx.Next()
-	}
-}

s3api/middlewares/host-style-parser.go

@@ -0,0 +1,40 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+// HostStyleParser is a middleware which parses the bucket name
+// from the 'Host' header and appends in the request URL path
+func HostStyleParser(virtualDomain string) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		host := string(ctx.Request().Host())
+		// the host should match this pattern: '<bucket_name>.<virtual_domain>'
+		bucket, _, found := strings.Cut(host, "."+virtualDomain)
+		if !found || bucket == "" {
+			return ctx.Next()
+		}
+		path := ctx.Path()
+		pathStyleUrl := fmt.Sprintf("/%v%v", bucket, path)
+		ctx.Path(pathStyleUrl)
+
+		return ctx.Next()
+	}
+}

s3api/middlewares/logger.go

@@ -15,30 +15,15 @@
 package middlewares
 
 import (
-	"fmt"
-	"log"
-
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/debuglogger"
 )
 
-func RequestLogger(isDebug bool) fiber.Handler {
+func DebugLogger() fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		ctx.Locals("isDebug", isDebug)
-		if isDebug {
-			log.Println("Request headers: ")
-			ctx.Request().Header.VisitAll(func(key, val []byte) {
-				log.Printf("%s: %s", key, val)
-			})
-
-			if ctx.Request().URI().QueryArgs().Len() != 0 {
-				fmt.Println()
-				log.Println("Request query arguments: ")
-				ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
-					log.Printf("%s: %s", key, val)
-				})
-			}
-		}
-
-		return ctx.Next()
+		debuglogger.LogFiberRequestDetails(ctx)
+		err := ctx.Next()
+		debuglogger.LogFiberResponseDetails(ctx)
+		return err
 	}
 }

s3api/middlewares/md5.go

@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
 		}
 
 		sum := md5.Sum(ctx.Body())
-		calculatedSum := utils.Md5SumString(sum[:])
+		calculatedSum := utils.Base64SumString(sum[:])
 
 		if incomingSum != calculatedSum {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

s3api/middlewares/presign-auth.go

@@ -16,7 +16,7 @@ package middlewares
 
 import (
 	"io"
-	"time"
+	"strconv"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
@@ -30,19 +30,25 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
+		// The bucket is public, no need to check this signature
+		if utils.ContextKeyPublicBucket.IsSet(ctx) {
+			return ctx.Next()
+		}
 		if ctx.Query("X-Amz-Signature") == "" {
 			return ctx.Next()
 		}
 
-		ctx.Locals("region", region)
-		ctx.Locals("startTime", time.Now())
+		// Set in the context the "authenticated" key, in case the authentication succeeds,
+		// otherwise the middleware will return the caucht error
+		utils.ContextKeyAuthenticated.Set(ctx, true)
 
 		authData, err := utils.ParsePresignedURIParts(ctx)
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)
 		}
 
-		ctx.Locals("isRoot", authData.Access == root.Access)
+		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
+
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
 			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
@@ -50,9 +56,28 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 		if err != nil {
 			return sendResponse(ctx, err, logger, mm)
 		}
-		ctx.Locals("account", account)
+		utils.ContextKeyAccount.Set(ctx, account)
+
+		var contentLength int64
+		contentLengthStr := ctx.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+			//TODO: not sure if InvalidRequest should be returned in this case
+			if err != nil {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
+			}
+		}
 
 		if utils.IsBigDataAction(ctx) {
+			// Content-Length has to be set for data uploads: PutObject, UploadPart
+			if contentLengthStr == "" {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingContentLength), logger, mm)
+			}
+			// the upload limit for big data actions: PutObject, UploadPart
+			// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
+			if contentLength > maxObjSizeLimit {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrEntityTooLarge), logger, mm)
+			}
 			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
 				return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
 			})

s3api/middlewares/public-bucket.go

@@ -0,0 +1,298 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func AuthorizePublicBucketAccess(be backend.Backend, l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		// skip for auhtneicated requests
+		if ctx.Query("X-Amz-Algorithm") != "" || ctx.Get("Authorization") != "" {
+			return ctx.Next()
+		}
+
+		bucket, object := parsePath(ctx.Path())
+
+		action, permission, err := detectS3Action(ctx, object == "")
+		if err != nil {
+			return sendResponse(ctx, err, l, mm)
+		}
+
+		err = auth.VerifyPublicAccess(ctx.Context(), be, action, permission, bucket, object)
+		if err != nil {
+			return sendResponse(ctx, err, l, mm)
+		}
+
+		if utils.IsBigDataAction(ctx) {
+			payloadType := ctx.Get("X-Amz-Content-Sha256")
+			if utils.IsUnsignedStreamingPayload(payloadType) {
+				checksumType, err := utils.ExtractChecksumType(ctx)
+				if err != nil {
+					return sendResponse(ctx, err, l, mm)
+				}
+
+				wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+					var cr io.Reader
+					cr, err = utils.NewUnsignedChunkReader(r, checksumType)
+					return cr
+				})
+				if err != nil {
+					return sendResponse(ctx, err, l, mm)
+				}
+			}
+		}
+
+		utils.ContextKeyPublicBucket.Set(ctx, true)
+
+		return ctx.Next()
+	}
+}
+
+func detectS3Action(ctx *fiber.Ctx, isBucketAction bool) (auth.Action, auth.Permission, error) {
+	path := ctx.Path()
+	// ListBuckets is not publically available
+	if path == "/" {
+		//TODO: Still not clear what kind of error should be returned in this case(ListBuckets)
+		return "", auth.PermissionRead, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	queryArgs := ctx.Context().QueryArgs()
+
+	switch ctx.Method() {
+	case fiber.MethodPatch:
+		// Admin apis should always be protected
+		return "", "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	case fiber.MethodHead:
+		// HeadBucket
+		if isBucketAction {
+			return auth.ListBucketAction, auth.PermissionRead, nil
+		}
+
+		// HeadObject
+		return auth.GetObjectAction, auth.PermissionRead, nil
+	case fiber.MethodGet:
+		if isBucketAction {
+			if queryArgs.Has("tagging") {
+				// GetBucketTagging
+				return auth.GetBucketTaggingAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("ownershipControls") {
+				// GetBucketOwnershipControls
+				return auth.GetBucketOwnershipControlsAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousGetBucketOwnership)
+			} else if queryArgs.Has("versioning") {
+				// GetBucketVersioning
+				return auth.GetBucketVersioningAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("policy") {
+				// GetBucketPolicy
+				return auth.GetBucketPolicyAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("cors") {
+				// GetBucketCors
+				return auth.GetBucketCorsAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("versions") {
+				// ListObjectVersions
+				return auth.ListBucketVersionsAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("object-lock") {
+				// GetObjectLockConfiguration
+				return auth.GetBucketObjectLockConfigurationAction, auth.PermissionReadAcp, nil
+			} else if queryArgs.Has("acl") {
+				// GetBucketAcl
+				return auth.GetBucketAclAction, auth.PermissionRead, nil
+			} else if queryArgs.Has("uploads") {
+				// ListMultipartUploads
+				return auth.ListBucketMultipartUploadsAction, auth.PermissionRead, nil
+			} else if queryArgs.GetUintOrZero("list-type") == 2 {
+				// ListObjectsV2
+				return auth.ListBucketAction, auth.PermissionRead, nil
+			}
+			// All the other requests are considerd as ListObjects in the router
+			// no matter what kind of query arguments are provided apart from the ones above
+
+			return auth.ListBucketAction, auth.PermissionRead, nil
+		}
+
+		if queryArgs.Has("tagging") {
+			// GetObjectTagging
+			return auth.GetObjectTaggingAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("retention") {
+			// GetObjectRetention
+			return auth.GetObjectRetentionAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("legal-hold") {
+			// GetObjectLegalHold
+			return auth.GetObjectLegalHoldAction, auth.PermissionReadAcp, nil
+		} else if queryArgs.Has("acl") {
+			// GetObjectAcl
+			return auth.GetObjectAclAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("attributes") {
+			// GetObjectAttributes
+			return auth.GetObjectAttributesAction, auth.PermissionRead, nil
+		} else if queryArgs.Has("uploadId") {
+			// ListParts
+			return auth.ListMultipartUploadPartsAction, auth.PermissionRead, nil
+		}
+
+		// All the other requests are considerd as GetObject in the router
+		// no matter what kind of query arguments are provided apart from the ones above
+		if queryArgs.Has("versionId") {
+			return auth.GetObjectVersionAction, auth.PermissionRead, nil
+		}
+		return auth.GetObjectAction, auth.PermissionRead, nil
+	case fiber.MethodPut:
+		if isBucketAction {
+			if queryArgs.Has("tagging") {
+				// PutBucketTagging
+				return auth.PutBucketTaggingAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("ownershipControls") {
+				// PutBucketOwnershipControls
+				return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership)
+			}
+			if queryArgs.Has("versioning") {
+				// PutBucketVersioning
+				return auth.PutBucketVersioningAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("object-lock") {
+				// PutObjectLockConfiguration
+				return auth.PutBucketObjectLockConfigurationAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("cors") {
+				// PutBucketCors
+				return auth.PutBucketCorsAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("policy") {
+				// PutBucketPolicy
+				return auth.PutBucketPolicyAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("acl") {
+				// PutBucketAcl
+				return auth.PutBucketAclAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousRequest)
+			}
+
+			// All the other rquestes are considered as 'CreateBucket' in the router
+			return "", "", s3err.GetAPIError(s3err.ErrAnonymousRequest)
+		}
+
+		if queryArgs.Has("tagging") {
+			// PutObjectTagging
+			return auth.PutObjectTaggingAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("retention") {
+			// PutObjectRetention
+			return auth.PutObjectRetentionAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("legal-hold") {
+			// PutObjectLegalHold
+			return auth.PutObjectLegalHoldAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("acl") {
+			// PutObjectAcl
+			return auth.PutObjectAclAction, auth.PermissionWriteAcp, s3err.GetAPIError(s3err.ErrAnonymousRequest)
+		}
+		if queryArgs.Has("uploadId") && queryArgs.Has("partNumber") {
+			if ctx.Get("X-Amz-Copy-Source") != "" {
+				// UploadPartCopy
+				//TODO: Add public access check for copy-source
+				// Return AccessDenied for now
+				return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied)
+			}
+
+			utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream())
+			// UploadPart
+			return auth.PutObjectAction, auth.PermissionWrite, nil
+		}
+		if ctx.Get("X-Amz-Copy-Source") != "" {
+			return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousCopyObject)
+		}
+
+		utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream())
+		// All the other requests are considered as 'PutObject' in the router
+		return auth.PutObjectAction, auth.PermissionWrite, nil
+	case fiber.MethodPost:
+		if isBucketAction {
+			// DeleteObjects
+			// FIXME: should be fixed with https://github.com/versity/versitygw/issues/1327
+			// Return AccessDenied for now
+			return auth.DeleteObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+
+		if queryArgs.Has("restore") {
+			return auth.RestoreObjectAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("select") && ctx.Query("select-type") == "2" {
+			// SelectObjectContent
+			return auth.GetObjectAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousRequest)
+		}
+		if queryArgs.Has("uploadId") {
+			// CompleteMultipartUpload
+			return auth.PutObjectAction, auth.PermissionWrite, nil
+		}
+
+		// All the other requests are considered as 'CreateMultipartUpload' in the router
+		return "", "", s3err.GetAPIError(s3err.ErrAnonymousCreateMp)
+	case fiber.MethodDelete:
+		if isBucketAction {
+			if queryArgs.Has("tagging") {
+				// DeleteBucketTagging
+				return auth.PutBucketTaggingAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("ownershipControls") {
+				// DeleteBucketOwnershipControls
+				return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership)
+			}
+			if queryArgs.Has("policy") {
+				// DeleteBucketPolicy
+				return auth.PutBucketPolicyAction, auth.PermissionWrite, nil
+			}
+			if queryArgs.Has("cors") {
+				// DeleteBucketCors
+				return auth.PutBucketCorsAction, auth.PermissionWrite, nil
+			}
+
+			// All the other requests are considered as 'DeleteBucket' in the router
+			return auth.DeleteBucketAction, auth.PermissionWrite, nil
+		}
+
+		if queryArgs.Has("tagging") {
+			// DeleteObjectTagging
+			return auth.PutObjectTaggingAction, auth.PermissionWrite, nil
+		}
+		if queryArgs.Has("uploadId") {
+			// AbortMultipartUpload
+			return auth.AbortMultipartUploadAction, auth.PermissionWrite, nil
+		}
+		// All the other requests are considered as 'DeleteObject' in the router
+		return auth.DeleteObjectAction, auth.PermissionWrite, nil
+	default:
+		// In no action is detected, return AccessDenied ?
+		return "", "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+}
+
+// parsePath extracts the bucket and object names from the path
+func parsePath(path string) (string, string) {
+	p := strings.TrimPrefix(path, "/")
+	bucket, object, _ := strings.Cut(p, "/")
+
+	return bucket, object
+}

s3api/middlewares/set-default-keys.go

@@ -0,0 +1,37 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
+)
+
+func SetDefaultValues(root RootUserConfig, region string) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		// These are necessary for the server access logs
+		utils.ContextKeyRegion.Set(ctx, region)
+		utils.ContextKeyStartTime.Set(ctx, time.Now())
+		utils.ContextKeyRootAccessKey.Set(ctx, root.Access)
+		// Set the account and isRoot to some defulat values, to avoid panics
+		// in case of public buckets
+		utils.ContextKeyAccount.Set(ctx, auth.Account{})
+		utils.ContextKeyIsRoot.Set(ctx, false)
+		return ctx.Next()
+	}
+}

s3api/middlewares/url-decoder.go

@@ -26,7 +26,7 @@ import (
 
 func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
+		unescp, err := url.PathUnescape(string(ctx.Request().URI().PathOriginal()))
 		if err != nil {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}

s3api/router.go

@@ -42,7 +42,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
 
 		// UpdateUser admin api
-		app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
+		app.Patch("/update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
 
 		// ListUsers admin api
 		app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)

s3api/router_test.go

@@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) {
 		iam auth.IAMService
 	}
 	tests := []struct {
-		args args
-		sa   *S3ApiRouter
 		name string
+		sa   *S3ApiRouter
+		args args
 	}{
 		{
 			name: "Initialize S3 api router",

s3api/server.go

@@ -29,15 +29,16 @@ import (
 )
 
 type S3ApiServer struct {
-	backend  backend.Backend
-	app      *fiber.App
-	router   *S3ApiRouter
-	cert     *tls.Certificate
-	port     string
-	health   string
-	quiet    bool
-	debug    bool
-	readonly bool
+	app           *fiber.App
+	backend       backend.Backend
+	router        *S3ApiRouter
+	port          string
+	cert          *tls.Certificate
+	quiet         bool
+	debug         bool
+	readonly      bool
+	health        string
+	virtualDomain string
 }
 
 func New(
@@ -76,12 +77,29 @@ func New(
 		})
 	}
 	app.Use(middlewares.DecodeURL(l, mm))
-	app.Use(middlewares.RequestLogger(server.debug))
+
+	// initialize host-style parser in virtual domain is specified
+	if server.virtualDomain != "" {
+		app.Use(middlewares.HostStyleParser(server.virtualDomain))
+	}
+
+	// initilaze the default value setter middleware
+	app.Use(middlewares.SetDefaultValues(root, region))
+
+	// initialize the debug logger in debug mode
+	if server.debug {
+		app.Use(middlewares.DebugLogger())
+	}
+
+	// initialize the bucket/object name validator
+	app.Use(middlewares.BucketObjectNameValidator(l, mm))
+
+	// Public buckets access checker
+	app.Use(middlewares.AuthorizePublicBucketAccess(be, l, mm))
 
 	// Authentication middlewares
 	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
-	app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l, server.readonly))
 
@@ -122,6 +140,11 @@ func WithReadOnly() Option {
 	return func(s *S3ApiServer) { s.readonly = true }
 }
 
+// WithHostStyle enabled host-style bucket addressing on the server
+func WithHostStyle(virtualDomain string) Option {
+	return func(s *S3ApiServer) { s.virtualDomain = virtualDomain }
+}
+
 func (sa *S3ApiServer) Serve() (err error) {
 	if sa.cert != nil {
 		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

s3api/server_test.go

@@ -39,9 +39,9 @@ func TestNew(t *testing.T) {
 	port := ":7070"
 
 	tests := []struct {
-		wantS3ApiServer *S3ApiServer
-		args            args
 		name            string
+		args            args
+		wantS3ApiServer *S3ApiServer
 		wantErr         bool
 	}{
 		{
@@ -78,8 +78,8 @@ func TestNew(t *testing.T) {
 
 func TestS3ApiServer_Serve(t *testing.T) {
 	tests := []struct {
-		sa      *S3ApiServer
 		name    string
+		sa      *S3ApiServer
 		wantErr bool
 	}{
 		{

s3api/utils/auth-reader.go

@@ -41,10 +41,10 @@ const (
 // the data is completely read.
 type AuthReader struct {
 	ctx    *fiber.Ctx
-	r      *HashReader
 	auth   AuthData
 	secret string
 	size   int
+	r      *HashReader
 	debug  bool
 }
 
@@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de
 	var hr *HashReader
 	hashPayload := ctx.Get("X-Amz-Content-Sha256")
 	if !IsSpecialPayload(hashPayload) {
-		hr, _ = NewHashReader(r, "", HashTypeSha256)
+		hr, _ = NewHashReader(r, "", HashTypeSha256Hex)
 	} else {
 		hr, _ = NewHashReader(r, "", HashTypeNone)
 	}
@@ -190,6 +190,10 @@ func ParseAuthorization(authorization string) (AuthData, error) {
 
 	algo := authParts[0]
 
+	if algo != "AWS4-HMAC-SHA256" {
+		return a, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported)
+	}
+
 	kvData := authParts[1]
 	kvPairs := strings.Split(kvData, ",")
 	// we are expecting at least Credential, SignedHeaders, and Signature
@@ -260,19 +264,3 @@ func removeSpace(str string) string {
 	}
 	return b.String()
 }
-
-var (
-	specialValues = map[string]bool{
-		"UNSIGNED-PAYLOAD":                                 true,
-		"STREAMING-UNSIGNED-PAYLOAD-TRAILER":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER":       true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD":         true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
-	}
-)
-
-// IsSpecialPayload checks for streaming/unsigned authorization types
-func IsSpecialPayload(str string) bool {
-	return specialValues[str]
-}

s3api/utils/chunk-reader.go

@@ -15,260 +15,155 @@
 package utils
 
 import (
-	"bytes"
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
 	"fmt"
-	"hash"
 	"io"
-	"math"
+	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/debuglogger"
 	"github.com/versity/versitygw/s3err"
 )
 
-// chunked uploads described in:
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
-
 const (
-	chunkHdrStr       = ";chunk-signature="
-	chunkHdrDelim     = "\r\n"
-	zeroLenSig        = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-	awsV4             = "AWS4"
-	awsS3Service      = "s3"
-	awsV4Request      = "aws4_request"
-	streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
+	maxObjSizeLimit = 5 * 1024 * 1024 * 1024 // 5gb
 )
 
-// ChunkReader reads from chunked upload request body, and returns
-// object data stream
-type ChunkReader struct {
-	r                io.Reader
-	chunkHash        hash.Hash
-	prevSig          string
-	parsedSig        string
-	strToSignPrefix  string
-	signingKey       []byte
-	stash            []byte
-	currentChunkSize int64
-	chunkDataLeft    int64
-	trailerExpected  int
-	skipcheck        bool
-}
+type payloadType string
 
-// NewChunkReader reads from request body io.Reader and parses out the
-// chunk metadata in stream. The headers are validated for proper signatures.
-// Reading from the chunk reader will read only the object data stream
-// without the chunk headers/trailers.
-func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (*ChunkReader, error) {
-	return &ChunkReader{
-		r:          r,
-		signingKey: getSigningKey(secret, region, date),
-		// the authdata.Signature is validated in the auth-reader,
-		// so we can use that here without any other checks
-		prevSig:         authdata.Signature,
-		chunkHash:       sha256.New(),
-		strToSignPrefix: getStringToSignPrefix(date, region),
-	}, nil
-}
+const (
+	payloadTypeUnsigned                 payloadType = "UNSIGNED-PAYLOAD"
+	payloadTypeStreamingUnsignedTrailer payloadType = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+	payloadTypeStreamingSigned          payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	payloadTypeStreamingSignedTrailer   payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
+	payloadTypeStreamingEcdsa           payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
+	payloadTypeStreamingEcdsaTrailer    payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
+)
 
-// Read satisfies the io.Reader for this type
-func (cr *ChunkReader) Read(p []byte) (int, error) {
-	n, err := cr.r.Read(p)
-	if err != nil && err != io.EOF {
-		return n, err
+func getPayloadTypeNotSupportedErr(p payloadType) error {
+	return s3err.APIError{
+		HTTPStatusCode: http.StatusNotImplemented,
+		Code:           "NotImplemented",
+		Description:    fmt.Sprintf("The chunk encoding algorithm %v is not supported.", p),
 	}
-
-	if cr.chunkDataLeft < int64(n) {
-		chunkSize := cr.chunkDataLeft
-		if chunkSize > 0 {
-			cr.chunkHash.Write(p[:chunkSize])
-		}
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		n += int(chunkSize)
-		return n, err
-	}
-
-	cr.chunkDataLeft -= int64(n)
-	cr.chunkHash.Write(p[:n])
-	return n, err
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
-// This part is the same for all chunks,
-// only the previous signature and hash of current chunk changes
-func getStringToSignPrefix(date time.Time, region string) string {
-	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
-		date.Format("20060102"),
-		region,
-		awsS3Service,
-		awsV4Request)
-
-	return fmt.Sprintf("%s\n%s\n%s",
-		streamPayloadAlgo,
-		date.Format("20060102T150405Z"),
-		credentialScope)
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
-// signature For each chunk, you calculate the signature using the following
-// string to sign. For the first chunk, you use the seed-signature as the
-// previous signature.
-func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
-	return fmt.Sprintf("%s\n%s\n%s\n%s",
-		prefix,
-		prevSig,
-		zeroLenSig,
-		hex.EncodeToString(chunkHash))
-}
-
-// The provided p should have all of the previous chunk data and trailer
-// consumed already. The positioning here is expected that p[0] starts the
-// new chunk size with the ";chunk-signature=" following. The only exception
-// is if we started consuming the trailer, but hit the end of the read buffer.
-// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
-// finish consuming the final trailer bytes.
-// This parses the chunk metadata in situ without allocating an extra buffer.
-// It will just read and validate the chunk metadata and then move the
-// following chunk data to overwrite the metadata in the provided buffer.
-func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
-	n := len(p)
-
-	if !cr.skipcheck && cr.parsedSig != "" {
-		chunkhash := cr.chunkHash.Sum(nil)
-		cr.chunkHash.Reset()
-
-		sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
-		cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
-
-		if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
-		}
-	}
-
-	if cr.trailerExpected != 0 {
-		if len(p) < len(chunkHdrDelim) {
-			// This is the special case where we need to consume the
-			// trailer, but instead hit the end of the buffer. The
-			// subsequent call will finish consuming the trailer.
-			cr.chunkDataLeft = 0
-			cr.trailerExpected -= len(p)
-			cr.skipcheck = true
-			return 0, nil
-		}
-		// move data up to remove trailer
-		copy(p, p[cr.trailerExpected:])
-		n -= cr.trailerExpected
-	}
-
-	cr.skipcheck = false
-
-	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
-	cr.currentChunkSize = chunkSize
-	cr.parsedSig = sig
-	if err == errskipHeader {
-		cr.chunkDataLeft = 0
-		return 0, nil
-	}
-	if err != nil {
-		return 0, err
-	}
-	if chunkSize == 0 {
-		return 0, io.EOF
-	}
-
-	cr.trailerExpected = len(chunkHdrDelim)
-
-	// move data up to remove chunk header
-	copy(p, p[bufOffset:n])
-	n -= bufOffset
-
-	// if remaining buffer larger than chunk data,
-	// parse next header in buffer
-	if int64(n) > chunkSize {
-		cr.chunkDataLeft = 0
-		cr.chunkHash.Write(p[:chunkSize])
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		if (chunkSize + int64(n)) > math.MaxInt {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
-		}
-		return n + int(chunkSize), err
-	}
-
-	cr.chunkDataLeft = chunkSize - int64(n)
-	cr.chunkHash.Write(p[:n])
-
-	return n, nil
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
-// Task 3: Calculate Signature
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
-func getSigningKey(secret, region string, date time.Time) []byte {
-	dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
-	dateRegionKey := hmac256(dateKey, []byte(region))
-	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
-	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
-	return signingKey
-}
-
-func hmac256(key []byte, data []byte) []byte {
-	hash := hmac.New(sha256.New, key)
-	hash.Write(data)
-	return hash.Sum(nil)
 }
 
 var (
-	errInvalidChunkFormat = errors.New("invalid chunk header format")
-	errskipHeader         = errors.New("skip to next header")
+	specialValues = map[payloadType]bool{
+		payloadTypeUnsigned:                 true,
+		payloadTypeStreamingUnsignedTrailer: true,
+		payloadTypeStreamingSigned:          true,
+		payloadTypeStreamingSignedTrailer:   true,
+		payloadTypeStreamingEcdsa:           true,
+		payloadTypeStreamingEcdsaTrailer:    true,
+	}
 )
 
+func (pt payloadType) isValid() bool {
+	return pt == payloadTypeUnsigned ||
+		pt == payloadTypeStreamingUnsignedTrailer ||
+		pt == payloadTypeStreamingSigned ||
+		pt == payloadTypeStreamingSignedTrailer ||
+		pt == payloadTypeStreamingEcdsa ||
+		pt == payloadTypeStreamingEcdsaTrailer
+}
+
+type checksumType string
+
 const (
-	maxHeaderSize = 1024
+	checksumTypeCrc32     checksumType = "x-amz-checksum-crc32"
+	checksumTypeCrc32c    checksumType = "x-amz-checksum-crc32c"
+	checksumTypeSha1      checksumType = "x-amz-checksum-sha1"
+	checksumTypeSha256    checksumType = "x-amz-checksum-sha256"
+	checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
 )
 
-// Theis returns the chunk payload size, signature, data start offset, and
-// error if any. See the AWS documentation for the chunk header format. The
-// header[0] byte is expected to be the first byte of the chunk size here.
-func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
-	stashLen := len(cr.stash)
-	if cr.stash != nil {
-		tmp := make([]byte, maxHeaderSize)
-		copy(tmp, cr.stash)
-		copy(tmp[len(cr.stash):], header)
-		header = tmp
-		cr.stash = nil
-	}
-
-	semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
-	if semicolonIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
-	}
-
-	sigIndex := semicolonIndex + len(chunkHdrStr)
-	sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
-	if sigEndIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
-	}
-
-	chunkSizeBytes := header[:semicolonIndex]
-	chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
-	if err != nil {
-		return 0, "", 0, errInvalidChunkFormat
-	}
-
-	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
-	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
-
-	return chunkSize, signature, dataStartOffset - stashLen, nil
+func (c checksumType) isValid() bool {
+	return c == checksumTypeCrc32 ||
+		c == checksumTypeCrc32c ||
+		c == checksumTypeSha1 ||
+		c == checksumTypeSha256 ||
+		c == checksumTypeCrc64nvme
+}
+
+// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
+func ExtractChecksumType(ctx *fiber.Ctx) (checksumType, error) {
+	trailer := ctx.Get("X-Amz-Trailer")
+	chType := checksumType(strings.ToLower(trailer))
+	if chType != "" && !chType.isValid() {
+		debuglogger.Logf("invalid value for 'X-Amz-Trailer': %v", chType)
+		return "", s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
+	}
+
+	return chType, nil
+}
+
+// IsSpecialPayload checks for special authorization types
+func IsSpecialPayload(str string) bool {
+	return specialValues[payloadType(str)]
+}
+
+// Checks if the provided string is unsigned payload trailer type
+func IsUnsignedStreamingPayload(str string) bool {
+	return payloadType(str) == payloadTypeStreamingUnsignedTrailer
+}
+
+// IsChunkEncoding checks for streaming/unsigned authorization types
+func IsStreamingPayload(str string) bool {
+	pt := payloadType(str)
+	return pt == payloadTypeStreamingUnsignedTrailer ||
+		pt == payloadTypeStreamingSigned ||
+		pt == payloadTypeStreamingSignedTrailer
+}
+
+func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
+	decContLengthStr := ctx.Get("X-Amz-Decoded-Content-Length")
+	if decContLengthStr == "" {
+		debuglogger.Logf("missing required header 'X-Amz-Decoded-Content-Length'")
+		return nil, s3err.GetAPIError(s3err.ErrMissingContentLength)
+	}
+	decContLength, err := strconv.ParseInt(decContLengthStr, 10, 64)
+	//TODO: not sure if InvalidRequest should be returned in this case
+	if err != nil {
+		debuglogger.Logf("invalid value for 'X-Amz-Decoded-Content-Length': %v", decContLengthStr)
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	if decContLength > maxObjSizeLimit {
+		debuglogger.Logf("the object size exceeds the allowed limit: (size): %v, (limit): %v", decContLength, maxObjSizeLimit)
+		return nil, s3err.GetAPIError(s3err.ErrEntityTooLarge)
+	}
+
+	contentSha256 := payloadType(ctx.Get("X-Amz-Content-Sha256"))
+	if !contentSha256.isValid() {
+		//TODO: Add proper APIError
+		debuglogger.Logf("invalid value for 'X-Amz-Content-Sha256': %v", contentSha256)
+		return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
+	}
+
+	checksumType, err := ExtractChecksumType(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if contentSha256 != payloadTypeStreamingSigned && checksumType == "" {
+		debuglogger.Logf("empty value for required trailer header 'X-Amz-Trailer': %v", checksumType)
+		return nil, s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
+	}
+
+	switch contentSha256 {
+	case payloadTypeStreamingUnsignedTrailer:
+		return NewUnsignedChunkReader(r, checksumType)
+	case payloadTypeStreamingSignedTrailer:
+		return NewSignedChunkReader(r, authdata, region, secret, date, checksumType)
+	case payloadTypeStreamingSigned:
+		return NewSignedChunkReader(r, authdata, region, secret, date, "")
+	// return not supported for:
+	// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
+	// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
+	default:
+		debuglogger.Logf("unsupported chunk reader algorithm: %v", contentSha256)
+		return nil, getPayloadTypeNotSupportedErr(contentSha256)
+	}
 }

s3api/utils/context-keys.go

@@ -0,0 +1,65 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"github.com/gofiber/fiber/v2"
+)
+
+// Region, StartTime, IsRoot, Account, AccessKey context locals
+// are set to defualut values in middlewares.SetDefaultValues
+// to avoid the nil interface conversions
+type ContextKey string
+
+const (
+	ContextKeyRegion         ContextKey = "region"
+	ContextKeyStartTime      ContextKey = "start-time"
+	ContextKeyIsRoot         ContextKey = "is-root"
+	ContextKeyRootAccessKey  ContextKey = "root-access-key"
+	ContextKeyAccount        ContextKey = "account"
+	ContextKeyAuthenticated  ContextKey = "authenticated"
+	ContextKeyPublicBucket   ContextKey = "public-bucket"
+	ContextKeyParsedAcl      ContextKey = "parsed-acl"
+	ContextKeySkipResBodyLog ContextKey = "skip-res-body-log"
+	ContextKeyBodyReader     ContextKey = "body-reader"
+)
+
+func (ck ContextKey) Values() []ContextKey {
+	return []ContextKey{
+		ContextKeyRegion,
+		ContextKeyStartTime,
+		ContextKeyIsRoot,
+		ContextKeyRootAccessKey,
+		ContextKeyAccount,
+		ContextKeyAuthenticated,
+		ContextKeyPublicBucket,
+		ContextKeyParsedAcl,
+		ContextKeySkipResBodyLog,
+		ContextKeyBodyReader,
+	}
+}
+
+func (ck ContextKey) Set(ctx *fiber.Ctx, val any) {
+	ctx.Locals(string(ck), val)
+}
+
+func (ck ContextKey) IsSet(ctx *fiber.Ctx) bool {
+	val := ctx.Locals(string(ck))
+	return val != nil
+}
+
+func (ck ContextKey) Get(ctx *fiber.Ctx) any {
+	return ctx.Locals(string(ck))
+}

s3api/utils/crc.go

@@ -0,0 +1,180 @@
+// Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+//
+// This software is provided 'as-is', without any express or implied
+// warranty.  In no event will the authors be held liable for any damages
+// arising from the use of this software.
+//
+// Permission is granted to anyone to use this software for any purpose,
+// including commercial applications, and to alter it and redistribute it
+// freely, subject to the following restrictions:
+//
+// 1. The origin of this software must not be misrepresented; you must not
+//    claim that you wrote the original software. If you use this software
+//    in a product, an acknowledgment in the product documentation would be
+//    appreciated but is not required.
+// 2. Altered source versions must be plainly marked as such, and must not be
+//    misrepresented as being the original software.
+// 3. This notice may not be removed or altered from any source distribution.
+//
+// Jean-loup Gailly        Mark Adler
+// jloup@gzip.org          madler@alumni.caltech.edu
+
+// Original implementation is from
+// https://github.com/vimeo/go-util/blob/8cd4c737f091d9317f72b25df78ce6cf869f7d30/crc32combine/crc32combine.go
+// extended for crc64 support.
+
+// Following is ported from C to Go in 2016 by Justin Ruggles, with minimal alteration.
+// Used uint for unsigned long. Used uint32 for input arguments in order to match
+// the Go hash/crc32 package. zlib CRC32 combine (https://github.com/madler/zlib)
+
+package utils
+
+import (
+	"hash/crc64"
+)
+
+const crc64NVME = 0x9a6c_9329_ac4b_c9b5
+
+var crc64NVMETable = crc64.MakeTable(crc64NVME)
+
+func gf2MatrixTimes(mat []uint64, vec uint64) uint64 {
+	var sum uint64
+
+	for vec != 0 {
+		if vec&1 != 0 {
+			sum ^= mat[0]
+		}
+		vec >>= 1
+		mat = mat[1:]
+	}
+	return sum
+}
+
+func gf2MatrixSquare(square, mat []uint64) {
+	if len(square) != len(mat) {
+		panic("square matrix size mismatch")
+	}
+	for n := range mat {
+		square[n] = gf2MatrixTimes(mat, mat[n])
+	}
+}
+
+// crc32Combine returns the combined CRC-32 hash value of the two passed CRC-32
+// hash values crc1 and crc2. poly represents the generator polynomial
+// and len2 specifies the byte length that the crc2 hash covers.
+func crc32Combine(poly uint32, crc1, crc2 uint32, len2 int64) uint32 {
+	// degenerate case (also disallow negative lengths)
+	if len2 <= 0 {
+		return crc1
+	}
+
+	even := make([]uint64, 32) // even-power-of-two zeros operator
+	odd := make([]uint64, 32)  // odd-power-of-two zeros operator
+
+	// put operator for one zero bit in odd
+	odd[0] = uint64(poly) // CRC-32 polynomial
+	row := uint64(1)
+	for n := 1; n < 32; n++ {
+		odd[n] = row
+		row <<= 1
+	}
+
+	// put operator for two zero bits in even
+	gf2MatrixSquare(even, odd)
+
+	// put operator for four zero bits in odd
+	gf2MatrixSquare(odd, even)
+
+	// apply len2 zeros to crc1 (first square will put the operator for one
+	// zero byte, eight zero bits, in even)
+	crc1n := uint64(crc1)
+	for {
+		// apply zeros operator for this bit of len2
+		gf2MatrixSquare(even, odd)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(even, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+
+		// another iteration of the loop with odd and even swapped
+		gf2MatrixSquare(odd, even)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(odd, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+	}
+
+	// return combined crc
+	crc1n ^= uint64(crc2)
+	return uint32(crc1n)
+}
+
+// crc64Combine returns the combined CRC-64 hash value of the two passed CRC-64
+// hash values crc1 and crc2. poly represents the generator polynomial
+// and len2 specifies the byte length that the crc2 hash covers.
+func crc64Combine(poly uint64, crc1, crc2 uint64, len2 int64) uint64 {
+	// degenerate case (also disallow negative lengths)
+	if len2 <= 0 {
+		return crc1
+	}
+
+	even := make([]uint64, 64) // even-power-of-two zeros operator
+	odd := make([]uint64, 64)  // odd-power-of-two zeros operator
+
+	// put operator for one zero bit in odd
+	odd[0] = poly // CRC-64 polynomial
+	row := uint64(1)
+	for n := 1; n < 64; n++ {
+		odd[n] = row
+		row <<= 1
+	}
+
+	// put operator for two zero bits in even
+	gf2MatrixSquare(even, odd)
+
+	// put operator for four zero bits in odd
+	gf2MatrixSquare(odd, even)
+
+	// apply len2 zeros to crc1 (first square will put the operator for one
+	// zero byte, eight zero bits, in even)
+	crc1n := crc1
+	for {
+		// apply zeros operator for this bit of len2
+		gf2MatrixSquare(even, odd)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(even, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+
+		// another iteration of the loop with odd and even swapped
+		gf2MatrixSquare(odd, even)
+		if len2&1 != 0 {
+			crc1n = gf2MatrixTimes(odd, crc1n)
+		}
+		len2 >>= 1
+
+		// if no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+	}
+
+	// return combined crc
+	crc1n ^= crc2
+	return crc1n
+}

s3api/utils/crc_test.go

@@ -0,0 +1,57 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"hash/crc32"
+	"hash/crc64"
+	"testing"
+)
+
+func TestCRC32Combine(t *testing.T) {
+	data := []byte("The quick brown fox jumps over the lazy dog")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	var poly uint32 = crc32.IEEE
+	tab := crc32.MakeTable(poly)
+	crc1 := crc32.Checksum(part1, tab)
+	crc2 := crc32.Checksum(part2, tab)
+	combined := crc32Combine(poly, crc1, crc2, int64(len(part2)))
+	full := crc32.Checksum(data, tab)
+
+	if combined != full {
+		t.Errorf("crc32Combine failed: got %08x, want %08x", combined, full)
+	}
+}
+
+func TestCRC64Combine(t *testing.T) {
+	data := []byte("The quick brown fox jumps over the lazy dog")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	var poly uint64 = crc64NVME
+	tab := crc64NVMETable
+	crc1 := crc64.Checksum(part1, tab)
+	crc2 := crc64.Checksum(part2, tab)
+	combined := crc64Combine(poly, crc1, crc2, int64(len(part2)))
+	full := crc64.Checksum(data, tab)
+
+	if combined != full {
+		t.Errorf("crc64Combine failed: got %016x, want %016x", combined, full)
+	}
+}

s3api/utils/csum-reader.go

@@ -16,13 +16,18 @@ package utils
 
 import (
 	"crypto/md5"
+	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"hash"
+	"hash/crc32"
+	"hash/crc64"
 	"io"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -31,11 +36,21 @@ type HashType string
 
 const (
 	// HashTypeMd5 generates MD5 checksum for the data stream
-	HashTypeMd5 = "md5"
-	// HashTypeSha256 generates SHA256 checksum for the data stream
-	HashTypeSha256 = "sha256"
+	HashTypeMd5 HashType = "md5"
+	// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
+	HashTypeSha256 HashType = "sha256"
+	// HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream
+	HashTypeSha256Hex HashType = "sha256-hex"
+	// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
+	HashTypeSha1 HashType = "sha1"
+	// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
+	HashTypeCRC32 HashType = "crc32"
+	// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
+	HashTypeCRC32C HashType = "crc32c"
+	// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
+	HashTypeCRC64NVME HashType = "crc64nvme"
 	// HashTypeNone is a no-op checksum for the data stream
-	HashTypeNone = "none"
+	HashTypeNone HashType = "none"
 )
 
 // HashReader is an io.Reader that calculates the checksum
@@ -62,8 +77,18 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
 	switch ht {
 	case HashTypeMd5:
 		hash = md5.New()
+	case HashTypeSha256Hex:
+		hash = sha256.New()
 	case HashTypeSha256:
 		hash = sha256.New()
+	case HashTypeSha1:
+		hash = sha1.New()
+	case HashTypeCRC32:
+		hash = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeCRC64NVME:
+		hash = crc64.New(crc64NVMETable)
 	case HashTypeNone:
 		hash = noop{}
 	default:
@@ -88,15 +113,40 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	if errors.Is(readerr, io.EOF) && hr.sum != "" {
 		switch hr.hashType {
 		case HashTypeMd5:
-			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
 			}
-		case HashTypeSha256:
-			sum := hex.EncodeToString(hr.hash.Sum(nil))
+		case HashTypeSha256Hex:
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
 			}
+		case HashTypeCRC32:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32)
+			}
+		case HashTypeCRC32C:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c)
+			}
+		case HashTypeSha1:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1)
+			}
+		case HashTypeSha256:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
+			}
+		case HashTypeCRC64NVME:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
+			}
 		default:
 			return n, errInvalidHashType
 		}
@@ -104,20 +154,38 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	return n, readerr
 }
 
+func (hr *HashReader) SetReader(r io.Reader) {
+	hr.r = r
+}
+
 // Sum returns the checksum hash of the data read so far
 func (hr *HashReader) Sum() string {
 	switch hr.hashType {
 	case HashTypeMd5:
-		return Md5SumString(hr.hash.Sum(nil))
-	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256Hex:
 		return hex.EncodeToString(hr.hash.Sum(nil))
+	case HashTypeCRC32:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC32C:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha1:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC64NVME:
+		return Base64SumString(hr.hash.Sum(nil))
 	default:
 		return ""
 	}
 }
 
-// Md5SumString converts the hash bytes to the string checksum value
-func Md5SumString(b []byte) string {
+func (hr *HashReader) Type() HashType {
+	return hr.hashType
+}
+
+// Base64SumString converts the hash bytes to the b64 encoded string checksum value
+func Base64SumString(b []byte) string {
 	return base64.StdEncoding.EncodeToString(b)
 }
 
@@ -128,3 +196,161 @@ func (n noop) Sum(b []byte) []byte         { return []byte{} }
 func (n noop) Reset()                      {}
 func (n noop) Size() int                   { return 0 }
 func (n noop) BlockSize() int              { return 1 }
+
+// IsChecksumComposable tests if the final foll object crc can be calculated
+// based on the part crc values.
+func IsChecksumComposable(algo types.ChecksumAlgorithm) bool {
+	switch algo {
+	case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmCrc64nvme:
+		return true
+	default:
+		return false
+	}
+}
+
+// AddCRCChecksum calculates the composite CRC checksum after adding the part crc.
+// Only CRC32, CRC32C, and CRC64NVME are supported. The input checksums must be base64-encoded strings.
+func AddCRCChecksum(algo types.ChecksumAlgorithm, crc, partCrc string, partLen int64) (string, error) {
+	switch algo {
+	case types.ChecksumAlgorithmCrc32:
+		data, err := base64.StdEncoding.DecodeString(partCrc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode partCrc: %w", err)
+		}
+		if len(data) != 4 {
+			return "", fmt.Errorf("invalid crc32 part checksum length: %d", len(data))
+		}
+		currentCRC, err := base64.StdEncoding.DecodeString(crc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode crc: %w", err)
+		}
+		if len(currentCRC) != 4 {
+			return "", fmt.Errorf("invalid crc32 checksum length: %d", len(currentCRC))
+		}
+
+		currentVal := uint32(currentCRC[0])<<24 | uint32(currentCRC[1])<<16 | uint32(currentCRC[2])<<8 | uint32(currentCRC[3])
+		val := uint32(data[0])<<24 | uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3])
+		composite := crc32Combine(crc32.IEEE, currentVal, val, partLen)
+
+		out := []byte{
+			byte(composite >> 24),
+			byte(composite >> 16),
+			byte(composite >> 8),
+			byte(composite),
+		}
+		return base64.StdEncoding.EncodeToString(out), nil
+	case types.ChecksumAlgorithmCrc32c:
+		data, err := base64.StdEncoding.DecodeString(partCrc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode partCrc: %w", err)
+		}
+		if len(data) != 4 {
+			return "", fmt.Errorf("invalid crc32 part checksum length: %d", len(data))
+		}
+		currentCRC, err := base64.StdEncoding.DecodeString(crc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode crc: %w", err)
+		}
+		if len(currentCRC) != 4 {
+			return "", fmt.Errorf("invalid crc32 checksum length: %d", len(currentCRC))
+		}
+
+		currentVal := uint32(currentCRC[0])<<24 | uint32(currentCRC[1])<<16 | uint32(currentCRC[2])<<8 | uint32(currentCRC[3])
+		val := uint32(data[0])<<24 | uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3])
+		composite := crc32Combine(crc32.Castagnoli, currentVal, val, partLen)
+
+		// Convert composite to big-endian bytes
+		out := []byte{
+			byte(composite >> 24),
+			byte(composite >> 16),
+			byte(composite >> 8),
+			byte(composite),
+		}
+		return base64.StdEncoding.EncodeToString(out), nil
+	case types.ChecksumAlgorithmCrc64nvme:
+		data, err := base64.StdEncoding.DecodeString(partCrc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode partCrc: %w", err)
+		}
+		if len(data) != 8 {
+			return "", fmt.Errorf("invalid crc64 part checksum length: %d", len(data))
+		}
+		currentCRC, err := base64.StdEncoding.DecodeString(crc)
+		if err != nil {
+			return "", fmt.Errorf("base64 decode crc: %w", err)
+		}
+		if len(currentCRC) != 8 {
+			return "", fmt.Errorf("invalid crc64 checksum length: %d", len(currentCRC))
+		}
+
+		currentVal := uint64(currentCRC[0])<<56 | uint64(currentCRC[1])<<48 | uint64(currentCRC[2])<<40 | uint64(currentCRC[3])<<32 |
+			uint64(currentCRC[4])<<24 | uint64(currentCRC[5])<<16 | uint64(currentCRC[6])<<8 | uint64(currentCRC[7])
+		val := uint64(data[0])<<56 | uint64(data[1])<<48 | uint64(data[2])<<40 | uint64(data[3])<<32 |
+			uint64(data[4])<<24 | uint64(data[5])<<16 | uint64(data[6])<<8 | uint64(data[7])
+		composite := crc64Combine(crc64NVME, currentVal, val, partLen)
+
+		out := []byte{
+			byte(composite >> 56), byte(composite >> 48), byte(composite >> 40), byte(composite >> 32),
+			byte(composite >> 24), byte(composite >> 16), byte(composite >> 8), byte(composite),
+		}
+		return base64.StdEncoding.EncodeToString(out), nil
+	default:
+		return "", fmt.Errorf("composite checksum not supported for algorithm: %v", algo)
+	}
+}
+
+// NewCompositeChecksumReader initializes a composite checksum
+// processor, which decodes and validates the provided
+// checksums and returns the final checksum based on
+// the previous processings.
+//
+// The supported checksum types are:
+// - CRC32
+// - CRC32C
+// - SHA1
+// - SHA256
+func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
+	var hasher hash.Hash
+	switch ht {
+	case HashTypeSha256:
+		hasher = sha256.New()
+	case HashTypeSha1:
+		hasher = sha1.New()
+	case HashTypeCRC32:
+		hasher = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeNone:
+		hasher = noop{}
+	default:
+		return nil, errInvalidHashType
+	}
+
+	return &CompositeChecksumReader{
+		hasher: hasher,
+	}, nil
+}
+
+type CompositeChecksumReader struct {
+	hasher hash.Hash
+}
+
+// Decodes and writes the checksum in the hasher
+func (ccr *CompositeChecksumReader) Process(checksum string) error {
+	data, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		return fmt.Errorf("base64 decode: %w", err)
+	}
+
+	_, err = ccr.hasher.Write(data)
+	if err != nil {
+		return fmt.Errorf("hash write: %w", err)
+	}
+
+	return nil
+}
+
+// Returns the base64 encoded composite checksum
+func (ccr *CompositeChecksumReader) Sum() string {
+	return Base64SumString(ccr.hasher.Sum(nil))
+}

s3api/utils/csum-reader_test.go

@@ -0,0 +1,120 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"encoding/base64"
+	"hash/crc32"
+	"hash/crc64"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+func TestAddCRCChecksum_CRC32(t *testing.T) {
+	data := []byte("this is a test buffer for crc32")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	crc1 := crc32.Checksum(part1, crc32.IEEETable)
+	crc2 := crc32.Checksum(part2, crc32.IEEETable)
+	crcFull := crc32.Checksum(data, crc32.IEEETable)
+
+	crc1b := []byte{byte(crc1 >> 24), byte(crc1 >> 16), byte(crc1 >> 8), byte(crc1)}
+	crc2b := []byte{byte(crc2 >> 24), byte(crc2 >> 16), byte(crc2 >> 8), byte(crc2)}
+	crc1b64 := base64.StdEncoding.EncodeToString(crc1b)
+	crc2b64 := base64.StdEncoding.EncodeToString(crc2b)
+
+	combined, err := AddCRCChecksum(types.ChecksumAlgorithmCrc32, crc1b64, crc2b64, int64(len(part2)))
+	if err != nil {
+		t.Fatalf("AddCRCChecksum failed: %v", err)
+	}
+	combinedBytes, err := base64.StdEncoding.DecodeString(combined)
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	combinedVal := uint32(combinedBytes[0])<<24 | uint32(combinedBytes[1])<<16 | uint32(combinedBytes[2])<<8 | uint32(combinedBytes[3])
+	if combinedVal != crcFull {
+		t.Errorf("CRC32 combine mismatch: got %x, want %x", combinedVal, crcFull)
+	}
+}
+
+func TestAddCRCChecksum_CRC32c(t *testing.T) {
+	data := []byte("this is a test buffer for crc32c")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	castagnoli := crc32.MakeTable(crc32.Castagnoli)
+	crc1 := crc32.Checksum(part1, castagnoli)
+	crc2 := crc32.Checksum(part2, castagnoli)
+	crcFull := crc32.Checksum(data, castagnoli)
+
+	crc1b := []byte{byte(crc1 >> 24), byte(crc1 >> 16), byte(crc1 >> 8), byte(crc1)}
+	crc2b := []byte{byte(crc2 >> 24), byte(crc2 >> 16), byte(crc2 >> 8), byte(crc2)}
+	crc1b64 := base64.StdEncoding.EncodeToString(crc1b)
+	crc2b64 := base64.StdEncoding.EncodeToString(crc2b)
+
+	combined, err := AddCRCChecksum(types.ChecksumAlgorithmCrc32c, crc1b64, crc2b64, int64(len(part2)))
+	if err != nil {
+		t.Fatalf("AddCRCChecksum failed: %v", err)
+	}
+	combinedBytes, err := base64.StdEncoding.DecodeString(combined)
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	combinedVal := uint32(combinedBytes[0])<<24 | uint32(combinedBytes[1])<<16 | uint32(combinedBytes[2])<<8 | uint32(combinedBytes[3])
+	if combinedVal != crcFull {
+		t.Errorf("CRC32c combine mismatch: got %x, want %x", combinedVal, crcFull)
+	}
+}
+
+func TestAddCRCChecksum_CRC64NVME(t *testing.T) {
+	data := []byte("this is a test buffer for crc64nvme")
+	mid := len(data) / 2
+	part1 := data[:mid]
+	part2 := data[mid:]
+
+	table := crc64NVMETable
+	crc1 := crc64.Checksum(part1, table)
+	crc2 := crc64.Checksum(part2, table)
+	crcFull := crc64.Checksum(data, table)
+
+	crc1b := []byte{
+		byte(crc1 >> 56), byte(crc1 >> 48), byte(crc1 >> 40), byte(crc1 >> 32),
+		byte(crc1 >> 24), byte(crc1 >> 16), byte(crc1 >> 8), byte(crc1),
+	}
+	crc2b := []byte{
+		byte(crc2 >> 56), byte(crc2 >> 48), byte(crc2 >> 40), byte(crc2 >> 32),
+		byte(crc2 >> 24), byte(crc2 >> 16), byte(crc2 >> 8), byte(crc2),
+	}
+	crc1b64 := base64.StdEncoding.EncodeToString(crc1b)
+	crc2b64 := base64.StdEncoding.EncodeToString(crc2b)
+
+	combined, err := AddCRCChecksum(types.ChecksumAlgorithmCrc64nvme, crc1b64, crc2b64, int64(len(part2)))
+	if err != nil {
+		t.Fatalf("AddCRCChecksum failed: %v", err)
+	}
+	combinedBytes, err := base64.StdEncoding.DecodeString(combined)
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	combinedVal := uint64(combinedBytes[0])<<56 | uint64(combinedBytes[1])<<48 | uint64(combinedBytes[2])<<40 | uint64(combinedBytes[3])<<32 |
+		uint64(combinedBytes[4])<<24 | uint64(combinedBytes[5])<<16 | uint64(combinedBytes[6])<<8 | uint64(combinedBytes[7])
+	if combinedVal != crcFull {
+		t.Errorf("CRC64NVME combine mismatch: got %x, want %x", combinedVal, crcFull)
+	}
+}

s3api/utils/logger.go

@@ -1,55 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package utils
-
-import (
-	"fmt"
-	"log"
-
-	"github.com/gofiber/fiber/v2"
-)
-
-func LogCtxDetails(ctx *fiber.Ctx, respBody []byte) {
-	isDebug, ok := ctx.Locals("isDebug").(bool)
-	_, notLogReqBody := ctx.Locals("logReqBody").(bool)
-	_, notLogResBody := ctx.Locals("logResBody").(bool)
-	if isDebug && ok {
-		// Log request body
-		if !notLogReqBody {
-			fmt.Println()
-			log.Printf("Request Body: %s", ctx.Request().Body())
-		}
-
-		// Log path parameters
-		fmt.Println()
-		log.Println("Path parameters: ")
-		for key, val := range ctx.AllParams() {
-			log.Printf("%s: %s", key, val)
-		}
-
-		// Log response headers
-		fmt.Println()
-		log.Println("Response Headers: ")
-		ctx.Response().Header.VisitAll(func(key, val []byte) {
-			log.Printf("%s: %s", key, val)
-		})
-
-		// Log response body
-		if !notLogResBody && len(respBody) > 0 {
-			fmt.Println()
-			log.Printf("Response body %s", ctx.Response().Body())
-		}
-	}
-}

s3api/utils/name_validate.go

@@ -0,0 +1,24 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+func IsObjectNameValid(name string) bool {
+	switch clean(name) {
+	case "", ".", "..", "/":
+		return false
+	}
+
+	return isObjectLocal(name)
+}

s3api/utils/path.go

@@ -0,0 +1,171 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// code modified from golang std library src/internal/filepathlite/path.go
+// to support path separator '/' for all platforms.
+package utils
+
+import (
+	"strings"
+)
+
+const separator = '/'
+
+// isObjectLocal checks if the given path would result in an object
+// that is local to the bucket.
+func isObjectLocal(path string) bool {
+	if path == "" || path == "." {
+		return true
+	}
+
+	path = strings.Join([]string{".", path}, string(separator))
+
+	hasDots := false
+	for p := path; p != ""; {
+		var part string
+		part, p, _ = strings.Cut(p, "/")
+		if part == "." || part == ".." {
+			hasDots = true
+			break
+		}
+	}
+	if hasDots {
+		path = clean(path)
+	}
+	if path == ".." || strings.HasPrefix(path, "../") {
+		return false
+	}
+	return true
+}
+
+func clean(path string) string {
+	originalPath := path
+	if path == "" {
+		return originalPath + "."
+	}
+	rooted := isPathSeparator(path[0])
+
+	// Invariants:
+	//	reading from path; r is index of next byte to process.
+	//	writing to buf; w is index of next byte to write.
+	//	dotdot is index in buf where .. must stop, either because
+	//		it is the leading slash or it is a leading ../../.. prefix.
+	n := len(path)
+	out := lazybuf{path: path, volAndPath: originalPath, volLen: 0}
+	r, dotdot := 0, 0
+	if rooted {
+		out.append(separator)
+		r, dotdot = 1, 1
+	}
+
+	for r < n {
+		switch {
+		case isPathSeparator(path[r]):
+			// empty path element
+			r++
+		case path[r] == '.' && (r+1 == n || isPathSeparator(path[r+1])):
+			// . element
+			r++
+		case path[r] == '.' && path[r+1] == '.' && (r+2 == n || isPathSeparator(path[r+2])):
+			// .. element: remove to last separator
+			r += 2
+			switch {
+			case out.w > dotdot:
+				// can backtrack
+				out.w--
+				for out.w > dotdot && !isPathSeparator(out.index(out.w)) {
+					out.w--
+				}
+			case !rooted:
+				// cannot backtrack, but not rooted, so append .. element.
+				if out.w > 0 {
+					out.append(separator)
+				}
+				out.append('.')
+				out.append('.')
+				dotdot = out.w
+			}
+		default:
+			// real path element.
+			// add slash if needed
+			if rooted && out.w != 1 || !rooted && out.w != 0 {
+				out.append(separator)
+			}
+			// copy element
+			for ; r < n && !isPathSeparator(path[r]); r++ {
+				out.append(path[r])
+			}
+		}
+	}
+
+	// Turn empty string into "."
+	if out.w == 0 {
+		out.append('.')
+	}
+
+	return FromSlash(out.string())
+}
+
+func isPathSeparator(c uint8) bool {
+	return c == '/'
+}
+
+func FromSlash(path string) string {
+	if separator == '/' {
+		return path
+	}
+	return replaceStringByte(path, '/', separator)
+}
+
+func replaceStringByte(s string, old, new byte) string {
+	if strings.IndexByte(s, old) == -1 {
+		return s
+	}
+	n := []byte(s)
+	for i := range n {
+		if n[i] == old {
+			n[i] = new
+		}
+	}
+	return string(n)
+}
+
+// A lazybuf is a lazily constructed path buffer.
+// It supports append, reading previously appended bytes,
+// and retrieving the final string. It does not allocate a buffer
+// to hold the output until that output diverges from s.
+type lazybuf struct {
+	path       string
+	buf        []byte
+	w          int
+	volAndPath string
+	volLen     int
+}
+
+func (b *lazybuf) index(i int) byte {
+	if b.buf != nil {
+		return b.buf[i]
+	}
+	return b.path[i]
+}
+
+func (b *lazybuf) append(c byte) {
+	if b.buf == nil {
+		if b.w < len(b.path) && b.path[b.w] == c {
+			b.w++
+			return
+		}
+		b.buf = make([]byte, len(b.path))
+		copy(b.buf, b.path[:b.w])
+	}
+	b.buf[b.w] = c
+	b.w++
+}
+
+func (b *lazybuf) string() string {
+	if b.buf == nil {
+		return b.volAndPath[:b.volLen+b.w]
+	}
+	return b.volAndPath[:b.volLen] + string(b.buf[:b.w])
+}

s3api/utils/path_test.go

@@ -0,0 +1,64 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils_test
+
+import (
+	"testing"
+
+	"github.com/versity/versitygw/s3api/utils"
+)
+
+func TestIsObjectNameValid(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  bool
+	}{
+		// valid names
+		{"simple file", "file.txt", true},
+		{"nested file", "dir/file.txt", true},
+		{"absolute nested file", "/dir/file.txt", true},
+		{"trailing slash", "dir/", true},
+		{"slash prefix", "/file.txt", true}, // treated as local after joined with bucket
+		{"dot slash prefix", "./file.txt", true},
+
+		// invalid names
+		{"dot dot only", "..", false},
+		{"dot only", ".", false},
+		{"dot slash", "./", false},
+		{"dot slash dot dot", "./..", false},
+		{"cleans to dot", "./../.", false},
+		{"empty", "", false},
+		{"file escapes 1", "../file.txt", false},
+		{"file escapes 2", "dir/../../file.txt", false},
+		{"file escapes 3", "../../../file.txt", false},
+		{"dir escapes 1", "../dir/", false},
+		{"dir escapes 2", "dir/../../dir/", false},
+		{"dir escapes 3", "../../../dir/", false},
+		{"dot escapes 1", "../.", false},
+		{"dot escapes 2", "dir/../../.", false},
+		{"dot escapes 3", "../../../.", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := utils.IsObjectNameValid(tt.input)
+			if got != tt.want {
+				t.Errorf("%v: IsObjectNameValid(%q) = %v, want %v",
+					tt.name, tt.input, got, tt.want)
+			}
+		})
+	}
+}

s3api/utils/presign-auth-reader.go

@@ -41,10 +41,10 @@ const (
 // data requests where the data size is not known until
 // the data is completely read.
 type PresignedAuthReader struct {
-	r      io.Reader
 	ctx    *fiber.Ctx
 	auth   AuthData
 	secret string
+	r      io.Reader
 	debug  bool
 }
 
@@ -180,7 +180,7 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
 		return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
 	}
 
-	if ctx.Locals("region") != creds[2] {
+	if ContextKeyRegion.Get(ctx) != creds[2] {
 		return a, s3err.APIError{
 			Code:           "SignatureDoesNotMatch",
 			Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),

s3api/utils/presign-auth-reader_test.go

@@ -23,13 +23,13 @@ import (
 
 func Test_validateExpiration(t *testing.T) {
 	type args struct {
-		date time.Time
 		str  string
+		date time.Time
 	}
 	tests := []struct {
+		name string
 		args args
 		err  error
-		name string
 	}{
 		{
 			name: "empty-expiration",

s3api/utils/signed-chunk-reader.go

@@ -0,0 +1,533 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3api/debuglogger"
+	"github.com/versity/versitygw/s3err"
+)
+
+// chunked uploads described in:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+
+const (
+	chunkHdrDelim            = "\r\n"
+	zeroLenSig               = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	awsV4                    = "AWS4"
+	awsS3Service             = "s3"
+	awsV4Request             = "aws4_request"
+	trailerSignatureHeader   = "x-amz-trailer-signature"
+	streamPayloadAlgo        = "AWS4-HMAC-SHA256-PAYLOAD"
+	streamPayloadTrailerAlgo = "AWS4-HMAC-SHA256-TRAILER"
+)
+
+// ChunkReader reads from chunked upload request body, and returns
+// object data stream
+type ChunkReader struct {
+	r              io.Reader
+	signingKey     []byte
+	prevSig        string
+	parsedSig      string
+	chunkDataLeft  int64
+	trailer        checksumType
+	trailerSig     string
+	parsedChecksum string
+	stash          []byte
+	chunkHash      hash.Hash
+	checksumHash   hash.Hash
+	isEOF          bool
+	isFirstHeader  bool
+	region         string
+	date           time.Time
+}
+
+// NewChunkReader reads from request body io.Reader and parses out the
+// chunk metadata in stream. The headers are validated for proper signatures.
+// Reading from the chunk reader will read only the object data stream
+// without the chunk headers/trailers.
+func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time, chType checksumType) (io.Reader, error) {
+	chRdr := &ChunkReader{
+		r:          r,
+		signingKey: getSigningKey(secret, region, date),
+		// the authdata.Signature is validated in the auth-reader,
+		// so we can use that here without any other checks
+		prevSig:       authdata.Signature,
+		chunkHash:     sha256.New(),
+		isFirstHeader: true,
+		date:          date,
+		region:        region,
+		trailer:       chType,
+	}
+
+	if chType != "" {
+		checksumHasher, err := getHasher(chType)
+		if err != nil {
+			debuglogger.Logf("failed to initialize hash calculator: %v", err)
+			return nil, err
+		}
+
+		chRdr.checksumHash = checksumHasher
+	}
+	if chType == "" {
+		debuglogger.Infof("initializing signed chunk reader")
+	} else {
+		debuglogger.Infof("initializing signed chunk reader with '%v' trailing checksum", chType)
+	}
+	return chRdr, nil
+}
+
+// Read satisfies the io.Reader for this type
+func (cr *ChunkReader) Read(p []byte) (int, error) {
+	n, err := cr.r.Read(p)
+	if err != nil && err != io.EOF {
+		return 0, err
+	}
+
+	cr.isEOF = err == io.EOF
+
+	if cr.chunkDataLeft < int64(n) {
+		chunkSize := cr.chunkDataLeft
+		if chunkSize > 0 {
+			cr.chunkHash.Write(p[:chunkSize])
+			if cr.checksumHash != nil {
+				cr.checksumHash.Write(p[:chunkSize])
+			}
+		}
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		n += int(chunkSize)
+		return n, err
+	}
+
+	cr.chunkDataLeft -= int64(n)
+	cr.chunkHash.Write(p[:n])
+	if cr.checksumHash != nil {
+		cr.checksumHash.Write(p[:n])
+	}
+	return n, err
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// This part is the same for all chunks,
+// only the previous signature and hash of current chunk changes
+func (cr *ChunkReader) getStringToSignPrefix(algo string) string {
+	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
+		cr.date.Format("20060102"),
+		cr.region,
+		awsS3Service,
+		awsV4Request)
+
+	return fmt.Sprintf("%s\n%s\n%s",
+		algo,
+		cr.date.Format("20060102T150405Z"),
+		credentialScope)
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// signature For each chunk, you calculate the signature using the following
+// string to sign. For the first chunk, you use the seed-signature as the
+// previous signature.
+func (cr *ChunkReader) getChunkStringToSign() string {
+	prefix := cr.getStringToSignPrefix(streamPayloadAlgo)
+	chunkHash := cr.chunkHash.Sum(nil)
+	strToSign := fmt.Sprintf("%s\n%s\n%s\n%s",
+		prefix,
+		cr.prevSig,
+		zeroLenSig,
+		hex.EncodeToString(chunkHash))
+	debuglogger.PrintInsideHorizontalBorders(debuglogger.Purple, "STRING TO SIGN", strToSign, 64)
+	return strToSign
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
+// Builds the final chunk trailing signature string to sign
+func (cr *ChunkReader) getTrailerChunkStringToSign() string {
+	trailer := fmt.Sprintf("%v:%v\n", cr.trailer, cr.parsedChecksum)
+	hsh := sha256.Sum256([]byte(trailer))
+	sig := hex.EncodeToString(hsh[:])
+
+	prefix := cr.getStringToSignPrefix(streamPayloadTrailerAlgo)
+
+	strToSign := fmt.Sprintf("%s\n%s\n%s",
+		prefix,
+		cr.prevSig,
+		sig,
+	)
+
+	debuglogger.PrintInsideHorizontalBorders(debuglogger.Purple, "TRAILER STRING TO SIGN", strToSign, 64)
+
+	return strToSign
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
+// Calculates and validates the final chunk trailer signature
+func (cr *ChunkReader) verifyTrailerSignature() error {
+	strToSign := cr.getTrailerChunkStringToSign()
+	sig := hex.EncodeToString(hmac256(cr.signingKey, []byte(strToSign)))
+
+	if sig != cr.trailerSig {
+		debuglogger.Logf("incorrect trailing signature: (calculated): %v, (got): %v", sig, cr.trailerSig)
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+
+	return nil
+}
+
+// Verifies the object checksum
+func (cr *ChunkReader) verifyChecksum() error {
+	checksumHash := cr.checksumHash.Sum(nil)
+	checksum := base64.StdEncoding.EncodeToString(checksumHash)
+	if checksum != cr.parsedChecksum {
+		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(cr.trailer), "x-amz-checksum-")))
+		debuglogger.Logf("incorrect trailing checksum: (calculated): %v, (got): %v", checksum, cr.parsedChecksum)
+		return s3err.GetChecksumBadDigestErr(algo)
+	}
+
+	return nil
+}
+
+// Calculates and verifies the chunk signature
+func (cr *ChunkReader) checkSignature() error {
+	sigstr := cr.getChunkStringToSign()
+	cr.chunkHash.Reset()
+	cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
+
+	if cr.prevSig != cr.parsedSig {
+		debuglogger.Logf("incorrect signature: (calculated): %v, (got) %v", cr.prevSig, cr.parsedSig)
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+	cr.parsedSig = ""
+	return nil
+}
+
+// The provided p should have all of the previous chunk data and trailer
+// consumed already. The positioning here is expected that p[0] starts the
+// new chunk size with the ";chunk-signature=" following. The only exception
+// is if we started consuming the trailer, but hit the end of the read buffer.
+// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
+// finish consuming the final trailer bytes.
+// This parses the chunk metadata in situ without allocating an extra buffer.
+// It will just read and validate the chunk metadata and then move the
+// following chunk data to overwrite the metadata in the provided buffer.
+func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
+	n := len(p)
+
+	if cr.parsedSig != "" {
+		err := cr.checkSignature()
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
+	if err == errskipHeader {
+		cr.chunkDataLeft = 0
+		return 0, nil
+	}
+	if err != nil {
+		debuglogger.Logf("failed to parse chunk headers: %v", err)
+		return 0, err
+	}
+	cr.parsedSig = sig
+	// If we hit the final chunk, calculate and validate the final
+	// chunk signature and finish reading
+	if chunkSize == 0 {
+		debuglogger.Infof("final chunk parsed:\nchunk size: %v\nsignature: %v\nbuffer offset: %v", chunkSize, sig, bufOffset)
+		cr.chunkHash.Reset()
+		err := cr.checkSignature()
+		if err != nil {
+			return 0, err
+		}
+
+		if cr.trailer != "" {
+			debuglogger.Infof("final chunk trailers parsed:\nchecksum: %v\ntrailing signature: %v", cr.parsedChecksum, cr.trailerSig)
+			err := cr.verifyChecksum()
+			if err != nil {
+				return 0, err
+			}
+			err = cr.verifyTrailerSignature()
+			if err != nil {
+				return 0, err
+			}
+		}
+
+		return 0, io.EOF
+	}
+	debuglogger.Infof("chunk headers parsed:\nchunk size: %v\nsignature: %v\nbuffer offset: %v", chunkSize, sig, bufOffset)
+
+	// move data up to remove chunk header
+	copy(p, p[bufOffset:n])
+	n -= bufOffset
+
+	// if remaining buffer larger than chunk data,
+	// parse next header in buffer
+	if int64(n) > chunkSize {
+		cr.chunkDataLeft = 0
+		cr.chunkHash.Write(p[:chunkSize])
+		if cr.checksumHash != nil {
+			cr.checksumHash.Write(p[:chunkSize])
+		}
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		if (chunkSize + int64(n)) > math.MaxInt {
+			debuglogger.Logf("exceeding the limit of maximum integer allowed: (value): %v, (limit): %v", chunkSize+int64(n), math.MaxInt)
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
+		return n + int(chunkSize), err
+	}
+
+	cr.chunkDataLeft = chunkSize - int64(n)
+	cr.chunkHash.Write(p[:n])
+	if cr.checksumHash != nil {
+		cr.checksumHash.Write(p[:n])
+	}
+
+	return n, nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+// Task 3: Calculate Signature
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
+func getSigningKey(secret, region string, date time.Time) []byte {
+	dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
+	dateRegionKey := hmac256(dateKey, []byte(region))
+	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
+	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
+	debuglogger.Infof("signing key: %s", hex.EncodeToString(signingKey))
+	return signingKey
+}
+
+func hmac256(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+var (
+	errInvalidChunkFormat = errors.New("invalid chunk header format")
+	errskipHeader         = errors.New("skip to next header")
+)
+
+const (
+	maxHeaderSize = 1024
+)
+
+// This returns the chunk payload size, signature, data start offset, and
+// error if any. See the AWS documentation for the chunk header format. The
+// header[0] byte is expected to be the first byte of the chunk size here.
+func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
+	stashLen := len(cr.stash)
+	if stashLen > maxHeaderSize {
+		debuglogger.Logf("the stash length exceeds the maximum allowed chunk header size: (stash len): %v, (header limit): %v", stashLen, maxHeaderSize)
+		return 0, "", 0, errInvalidChunkFormat
+	}
+	if cr.stash != nil {
+		debuglogger.Logf("recovering the stash: (stash len): %v", stashLen)
+		tmp := make([]byte, stashLen+len(header))
+		copy(tmp, cr.stash)
+		copy(tmp[len(cr.stash):], header)
+		header = tmp
+		cr.stash = nil
+	}
+
+	rdr := bufio.NewReader(bytes.NewReader(header))
+
+	// After the first chunk each chunk header should start
+	// with "\n\r\n"
+	if !cr.isFirstHeader {
+		err := readAndSkip(rdr, '\r', '\n')
+		if err != nil {
+			debuglogger.Logf("failed to read chunk header first 2 bytes: (should be): \\r\\n, (got): %q", header[:2])
+			return cr.handleRdrErr(err, header)
+		}
+	}
+
+	// read and parse the chunk size
+	chunkSizeStr, err := readAndTrim(rdr, ';')
+	if err != nil {
+		debuglogger.Logf("failed to read chunk size: %v", err)
+		return cr.handleRdrErr(err, header)
+	}
+	chunkSize, err := strconv.ParseInt(chunkSizeStr, 16, 64)
+	if err != nil {
+		debuglogger.Logf("failed to parse chunk size: (size): %v, (err): %v", chunkSizeStr, err)
+		return 0, "", 0, errInvalidChunkFormat
+	}
+
+	// read the chunk signature
+	err = readAndSkip(rdr, 'c', 'h', 'u', 'n', 'k', '-', 's', 'i', 'g', 'n', 'a', 't', 'u', 'r', 'e', '=')
+	if err != nil {
+		debuglogger.Logf("failed to read 'chunk-signature=': %v", err)
+		return cr.handleRdrErr(err, header)
+	}
+	sig, err := readAndTrim(rdr, '\r')
+	if err != nil {
+		debuglogger.Logf("failed to read '\\r', after chunk signature: %v", err)
+		return cr.handleRdrErr(err, header)
+	}
+
+	// read and parse the final chunk trailer and checksum
+	if chunkSize == 0 {
+		if cr.trailer != "" {
+			err = readAndSkip(rdr, '\n')
+			if err != nil {
+				debuglogger.Logf("failed to read \\n before the trailer: %v", err)
+				return cr.handleRdrErr(err, header)
+			}
+			// parse and validate the trailing header
+			trailer, err := readAndTrim(rdr, ':')
+			if err != nil {
+				debuglogger.Logf("failed to read trailer prefix: %v", err)
+				return cr.handleRdrErr(err, header)
+			}
+			if trailer != string(cr.trailer) {
+				debuglogger.Logf("incorrect trailer prefix: (expected): %v, (got): %v", cr.trailer, trailer)
+				return 0, "", 0, errInvalidChunkFormat
+			}
+
+			algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(trailer, "x-amz-checksum-")))
+
+			// parse the checksum
+			checksum, err := readAndTrim(rdr, '\r')
+			if err != nil {
+				debuglogger.Logf("failed to read checksum value: %v", err)
+				return cr.handleRdrErr(err, header)
+			}
+
+			if !IsValidChecksum(checksum, algo) {
+				debuglogger.Logf("invalid checksum value: %v", checksum)
+				return 0, "", 0, s3err.GetInvalidTrailingChecksumHeaderErr(trailer)
+			}
+
+			err = readAndSkip(rdr, '\n')
+			if err != nil {
+				debuglogger.Logf("failed to read \\n after checksum: %v", err)
+				return cr.handleRdrErr(err, header)
+			}
+
+			// parse the trailing signature
+			trailerSigPrefix, err := readAndTrim(rdr, ':')
+			if err != nil {
+				debuglogger.Logf("failed to read trailing signature prefix: %v", err)
+				return cr.handleRdrErr(err, header)
+			}
+
+			if trailerSigPrefix != trailerSignatureHeader {
+				debuglogger.Logf("invalid trailing signature prefix: (expected): %v, (got): %v", trailerSignatureHeader, trailerSigPrefix)
+				return 0, "", 0, errInvalidChunkFormat
+			}
+
+			trailerSig, err := readAndTrim(rdr, '\r')
+			if err != nil {
+				debuglogger.Logf("failed to read trailing signature: %v", err)
+				return cr.handleRdrErr(err, header)
+			}
+
+			cr.trailerSig = trailerSig
+			cr.parsedChecksum = checksum
+		}
+
+		// "\r\n\r\n" is followed after the last chunk
+		err = readAndSkip(rdr, '\n', '\r', '\n')
+		if err != nil {
+			debuglogger.Logf("failed to read \\n\\r\\n at the end of chunk header: %v", err)
+			return cr.handleRdrErr(err, header)
+		}
+
+		return 0, sig, 0, nil
+	}
+
+	err = readAndSkip(rdr, '\n')
+	if err != nil {
+		debuglogger.Logf("failed to read \\n at the end of chunk header: %v", err)
+		return cr.handleRdrErr(err, header)
+	}
+
+	// find the index of chunk ending: '\r\n'
+	// skip the first 2 bytes as it is the starting '\r\n'
+	// the first chunk doesn't contain the starting '\r\n', but
+	// anyway, trimming the first 2 bytes doesn't pollute the logic.
+	ind := bytes.Index(header[2:], []byte{'\r', '\n'})
+	cr.isFirstHeader = false
+
+	// the offset is the found index + 4 - the stash length
+	// where:
+	// ind is the index of '\r\n'
+	// 4 specifies the trimmed 2 bytes plus 2 to shift the index at the end of '\r\n'
+	offset := ind + 4 - stashLen
+	return chunkSize, sig, offset, nil
+}
+
+// Stashes the header in cr.stash and returns "errskipHeader"
+func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, error) {
+	cr.stash = make([]byte, len(header))
+	copy(cr.stash, header)
+	debuglogger.Logf("stashing the header: (header length): %v", len(header))
+	return 0, "", 0, errskipHeader
+}
+
+// Returns "errInvalidChunkFormat" if the passed err is "io.EOF" and cr.rdr EOF is reached
+// calls "cr.stashAndSkipHeader" if the passed err is "io.EOF" and cr.isEOF is false
+// Returns the error otherwise
+func (cr *ChunkReader) handleRdrErr(err error, header []byte) (int64, string, int, error) {
+	if err == io.EOF {
+		if cr.isEOF {
+			debuglogger.Logf("incomplete chunk encoding, EOF reached")
+			return 0, "", 0, errInvalidChunkFormat
+		}
+		return cr.stashAndSkipHeader(header)
+	}
+	return 0, "", 0, err
+}
+
+// reads data from the "rdr" and validates the passed data bytes
+func readAndSkip(rdr *bufio.Reader, data ...byte) error {
+	for _, d := range data {
+		b, err := rdr.ReadByte()
+		if err != nil {
+			return err
+		}
+
+		if b != d {
+			return errMalformedEncoding
+		}
+	}
+
+	return nil
+}
+
+// reads string by "delim" and trims the delimiter at the end
+func readAndTrim(r *bufio.Reader, delim byte) (string, error) {
+	str, err := r.ReadString(delim)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSuffix(str, string(delim)), nil
+}