Merge pull request #1240 from versity/sis/fix-signed-chunk-encoding-stashing

fix: fixes the signed chunk encoding reader stashing.

.github/workflows/go.yml

@@ -3,7 +3,7 @@ on: pull_request
 jobs:
 
   build:
-    name: Build
+    name: Go Basic Checks
     runs-on: ubuntu-latest
     steps:
 
@@ -23,9 +23,6 @@ jobs:
       run: |
         go get -v -t -d ./...
 
-    - name: Build
-      run: make
-
     - name: Test
       run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
 
@@ -35,4 +32,26 @@ jobs:
 
     - name: Run govulncheck
       run: govulncheck ./...
-      shell: bash
+      shell: bash
+
+  verify-build:
+    name: Verify Build Targets
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [darwin, freebsd, linux]
+        arch: [amd64, arm64]
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+
+      - name: Build for ${{ matrix.os }}/${{ matrix.arch }}
+        run: |
+          GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o versitygw-${{ matrix.os }}-${{ matrix.arch }} cmd/versitygw/*.go

.github/workflows/system.yml

@@ -12,66 +12,79 @@ jobs:
             IAM_TYPE: folder
             RUN_SET: "mc-non-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "mc, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "mc-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "REST, posix, non-static, all, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "rest"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3-non-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, policy, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, user, non-static, s3 IAM"
             IAM_TYPE: s3
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3api, posix, bucket, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-bucket"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, multipart, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-multipart"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, object, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-object"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, policy, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           - set: "s3api, posix, user, static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
             BACKEND: "posix"
           # TODO fix/debug s3 gateway
           #- set: "s3api, s3, multipart|object, non-static, folder IAM"
@@ -88,16 +101,19 @@ jobs:
             IAM_TYPE: folder
             RUN_SET: "s3cmd-file-count"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3cmd, posix, non-user, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-non-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
           - set: "s3cmd, posix, user, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-user"
             RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
             BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
@@ -150,6 +166,7 @@ jobs:
           RUN_VERSITYGW: true
           BACKEND: ${{ matrix.BACKEND }}
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
+          DELETE_BUCKETS_AFTER_TEST: ${{ matrix.DELETE_BUCKETS_AFTER_TEST }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
           LOCAL_FOLDER: /tmp/gw
@@ -172,6 +189,9 @@ jobs:
           VERSIONING_DIR: ${{ github.workspace }}/versioning
           COMMAND_LOG: command.log
           TIME_LOG: time.log
+          PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
+          AUTOGENERATE_USERS: true
+          USER_AUTOGENERATION_PREFIX: github-actions-test-
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST

README.md

@@ -40,7 +40,7 @@ Versity Gateway, a simple to use tool for seamless inline translation between AW
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
 
-The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versity’s open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.  
 
 The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
 

auth/acl.go

@@ -193,14 +193,25 @@ func ParseACL(data []byte) (ACL, error) {
 	return acl, nil
 }
 
-func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
+func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
+	grants := []Grant{}
+
+	if len(data) == 0 {
+		return GetBucketAclOutput{
+			Owner: &types.Owner{
+				ID: &owner,
+			},
+			AccessControlList: AccessControlList{
+				Grants: grants,
+			},
+		}, nil
+	}
+
 	var acl ACL
 	if err := json.Unmarshal(data, &acl); err != nil {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
-	grants := []Grant{}
-
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
 		grants = append(grants, Grant{

auth/bucket_policy.go

@@ -22,20 +22,46 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
-var (
-	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidResource = errors.New("Policy has invalid resource")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidPrincipal = errors.New("Invalid principal in policy")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidAction = errors.New("Policy has invalid action")
+type policyErr string
+
+func (p policyErr) Error() string {
+	return string(p)
+}
+
+const (
+	policyErrResourceMismatch     = policyErr("Action does not apply to any resource(s) in statement")
+	policyErrInvalidResource      = policyErr("Policy has invalid resource")
+	policyErrInvalidPrincipal     = policyErr("Invalid principal in policy")
+	policyErrInvalidAction        = policyErr("Policy has invalid action")
+	policyErrInvalidPolicy        = policyErr("This policy contains invalid Json")
+	policyErrInvalidFirstChar     = policyErr("Policies must be valid JSON and the first byte must be '{'")
+	policyErrEmptyStatement       = policyErr("Could not parse the policy: Statement is empty!")
+	policyErrMissingStatmentField = policyErr("Missing required field Statement")
 )
 
 type BucketPolicy struct {
 	Statement []BucketPolicyItem `json:"Statement"`
 }
 
+func (bp *BucketPolicy) UnmarshalJSON(data []byte) error {
+	var tmp struct {
+		Statement *[]BucketPolicyItem `json:"Statement"`
+	}
+
+	if err := json.Unmarshal(data, &tmp); err != nil {
+		return err
+	}
+
+	// If Statement is nil (not present in JSON), return an error
+	if tmp.Statement == nil {
+		return policyErrMissingStatmentField
+	}
+
+	// Assign the parsed value to the actual struct
+	bp.Statement = *tmp.Statement
+	return nil
+}
+
 func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 	for _, statement := range bp.Statement {
 		err := statement.Validate(bucket, iam)
@@ -48,18 +74,19 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 }
 
 func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	var isAllowed bool
 	for _, statement := range bp.Statement {
 		if statement.findMatch(principal, action, resource) {
 			switch statement.Effect {
 			case BucketPolicyAccessTypeAllow:
-				return true
+				isAllowed = true
 			case BucketPolicyAccessTypeDeny:
 				return false
 			}
 		}
 	}
 
-	return false
+	return isAllowed
 }
 
 type BucketPolicyItem struct {
@@ -89,10 +116,10 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 			break
 		}
 		if *isObjectAction && !containsObjectAction {
-			return errResourceMismatch
+			return policyErrResourceMismatch
 		}
 		if !*isObjectAction && !containsBucketAction {
-			return errResourceMismatch
+			return policyErrResourceMismatch
 		}
 	}
 
@@ -116,14 +143,20 @@ func getMalformedPolicyError(err error) error {
 }
 
 func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
+	if len(policyBin) == 0 || policyBin[0] != '{' {
+		return getMalformedPolicyError(policyErrInvalidFirstChar)
+	}
 	var policy BucketPolicy
 	if err := json.Unmarshal(policyBin, &policy); err != nil {
-		return getMalformedPolicyError(err)
+		var pe policyErr
+		if errors.As(err, &pe) {
+			return getMalformedPolicyError(err)
+		}
+		return getMalformedPolicyError(policyErrInvalidPolicy)
 	}
 
 	if len(policy.Statement) == 0 {
-		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
+		return getMalformedPolicyError(policyErrEmptyStatement)
 	}
 
 	if err := policy.Validate(bucket, iam); err != nil {

auth/bucket_policy_actions.go

@@ -58,6 +58,8 @@ const (
 	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
 	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
 	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
+	PutBucketCorsAction                    Action = "s3:PutBucketCORS"
+	GetBucketCorsAction                    Action = "s3:GetBucketCORS"
 	AllActions                             Action = "s3:*"
 )
 
@@ -97,6 +99,8 @@ var supportedActionList = map[Action]struct{}{
 	BypassGovernanceRetentionAction:        {},
 	PutBucketOwnershipControlsAction:       {},
 	GetBucketOwnershipControlsAction:       {},
+	PutBucketCorsAction:                    {},
+	GetBucketCorsAction:                    {},
 	AllActions:                             {},
 }
 
@@ -125,7 +129,7 @@ var supportedObjectActionList = map[Action]struct{}{
 // Validates Action: it should either wildcard match with supported actions list or be in it
 func (a Action) IsValid() error {
 	if !strings.HasPrefix(string(a), "s3:") {
-		return errInvalidAction
+		return policyErrInvalidAction
 	}
 
 	if a == AllActions {
@@ -140,12 +144,12 @@ func (a Action) IsValid() error {
 			}
 		}
 
-		return errInvalidAction
+		return policyErrInvalidAction
 	}
 
 	_, found := supportedActionList[a]
 	if !found {
-		return errInvalidAction
+		return policyErrInvalidAction
 	}
 	return nil
 }
@@ -191,7 +195,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return errInvalidAction
+			return policyErrInvalidAction
 		}
 		*a = make(Actions)
 		for _, s := range ss {
@@ -204,7 +208,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return errInvalidAction
+				return policyErrInvalidAction
 			}
 			*a = make(Actions)
 			err = a.Add(s)

auth/bucket_policy_principals.go

@@ -36,7 +36,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return errInvalidPrincipal
+			return policyErrInvalidPrincipal
 		}
 		*p = make(Principals)
 		for _, s := range ss {
@@ -45,7 +45,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &s); err == nil {
 		if s == "" {
-			return errInvalidPrincipal
+			return policyErrInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(s)
@@ -53,7 +53,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &k); err == nil {
 		if k.AWS == "" {
-			return errInvalidPrincipal
+			return policyErrInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(k.AWS)
@@ -65,7 +65,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		}
 		if err = json.Unmarshal(data, &sk); err == nil {
 			if len(sk.AWS) == 0 {
-				return errInvalidPrincipal
+				return policyErrInvalidPrincipal
 			}
 			*p = make(Principals)
 			for _, s := range sk.AWS {
@@ -97,7 +97,7 @@ func (p Principals) Validate(iam IAMService) error {
 		if len(p) == 1 {
 			return nil
 		}
-		return errInvalidPrincipal
+		return policyErrInvalidPrincipal
 	}
 
 	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -105,7 +105,7 @@ func (p Principals) Validate(iam IAMService) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return errInvalidPrincipal
+		return policyErrInvalidPrincipal
 	}
 
 	return nil

auth/bucket_policy_resources.go

@@ -29,7 +29,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return errInvalidResource
+			return policyErrInvalidResource
 		}
 		*r = make(Resources)
 		for _, s := range ss {
@@ -42,7 +42,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return errInvalidResource
+				return policyErrInvalidResource
 			}
 			*r = make(Resources)
 			err = r.Add(s)
@@ -59,7 +59,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 func (r Resources) Add(rc string) error {
 	ok, pattern := isValidResource(rc)
 	if !ok {
-		return errInvalidResource
+		return policyErrInvalidResource
 	}
 
 	r[pattern] = struct{}{}
@@ -93,7 +93,7 @@ func (r Resources) ContainsBucketPattern() bool {
 func (r Resources) Validate(bucket string) error {
 	for resource := range r {
 		if !strings.HasPrefix(resource, bucket) {
-			return errInvalidResource
+			return policyErrInvalidResource
 		}
 	}
 
@@ -102,21 +102,45 @@ func (r Resources) Validate(bucket string) error {
 
 func (r Resources) FindMatch(resource string) bool {
 	for res := range r {
-		if strings.HasSuffix(res, "*") {
-			pattern := strings.TrimSuffix(res, "*")
-			if strings.HasPrefix(resource, pattern) {
-				return true
-			}
-		} else {
-			if res == resource {
-				return true
-			}
+		if r.Match(res, resource) {
+			return true
 		}
 	}
 
 	return false
 }
 
+// Match checks if the input string matches the given pattern with wildcards (`*`, `?`).
+// - `?` matches exactly one occurrence of any character.
+// - `*` matches arbitrary many (including zero) occurrences of any character.
+func (r Resources) Match(pattern, input string) bool {
+	pIdx, sIdx := 0, 0
+	starIdx, matchIdx := -1, 0
+
+	for sIdx < len(input) {
+		if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
+			sIdx++
+			pIdx++
+		} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
+			starIdx = pIdx
+			matchIdx = sIdx
+			pIdx++
+		} else if starIdx != -1 {
+			pIdx = starIdx + 1
+			matchIdx++
+			sIdx = matchIdx
+		} else {
+			return false
+		}
+	}
+
+	for pIdx < len(pattern) && pattern[pIdx] == '*' {
+		pIdx++
+	}
+
+	return pIdx == len(pattern)
+}
+
 // Checks the resource to have arn prefix and not starting with /
 func isValidResource(rc string) (isValid bool, pattern string) {
 	if !strings.HasPrefix(rc, ResourceArnPrefix) {

auth/bucket_policy_resources_test.go

@@ -0,0 +1,182 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestUnmarshalJSON(t *testing.T) {
+	var r Resources
+
+	cases := []struct {
+		input    string
+		expected int
+		wantErr  bool
+	}{
+		{`"arn:aws:s3:::my-bucket/*"`, 1, false},
+		{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
+		{`""`, 0, true},
+		{`[]`, 0, true},
+		{`["invalid-bucket"]`, 0, true},
+	}
+
+	for _, tc := range cases {
+		r = Resources{}
+		err := json.Unmarshal([]byte(tc.input), &r)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+		if len(r) != tc.expected {
+			t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
+		}
+	}
+}
+
+func TestAdd(t *testing.T) {
+	r := Resources{}
+
+	cases := []struct {
+		input   string
+		wantErr bool
+	}{
+		{"arn:aws:s3:::valid-bucket/*", false},
+		{"arn:aws:s3:::valid-bucket/object", false},
+		{"invalid-bucket/*", true},
+		{"/invalid-start", true},
+	}
+
+	for _, tc := range cases {
+		err := r.Add(tc.input)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+	}
+}
+
+func TestContainsObjectPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, true},
+		{[]string{"arn:aws:s3:::my-bucket"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsObjectPattern() != tc.expected {
+			t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestContainsBucketPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsBucketPattern() != tc.expected {
+			t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestValidate(t *testing.T) {
+	cases := []struct {
+		resources []string
+		bucket    string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
+		{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
+		{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if (r.Validate(tc.bucket) == nil) != tc.expected {
+			t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
+		}
+	}
+}
+
+func TestFindMatch(t *testing.T) {
+	cases := []struct {
+		resources []string
+		input     string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
+		{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.FindMatch(tc.input) != tc.expected {
+			t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
+		}
+	}
+}
+
+func TestMatch(t *testing.T) {
+	r := Resources{}
+	cases := []struct {
+		pattern  string
+		input    string
+		expected bool
+	}{
+		{"my-bucket/*", "my-bucket/object", true},
+		{"my-bucket/?bject", "my-bucket/object", true},
+		{"my-bucket/*", "other-bucket/object", false},
+		{"*", "any-bucket/object", true},
+		{"my-bucket/*", "my-bucket/subdir/object", true},
+		{"my-bucket/*", "other-bucket", false},
+		{"my-bucket/*/*", "my-bucket/hello", false},
+		{"my-bucket/*/*", "my-bucket/hello/world", true},
+		{"foo/???/bar", "foo/qux/bar", true},
+		{"foo/???/bar", "foo/quxx/bar", false},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
+	}
+	for _, tc := range cases {
+		if r.Match(tc.pattern, tc.input) != tc.expected {
+			t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
+		}
+	}
+}

auth/iam.go

@@ -161,7 +161,7 @@ func New(o *Opts) (IAMService, error) {
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")
-		return IAMServiceSingle{}, nil
+		return NewIAMServiceSingle(o.RootAccount), nil
 	}
 
 	if err != nil {

auth/iam_ldap.go

@@ -111,11 +111,13 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 	entry := result.Entries[0]
 	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
 	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
+		return Account{}, fmt.Errorf("invalid entry value for group-id %q: %w",
+			entry.GetAttributeValue(ld.groupIdAtr), err)
 	}
 	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
 	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
+		return Account{}, fmt.Errorf("invalid entry value for user-id %q: %w",
+			entry.GetAttributeValue(ld.userIdAtr), err)
 	}
 	return Account{
 		Access:  entry.GetAttributeValue(ld.accessAtr),
@@ -183,11 +185,13 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 	for _, el := range resp.Entries {
 		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
 		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
+			return nil, fmt.Errorf("invalid entry value for group-id %q: %w",
+				el.GetAttributeValue(ld.groupIdAtr), err)
 		}
 		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
 		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
+			return nil, fmt.Errorf("invalid entry value for user-id %q: %w",
+				el.GetAttributeValue(ld.userIdAtr), err)
 		}
 		result = append(result, Account{
 			Access:  el.GetAttributeValue(ld.accessAtr),

auth/iam_single.go

@@ -19,18 +19,30 @@ import (
 )
 
 // IAMServiceSingle manages the single tenant (root-only) IAM service
-type IAMServiceSingle struct{}
+type IAMServiceSingle struct {
+	root Account
+}
 
 var _ IAMService = &IAMServiceSingle{}
 
+func NewIAMServiceSingle(r Account) IAMService {
+	return &IAMServiceSingle{
+		root: r,
+	}
+}
+
 // CreateAccount not valid in single tenant mode
 func (IAMServiceSingle) CreateAccount(account Account) error {
 	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
-// GetUserAccount no accounts in single tenant mode
-func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	return Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+// GetUserAccount returns root account, if the root access key
+// is provided and "ErrAdminUserNotFound" otherwise
+func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	if access == s.root.Access {
+		return s.root, nil
+	}
+	return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound)
 }
 
 // UpdateUserAccount no accounts in single tenant mode

auth/object_lock.go

@@ -95,7 +95,7 @@ func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfigur
 func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
 	var retention s3response.PutObjectRetentionInput
 	if err := xml.Unmarshal(input, &retention); err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
 	}
 
 	if retention.RetainUntilDate.Before(time.Now()) {
@@ -120,18 +120,18 @@ func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, e
 	return &retention, nil
 }
 
-func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
+func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResult {
 	if status == nil {
 		return nil
 	}
 
 	if *status {
-		return &types.ObjectLockLegalHold{
+		return &s3response.GetObjectLegalHoldResult{
 			Status: types.ObjectLockLegalHoldStatusOn,
 		}
 	}
 
-	return &types.ObjectLockLegalHold{
+	return &s3response.GetObjectLegalHoldResult{
 		Status: types.ObjectLockLegalHoldStatusOff,
 	}
 }

backend/azure/azure.go

@@ -27,7 +27,6 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -64,6 +63,7 @@ const (
 	keyBucketLock          key = "Bucketlock"
 	keyObjRetention        key = "Objectretention"
 	keyObjLegalHold        key = "Objectlegalhold"
+	keyExpires             key = "Vgwexpires"
 	onameAttr              key = "Objname"
 	onameAttrLower         key = "objname"
 	metaTmpMultipartPrefix key = ".sgwtmp" + "/multipart"
@@ -77,6 +77,7 @@ func (key) Table() map[string]struct{} {
 		"policy":            {},
 		"bucketlock":        {},
 		"objectretention":   {},
+		"vgwexpires":        {},
 		"objectlegalhold":   {},
 		"objname":           {},
 		".sgwtmp/multipart": {},
@@ -293,14 +294,27 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin
 	return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership))
 }
 
-func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 	tags, err := parseTags(po.Tagging)
 	if err != nil {
 		return s3response.PutObjectOutput{}, err
 	}
 
+	metadata := parseMetadata(po.Metadata)
+
+	// Store the "Expires" property in the object metadata
+	if getString(po.Expires) != "" {
+		if metadata == nil {
+			metadata = map[string]*string{
+				string(keyExpires): po.Expires,
+			}
+		} else {
+			metadata[string(keyExpires)] = po.Expires
+		}
+	}
+
 	opts := &blockblob.UploadStreamOptions{
-		Metadata: parseMetadata(po.Metadata),
+		Metadata: metadata,
 		Tags:     tags,
 	}
 
@@ -308,6 +322,8 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respon
 	opts.HTTPHeaders.BlobContentEncoding = po.ContentEncoding
 	opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage
 	opts.HTTPHeaders.BlobContentDisposition = po.ContentDisposition
+	opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage
+	opts.HTTPHeaders.BlobCacheControl = po.CacheControl
 	if strings.HasSuffix(*po.Key, "/") {
 		// Hardcode "application/x-directory" for direcoty objects
 		opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DirContentType)
@@ -390,17 +406,29 @@ func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error {
 }
 
 func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	client, err := az.getBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
 	var opts *azblob.DownloadStreamOptions
 	if *input.Range != "" {
-		offset, count, err := backend.ParseRange(0, *input.Range)
+		offset, count, isValid, err := backend.ParseGetObjectRange(*resp.ContentLength, *input.Range)
 		if err != nil {
 			return nil, err
 		}
-		opts = &azblob.DownloadStreamOptions{
-			Range: blob.HTTPRange{
-				Count:  count,
-				Offset: offset,
-			},
+		if isValid {
+			opts = &azblob.DownloadStreamOptions{
+				Range: blob.HTTPRange{
+					Count:  count,
+					Offset: offset,
+				},
+			}
 		}
 	}
 	blobDownloadResponse, err := az.client.DownloadStream(ctx, *input.Bucket, *input.Key, opts)
@@ -419,17 +447,21 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G
 	}
 
 	return &s3.GetObjectOutput{
-		AcceptRanges:    input.Range,
-		ContentLength:   blobDownloadResponse.ContentLength,
-		ContentEncoding: blobDownloadResponse.ContentEncoding,
-		ContentType:     contentType,
-		ETag:            (*string)(blobDownloadResponse.ETag),
-		LastModified:    blobDownloadResponse.LastModified,
-		Metadata:        parseAzMetadata(blobDownloadResponse.Metadata),
-		TagCount:        &tagcount,
-		ContentRange:    blobDownloadResponse.ContentRange,
-		Body:            blobDownloadResponse.Body,
-		StorageClass:    types.StorageClassStandard,
+		AcceptRanges:       backend.GetPtrFromString("bytes"),
+		ContentLength:      blobDownloadResponse.ContentLength,
+		ContentEncoding:    blobDownloadResponse.ContentEncoding,
+		ContentType:        contentType,
+		ContentDisposition: blobDownloadResponse.ContentDisposition,
+		ContentLanguage:    blobDownloadResponse.ContentLanguage,
+		CacheControl:       blobDownloadResponse.CacheControl,
+		ExpiresString:      blobDownloadResponse.Metadata[string(keyExpires)],
+		ETag:               (*string)(blobDownloadResponse.ETag),
+		LastModified:       blobDownloadResponse.LastModified,
+		Metadata:           parseAndFilterAzMetadata(blobDownloadResponse.Metadata),
+		TagCount:           &tagcount,
+		ContentRange:       blobDownloadResponse.ContentRange,
+		Body:               blobDownloadResponse.Body,
+		StorageClass:       types.StorageClassStandard,
 	}, nil
 }
 
@@ -483,10 +515,11 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 		ContentEncoding:    resp.ContentEncoding,
 		ContentLanguage:    resp.ContentLanguage,
 		ContentDisposition: resp.ContentDisposition,
+		CacheControl:       resp.CacheControl,
+		ExpiresString:      resp.Metadata[string(keyExpires)],
 		ETag:               (*string)(resp.ETag),
 		LastModified:       resp.LastModified,
-		Metadata:           parseAzMetadata(resp.Metadata),
-		Expires:            resp.ExpiresOn,
+		Metadata:           parseAndFilterAzMetadata(resp.Metadata),
 		StorageClass:       types.StorageClassStandard,
 	}
 
@@ -521,7 +554,7 @@ func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAtt
 	}
 
 	return s3response.GetObjectAttributesResponse{
-		ETag:         data.ETag,
+		ETag:         backend.TrimEtag(data.ETag),
 		ObjectSize:   data.ContentLength,
 		StorageClass: data.StorageClass,
 		LastModified: data.LastModified,
@@ -551,6 +584,18 @@ func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s
 		maxKeys = *input.MaxKeys
 	}
 
+	// Retrieve the bucket acl to get the bucket owner
+	// All the objects in the bucket are owner by the bucket owner
+	aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
+	if err != nil {
+		return s3response.ListObjectsResult{}, azureErrToS3Err(err)
+	}
+
+	var acl auth.ACL
+	if err := json.Unmarshal(aclBytes, &acl); err != nil {
+		return s3response.ListObjectsResult{}, fmt.Errorf("unmarshal acl: %w", err)
+	}
+
 Pager:
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
@@ -564,11 +609,14 @@ Pager:
 				break Pager
 			}
 			objects = append(objects, s3response.Object{
-				ETag:         (*string)(v.Properties.ETag),
+				ETag:         backend.GetPtrFromString(fmt.Sprintf("%q", *v.Properties.ETag)),
 				Key:          v.Name,
 				LastModified: v.Properties.LastModified,
 				Size:         v.Properties.ContentLength,
 				StorageClass: types.ObjectStorageClassStandard,
+				Owner: &types.Owner{
+					ID: &acl.Owner,
+				},
 			})
 		}
 		for _, v := range resp.Segment.BlobPrefixes {
@@ -628,10 +676,28 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input
 	var nextMarker *string
 	var isTruncated bool
 	var maxKeys int32 = math.MaxInt32
+	var fetchOwner bool
 
 	if input.MaxKeys != nil {
 		maxKeys = *input.MaxKeys
 	}
+	if input.FetchOwner != nil {
+		fetchOwner = *input.FetchOwner
+	}
+
+	// Retrieve the bucket acl to get the bucket owner, if "fetchOwner" is true
+	// All the objects in the bucket are owner by the bucket owner
+	var acl auth.ACL
+	if fetchOwner {
+		aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
+		if err != nil {
+			return s3response.ListObjectsV2Result{}, azureErrToS3Err(err)
+		}
+
+		if err := json.Unmarshal(aclBytes, &acl); err != nil {
+			return s3response.ListObjectsV2Result{}, fmt.Errorf("unmarshal acl: %w", err)
+		}
+	}
 
 Pager:
 	for pager.More() {
@@ -645,13 +711,20 @@ Pager:
 				isTruncated = true
 				break Pager
 			}
-			objects = append(objects, s3response.Object{
-				ETag:         (*string)(v.Properties.ETag),
+
+			obj := s3response.Object{
+				ETag:         backend.GetPtrFromString(fmt.Sprintf("%q", *v.Properties.ETag)),
 				Key:          v.Name,
 				LastModified: v.Properties.LastModified,
 				Size:         v.Properties.ContentLength,
 				StorageClass: types.ObjectStorageClassStandard,
-			})
+			}
+			if fetchOwner {
+				obj.Owner = &types.Owner{
+					ID: &acl.Owner,
+				}
+			}
+			objects = append(objects, obj)
 		}
 		for _, v := range resp.Segment.BlobPrefixes {
 			if *v.Name <= marker {
@@ -734,41 +807,158 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
 	}, nil
 }
 
-func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	bclient, err := az.getBlobClient(*input.Bucket, *input.Key)
+func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	dstClient, err := az.getBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
 		return nil, err
 	}
-
 	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource {
-		props, err := bclient.GetProperties(ctx, nil)
+		if input.MetadataDirective != types.MetadataDirectiveReplace {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+		}
+
+		// Set object meta http headers
+		res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{
+			BlobCacheControl:       input.CacheControl,
+			BlobContentDisposition: input.ContentDisposition,
+			BlobContentEncoding:    input.ContentEncoding,
+			BlobContentLanguage:    input.ContentLanguage,
+			BlobContentType:        input.ContentType,
+		}, nil)
 		if err != nil {
 			return nil, azureErrToS3Err(err)
 		}
 
-		mdmap := props.Metadata
-		if isMetaSame(mdmap, input.Metadata) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+		meta := input.Metadata
+		if meta == nil {
+			meta = make(map[string]string)
 		}
+
+		// Embed "Expires" in object metadata
+		if getString(input.Expires) != "" {
+			meta[string(keyExpires)] = *input.Expires
+		}
+		// Set object metadata
+		_, err = dstClient.SetMetadata(ctx, parseMetadata(meta), nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+
+		// Set object legal hold
+		if input.ObjectLockLegalHoldStatus != "" {
+			err = az.PutObjectLegalHold(ctx, *input.Bucket, *input.Key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn)
+			if err != nil {
+				return nil, azureErrToS3Err(err)
+			}
+		}
+		// Set object retention
+		if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil {
+			retention := s3response.PutObjectRetentionInput{
+				Mode: types.ObjectLockRetentionMode(input.ObjectLockMode),
+				RetainUntilDate: s3response.AmzDate{
+					Time: *input.ObjectLockRetainUntilDate,
+				},
+			}
+
+			retParsed, err := json.Marshal(retention)
+			if err != nil {
+				return nil, fmt.Errorf("parse object retention: %w", err)
+			}
+			err = az.PutObjectRetention(ctx, *input.Bucket, *input.Key, "", true, retParsed)
+			if err != nil {
+				return nil, azureErrToS3Err(err)
+			}
+		}
+
+		// Set object Tagging, if tagging directive is "REPLACE"
+		if input.TaggingDirective == types.TaggingDirectiveReplace {
+			tags, err := parseTags(input.Tagging)
+			if err != nil {
+				return nil, err
+			}
+			_, err = dstClient.SetTags(ctx, tags, nil)
+			if err != nil {
+				return nil, azureErrToS3Err(err)
+			}
+		}
+
+		return &s3.CopyObjectOutput{
+			CopyObjectResult: &types.CopyObjectResult{
+				LastModified: res.LastModified,
+				ETag:         (*string)(res.ETag),
+			},
+		}, nil
 	}
 
-	tags, err := parseTags(input.Tagging)
+	srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
 	if err != nil {
 		return nil, err
 	}
 
-	resp, err := bclient.CopyFromURL(ctx, az.serviceURL+"/"+*input.CopySource, &blob.CopyFromURLOptions{
-		BlobTags: tags,
-		Metadata: parseMetadata(input.Metadata),
-	})
+	// Get the source object
+	downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
 	if err != nil {
 		return nil, azureErrToS3Err(err)
 	}
 
+	pInput := s3response.PutObjectInput{
+		Body:                      downloadResp.Body,
+		Bucket:                    input.Bucket,
+		Key:                       input.Key,
+		ContentLength:             downloadResp.ContentLength,
+		ContentType:               input.ContentType,
+		ContentEncoding:           input.ContentEncoding,
+		ContentDisposition:        input.ContentDisposition,
+		ContentLanguage:           input.ContentLanguage,
+		CacheControl:              input.CacheControl,
+		Expires:                   input.Expires,
+		Metadata:                  input.Metadata,
+		ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate,
+		ObjectLockMode:            input.ObjectLockMode,
+		ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus,
+	}
+
+	if input.MetadataDirective == types.MetadataDirectiveCopy {
+		// Expires is in downloadResp.Metadata
+		pInput.Expires = nil
+		pInput.CacheControl = downloadResp.CacheControl
+		pInput.ContentDisposition = downloadResp.ContentDisposition
+		pInput.ContentEncoding = downloadResp.ContentEncoding
+		pInput.ContentLanguage = downloadResp.ContentLanguage
+		pInput.ContentType = downloadResp.ContentType
+		pInput.Metadata = parseAzMetadata(downloadResp.Metadata)
+	}
+
+	if input.TaggingDirective == types.TaggingDirectiveReplace {
+		pInput.Tagging = input.Tagging
+	}
+
+	// Create the destination object
+	resp, err := az.PutObject(ctx, pInput)
+	if err != nil {
+		return nil, err
+	}
+
+	// Copy the object tagging, if tagging directive is "COPY"
+	if input.TaggingDirective == types.TaggingDirectiveCopy {
+		srcClient, err := az.getBlobClient(srcBucket, srcObj)
+		if err != nil {
+			return nil, err
+		}
+		res, err := srcClient.GetTags(ctx, nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+
+		_, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+	}
+
 	return &s3.CopyObjectOutput{
 		CopyObjectResult: &types.CopyObjectResult{
-			ETag:         (*string)(resp.ETag),
-			LastModified: resp.LastModified,
+			ETag: &resp.ETag,
 		},
 	}, nil
 }
@@ -815,7 +1005,7 @@ func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object string)
 	return nil
 }
 
-func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	if input.ObjectLockLegalHoldStatus != "" || input.ObjectLockMode != "" {
 		bucketLock, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyBucketLock))
 		if err != nil {
@@ -839,6 +1029,10 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 	meta := parseMetadata(input.Metadata)
 	meta[string(onameAttr)] = input.Key
 
+	if getString(input.Expires) != "" {
+		meta[string(keyExpires)] = input.Expires
+	}
+
 	// parse object tags
 	tagsStr := getString(input.Tagging)
 	tags := map[string]string{}
@@ -847,10 +1041,10 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 		for _, prt := range tagParts {
 			p := strings.Split(prt, "=")
 			if len(p) != 2 {
-				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTag)
+				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTagValue)
 			}
 			if len(p[0]) > 128 || len(p[1]) > 256 {
-				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTag)
+				return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidTagValue)
 			}
 			tags[p[0]] = p[1]
 		}
@@ -881,12 +1075,13 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 	opts := &blockblob.UploadBufferOptions{
 		Metadata: meta,
 		Tags:     tags,
-	}
-	if getString(input.ContentType) != "" {
-		opts.HTTPHeaders = &blob.HTTPHeaders{
-			BlobContentType:     input.ContentType,
-			BlobContentEncoding: input.ContentEncoding,
-		}
+		HTTPHeaders: &blob.HTTPHeaders{
+			BlobContentType:        input.ContentType,
+			BlobContentEncoding:    input.ContentEncoding,
+			BlobCacheControl:       input.CacheControl,
+			BlobContentDisposition: input.ContentDisposition,
+			BlobContentLanguage:    input.ContentLanguage,
+		},
 	}
 
 	// Create and empty blob in .sgwtmp/multipart/<uploadId>/<object hash>
@@ -905,9 +1100,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 }
 
 // Each part is translated into an uncommitted block in a newly created blob in staging area
-func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 	if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
-		return "", err
+		return nil, err
 	}
 
 	// TODO: request streamable version of StageBlock()
@@ -916,32 +1111,34 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta
 	// the body in memory to create an io.ReadSeekCloser
 	rdr, err := getReadSeekCloser(input.Body)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	// block id serves as etag here
-	etag = blockIDInt32ToBase64(*input.PartNumber)
+	etag := blockIDInt32ToBase64(*input.PartNumber)
 	_, err = client.StageBlock(ctx, etag, rdr, nil)
 	if err != nil {
-		return "", parseMpError(err)
+		return nil, parseMpError(err)
 	}
 
-	return etag, nil
+	return &s3.UploadPartOutput{
+		ETag: &etag,
+	}, nil
 }
 
-func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return s3response.CopyObjectResult{}, nil
+		return s3response.CopyPartResult{}, nil
 	}
 
 	if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
-		return s3response.CopyObjectResult{}, err
+		return s3response.CopyPartResult{}, err
 	}
 
 	eTag := blockIDInt32ToBase64(*input.PartNumber)
@@ -949,10 +1146,10 @@ func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInp
 	//TODO: the action returns not implemented on azurite, maybe in production this will work?
 	_, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil)
 	if err != nil {
-		return s3response.CopyObjectResult{}, parseMpError(err)
+		return s3response.CopyPartResult{}, parseMpError(err)
 	}
 
-	return s3response.CopyObjectResult{}, nil
+	return s3response.CopyPartResult{}, nil
 }
 
 // Lists all uncommitted parts from the blob
@@ -1195,34 +1392,63 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 		return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 	}
 
-	slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int {
-		ptNumber, _ := decodeBlockId(*a.Name)
-		nextPtNumber, _ := decodeBlockId(*b.Name)
-		return ptNumber - nextPtNumber
-	})
-
-	for i, block := range blockList.UncommittedBlocks {
-		ptNumber, err := decodeBlockId(*block.Name)
+	uncommittedBlocks := map[int32]*blockblob.Block{}
+	for _, el := range blockList.UncommittedBlocks {
+		ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name))
 		if err != nil {
+			return nil, fmt.Errorf("invalid block name: %w", err)
+		}
+
+		uncommittedBlocks[int32(ptNumber)] = el
+	}
+
+	// The initialie values is the lower limit of partNumber: 0
+	var totalSize int64
+	var partNumber int32
+	last := len(blockList.UncommittedBlocks) - 1
+	for i, part := range input.MultipartUpload.Parts {
+		if part.PartNumber == nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
+		}
+		if *part.PartNumber <= partNumber {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder)
+		}
+		partNumber = *part.PartNumber
+
+		block, ok := uncommittedBlocks[*part.PartNumber]
+		if !ok {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
-		if *input.MultipartUpload.Parts[i].ETag != *block.Name {
+		if *part.ETag != *block.Name {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		// all parts except the last need to be greater, than
+		// the minimum allowed size (5 Mib)
+		if i < last && *block.Size < backend.MinPartSize {
+			return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
 		}
+		totalSize += *block.Size
 		blockIds = append(blockIds, *block.Name)
 	}
 
+	if input.MpuObjectSize != nil && totalSize != *input.MpuObjectSize {
+		return nil, s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize)
+	}
+
 	opts := &blockblob.CommitBlockListOptions{
 		Metadata: props.Metadata,
 		Tags:     parseAzTags(tags.BlobTagSet),
 	}
 	opts.HTTPHeaders = &blob.HTTPHeaders{
-		BlobContentType:     props.ContentType,
-		BlobContentEncoding: props.ContentEncoding,
+		BlobContentType:        props.ContentType,
+		BlobContentEncoding:    props.ContentEncoding,
+		BlobContentDisposition: props.ContentDisposition,
+		BlobContentLanguage:    props.ContentLanguage,
+		BlobCacheControl:       props.CacheControl,
 	}
 
 	resp, err := client.CommitBlockList(ctx, blockIds, opts)
@@ -1560,7 +1786,7 @@ func parseMetadata(m map[string]string) map[string]*string {
 	return meta
 }
 
-func parseAzMetadata(m map[string]*string) map[string]string {
+func parseAndFilterAzMetadata(m map[string]*string) map[string]string {
 	if m == nil {
 		return nil
 	}
@@ -1579,6 +1805,19 @@ func parseAzMetadata(m map[string]*string) map[string]string {
 	return meta
 }
 
+func parseAzMetadata(m map[string]*string) map[string]string {
+	if m == nil {
+		return nil
+	}
+
+	meta := make(map[string]string)
+
+	for k, v := range m {
+		meta[k] = *v
+	}
+	return meta
+}
+
 func parseTags(tagstr *string) (map[string]string, error) {
 	tagsStr := getString(tagstr)
 	tags := make(map[string]string)
@@ -1588,7 +1827,7 @@ func parseTags(tagstr *string) (map[string]string, error) {
 		for _, prt := range tagParts {
 			p := strings.Split(prt, "=")
 			if len(p) != 2 {
-				return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+				return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
 			}
 			tags[p[0]] = p[1]
 		}
@@ -1761,24 +2000,6 @@ func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 	return &acl, nil
 }
 
-func isMetaSame(azMeta map[string]*string, awsMeta map[string]string) bool {
-	if len(azMeta) != len(awsMeta) {
-		return false
-	}
-
-	for key, val := range azMeta {
-		if key == string(keyAclCapital) || key == string(keyAclLower) {
-			continue
-		}
-		awsVal, ok := awsMeta[key]
-		if !ok || awsVal != *val {
-			return false
-		}
-	}
-
-	return true
-}
-
 func createMetaTmpPath(obj, uploadId string) string {
 	objNameSum := sha256.Sum256([]byte(obj))
 	return filepath.Join(string(metaTmpMultipartPrefix), uploadId, fmt.Sprintf("%x", objNameSum))

backend/azure/err.go

@@ -40,7 +40,7 @@ func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
 	case "BlobNotFound":
 		return s3err.GetAPIError(s3err.ErrNoSuchKey)
 	case "TagsTooLarge":
-		return s3err.GetAPIError(s3err.ErrInvalidTag)
+		return s3err.GetAPIError(s3err.ErrInvalidTagValue)
 	case "Requested Range Not Satisfiable":
 		return s3err.GetAPIError(s3err.ErrInvalidRange)
 	}

backend/backend.go

@@ -46,23 +46,26 @@ type Backend interface {
 	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
 	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
 	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
+	PutBucketCors(context.Context, []byte) error
+	GetBucketCors(_ context.Context, bucket string) ([]byte, error)
+	DeleteBucketCors(_ context.Context, bucket string) error
 
 	// multipart operations
-	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
+	CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
-	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
-	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// standard object operations
-	PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
+	PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
-	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
 	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
 	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
@@ -150,8 +153,17 @@ func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket s
 func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketCors(context.Context, []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
@@ -166,14 +178,14 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
 func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
 	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+	return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
@@ -188,7 +200,7 @@ func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (
 func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {

backend/common.go

@@ -17,12 +17,14 @@ package backend
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
-	"net/http"
+	"io/fs"
 	"os"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -34,6 +36,9 @@ const (
 	// this is the media type for directories in AWS and Nextcloud
 	DirContentType     = "application/x-directory"
 	DefaultContentType = "binary/octet-stream"
+
+	// this is the minimum allowed size for mp parts
+	MinPartSize = 5 * 1024 * 1024
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -68,45 +73,118 @@ func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
 
+func TrimEtag(etag *string) *string {
+	if etag == nil {
+		return nil
+	}
+
+	return GetPtrFromString(strings.Trim(*etag, "\""))
+}
+
 var (
-	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidRange           = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
 )
 
-// ParseRange parses input range header and returns startoffset, length, and
-// error. If no endoffset specified, then length is set to -1.
-func ParseRange(size int64, acceptRange string) (int64, int64, error) {
+// ParseGetObjectRange parses input range header and returns startoffset, length, isValid
+// and error. If no endoffset specified, then length is set to the object size
+// for invalid inputs, it returns no error, but isValid=false
+// `InvalidRange` error is returnd, only if startoffset is greater than the object size
+func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
+	if acceptRange == "" {
+		return 0, size, false, nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) != 2 {
+		return 0, size, false, nil
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, size, false, nil
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) != 2 {
+		return 0, size, false, nil
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, size, false, nil
+	}
+
+	if startOffset >= size {
+		return 0, 0, false, errInvalidRange
+	}
+
+	if bRange[1] == "" {
+		return startOffset, size - startOffset, true, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, size, false, nil
+	}
+
+	if endOffset < startOffset {
+		return 0, size, false, nil
+	}
+
+	if endOffset >= size {
+		return startOffset, size - startOffset, true, nil
+	}
+
+	return startOffset, endOffset - startOffset + 1, true, nil
+}
+
+// ParseCopySourceRange parses input range header and returns startoffset, length
+// and error. If no endoffset specified, then length is set to the object size
+func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
 		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
 
-	if len(rangeKv) < 2 {
-		return 0, 0, errInvalidRange
+	if len(rangeKv) != 2 {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 1 || len(bRange) > 2 {
-		return 0, 0, errInvalidRange
+	if len(bRange) != 2 {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
-	endOffset := int64(-1)
-	if len(bRange) == 1 || bRange[1] == "" {
-		return startOffset, endOffset, nil
+	if startOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
-	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
+	if bRange[1] == "" {
+		return startOffset, size - startOffset + 1, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	if endOffset < startOffset {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if endOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
@@ -136,12 +214,28 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
 	return srcBucket, srcObject, versionId, nil
 }
 
-func CreateExceedingRangeErr(objSize int64) s3err.APIError {
-	return s3err.APIError{
-		Code:           "InvalidArgument",
-		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
-		HTTPStatusCode: http.StatusBadRequest,
+// ParseObjectTags parses the url encoded input string into
+// map[string]string key-value tag set
+func ParseObjectTags(t string) (map[string]string, error) {
+	if t == "" {
+		return nil, nil
 	}
+
+	tagging := make(map[string]string)
+
+	tagParts := strings.Split(t, "&")
+	for _, prt := range tagParts {
+		p := strings.Split(prt, "=")
+		if len(p) != 2 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+		if len(p[0]) > 128 || len(p[1]) > 256 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+		tagging[p[0]] = p[1]
+	}
+
+	return tagging, nil
 }
 
 func GetMultipartMD5(parts []types.CompletedPart) string {
@@ -149,8 +243,8 @@ func GetMultipartMD5(parts []types.CompletedPart) string {
 	for _, part := range parts {
 		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
 	}
-	s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
-	return s3MD5
+
+	return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
 }
 
 func getEtagBytes(etag string) []byte {
@@ -178,3 +272,54 @@ func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
 func (f *FileSectionReadCloser) Close() error {
 	return f.F.Close()
 }
+
+// MoveFile moves a file from source to destination.
+func MoveFile(source, destination string, perm os.FileMode) error {
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	err := os.Rename(source, destination)
+	if err == nil || !errors.Is(err, syscall.EXDEV) {
+		return err
+	}
+
+	// Rename can fail if the source and destination are not on the same
+	// filesystem. The fallback is to copy the file and then remove the source.
+	// We need to be careful that the desination does not exist before copying
+	// to prevent any other simultaneous writes to the file.
+	sourceFile, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("open source: %w", err)
+	}
+	defer sourceFile.Close()
+
+	var destFile *os.File
+	for {
+		destFile, err = os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+		if err != nil {
+			if errors.Is(err, fs.ErrExist) {
+				if removeErr := os.Remove(destination); removeErr != nil {
+					return fmt.Errorf("remove existing destination: %w", removeErr)
+				}
+				continue
+			}
+			return fmt.Errorf("create destination: %w", err)
+		}
+		break
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return fmt.Errorf("copy data: %w", err)
+	}
+
+	err = os.Remove(source)
+	if err != nil {
+		return fmt.Errorf("remove source: %w", err)
+	}
+
+	return nil
+}

backend/meta/xattr.go

@@ -23,6 +23,7 @@ import (
 	"syscall"
 
 	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/s3err"
 )
 
 const (
@@ -56,10 +57,18 @@ func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute strin
 // StoreAttribute stores the value of a specific attribute for an object in a bucket.
 func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
 	if f != nil {
-		return xattr.FSet(f, xattrPrefix+attribute, value)
+		err := xattr.FSet(f, xattrPrefix+attribute, value)
+		if errors.Is(err, syscall.EROFS) {
+			return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return err
 	}
 
-	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
+	return err
 }
 
 // DeleteAttribute removes the value of a specific attribute for an object in a bucket.
@@ -68,6 +77,9 @@ func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
 	if errors.Is(err, xattr.ENOATTR) {
 		return ErrNoSuchKey
 	}
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
 	return err
 }
 

backend/posix/with_otmpfile.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 	"golang.org/x/sys/unix"
 )
 
@@ -51,9 +52,13 @@ var (
 	defaultFilePerm uint32 = 0644
 )
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool, forceNoTmpFile bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
+	if forceNoTmpFile {
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
+	}
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
@@ -62,38 +67,12 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	// this is not supported.
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
-		if err != nil {
-			return nil, fmt.Errorf("make temp dir: %w", err)
-		}
-		f, err := os.CreateTemp(dir,
-			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-		if err != nil {
-			return nil, err
-		}
-		tmp := &tmpfile{
-			f:          f,
-			bucket:     bucket,
-			objname:    obj,
-			size:       size,
-			needsChown: doChown,
-			uid:        uid,
-			gid:        gid,
-		}
-		// falloc is best effort, its fine if this fails
-		if size > 0 && dofalloc {
-			tmp.falloc()
-		}
-
-		if doChown {
-			err := f.Chown(uid, gid)
-			if err != nil {
-				return nil, fmt.Errorf("set temp file ownership: %w", err)
-			}
-		}
-
-		return tmp, nil
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
@@ -127,6 +106,46 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	return tmp, nil
 }
 
+func (p *Posix) openMkTemp(dir, bucket, obj string, size int64, dofalloc bool, uid, gid int, doChown bool) (*tmpfile, error) {
+	err := backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, err
+	}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+	// falloc is best effort, its fine if this fails
+	if size > 0 && dofalloc {
+		tmp.falloc()
+	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
+	return tmp, nil
+}
+
 func (tmp *tmpfile) falloc() error {
 	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
 	if err != nil {
@@ -217,7 +236,9 @@ func (tmp *tmpfile) fallbackLink() error {
 	objPath := filepath.Join(tmp.bucket, tmp.objname)
 	err = os.Rename(tempname, objPath)
 	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
+		// rename only works for files within the same filesystem
+		// if this fails fallback to copy
+		return backend.MoveFile(tempname, objPath, fs.FileMode(defaultFilePerm))
 	}
 
 	return nil

backend/posix/without_otmpfile.go

@@ -24,9 +24,11 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"syscall"
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 )
 
 type tmpfile struct {
@@ -36,18 +38,24 @@ type tmpfile struct {
 	size    int64
 }
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool, _ bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
 	// Create a temp file for upload while in progress (see link comments below).
 	var err error
 	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("create temp file: %w", err)
 	}
 
@@ -72,31 +80,17 @@ func (tmp *tmpfile) link() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
-	// We use Rename as the atomic operation for object puts. The upload is
-	// written to a temp file to not conflict with any other simultaneous
-	// uploads. The final operation is to move the temp file into place for
-	// the object. This ensures the object semantics of last upload completed
-	// wins and is not some combination of writes from simultaneous uploads.
 	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err := os.Remove(objPath)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove stale path: %w", err)
-	}
 
 	// reset default file mode because CreateTemp uses 0600
 	tmp.f.Chmod(defaultFilePerm)
 
-	err = tmp.f.Close()
+	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)
 	}
 
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
+	return backend.MoveFile(tempname, objPath, defaultFilePerm)
 }
 
 func (tmp *tmpfile) Write(b []byte) (int, error) {

backend/scoutfs/scoutfs.go

@@ -20,13 +20,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"io/fs"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -35,16 +34,18 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
 
 type ScoutfsOpts struct {
-	ChownUID    bool
-	ChownGID    bool
-	GlacierMode bool
-	BucketLinks bool
-	NewDirPerm  fs.FileMode
+	ChownUID         bool
+	ChownGID         bool
+	GlacierMode      bool
+	BucketLinks      bool
+	NewDirPerm       fs.FileMode
+	DisableNoArchive bool
 }
 
 type ScoutFS struct {
@@ -78,6 +79,11 @@ type ScoutFS struct {
 
 	// newDirPerm is the permissions to use when creating new directories
 	newDirPerm fs.FileMode
+
+	// disableNoArchive is used to disable setting scoutam noarchive flag
+	// on mutlipart parts. This is enabled by default to prevent archive
+	// copies of temporary multipart parts.
+	disableNoArchive bool
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -89,8 +95,13 @@ const (
 	metaHdr             = "X-Amz-Meta"
 	contentTypeHdr      = "content-type"
 	contentEncHdr       = "content-encoding"
+	contentLangHdr      = "content-language"
+	contentDispHdr      = "content-disposition"
+	cacheCtrlHdr        = "cache-control"
+	expiresHdr          = "expires"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
 	etagkey             = "etag"
+	checksumsKey        = "checksums"
 	objectRetentionKey  = "object-retention"
 	objectLegalHoldKey  = "object-legal-hold"
 )
@@ -107,8 +118,6 @@ const (
 	onameAttr    = systemPrefix + "objname"
 	flagskey     = systemPrefix + "sam_flags"
 	stagecopykey = systemPrefix + "sam_stagereq"
-
-	fsBlocksize = 4096
 )
 
 const (
@@ -156,6 +165,31 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
 	return uid, gid, needsChown
 }
 
+func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	out, err := s.Posix.UploadPart(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	if !s.disableNoArchive {
+		sum := sha256.Sum256([]byte(*input.Key))
+		partPath := filepath.Join(
+			*input.Bucket,                        // bucket
+			metaTmpMultipartDir,                  // temp multipart dir
+			fmt.Sprintf("%x", sum),               // hashed objname
+			*input.UploadId,                      // upload id
+			fmt.Sprintf("%v", *input.PartNumber), // part number
+		)
+
+		err = setNoArchive(partPath)
+		if err != nil {
+			return nil, fmt.Errorf("set noarchive: %w", err)
+		}
+	}
+
+	return out, err
+}
+
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
@@ -198,14 +232,39 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 
 	objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
 
+	checksums, err := s.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID))
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get mp checksums: %w", err)
+	}
+
+	// ChecksumType should be the same as specified on CreateMultipartUpload
+	if input.ChecksumType != "" && checksums.Type != input.ChecksumType {
+		checksumType := checksums.Type
+		if checksumType == "" {
+			checksumType = types.ChecksumType("null")
+		}
+
+		return nil, s3err.GetChecksumTypeMismatchOnMpErr(checksumType)
+	}
+
 	// check all parts ok
 	last := len(parts) - 1
-	partsize := int64(0)
 	var totalsize int64
+
+	// The initialie values is the lower limit of partNumber: 0
+	var partNumber int32
 	for i, part := range parts {
-		if part.PartNumber == nil || *part.PartNumber < 1 {
+		if part.PartNumber == nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
+		if *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
+		}
+		if *part.PartNumber <= partNumber {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder)
+		}
+
+		partNumber = *part.PartNumber
 
 		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
 		fullPartPath := filepath.Join(bucket, partObjPath)
@@ -214,20 +273,11 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
-		if i == 0 {
-			partsize = fi.Size()
-		}
-
-		// partsize must be a multiple of the filesystem blocksize
-		// except for last part
-		if i < last && partsize%fsBlocksize != 0 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
-
 		totalsize += fi.Size()
-		// all parts except the last need to be the same size
-		if i < last && partsize != fi.Size() {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		// all parts except the last need to be greater, thena
+		// the minimum allowed size (5 Mib)
+		if i < last && fi.Size() < backend.MinPartSize {
+			return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
 		}
 
 		b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
@@ -238,6 +288,21 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		if parts[i].ETag == nil || etag != *parts[i].ETag {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
+
+		partChecksum, err := s.retrieveChecksums(nil, bucket, partObjPath)
+		if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+			return nil, fmt.Errorf("get part checksum: %w", err)
+		}
+
+		// If checksum has been provided on mp initalization
+		err = validatePartChecksum(partChecksum, part)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if input.MpuObjectSize != nil && totalsize != *input.MpuObjectSize {
+		return nil, s3err.GetIncorrectMpObjectSizeErr(totalsize, *input.MpuObjectSize)
 	}
 
 	// use totalsize=0 because we wont be writing to the file, only moving
@@ -275,7 +340,11 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 
 	userMetaData := make(map[string]string)
 	upiddir := filepath.Join(objdir, uploadID)
-	cType, _ := s.loadUserMetaData(bucket, upiddir, userMetaData)
+	objMeta := s.loadUserMetaData(bucket, upiddir, userMetaData)
+	err = s.storeObjectMetadata(f.File(), bucket, object, objMeta)
+	if err != nil {
+		return nil, err
+	}
 
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
@@ -306,14 +375,6 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		}
 	}
 
-	// set content-type
-	if cType != "" {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
-		if err != nil {
-			return nil, fmt.Errorf("set object content type: %w", err)
-		}
-	}
-
 	// load and set legal hold
 	lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
 	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -352,10 +413,10 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}
 
 	// cleanup tmp dirs
-	os.RemoveAll(upiddir)
+	os.RemoveAll(filepath.Join(bucket, upiddir))
 	// use Remove for objdir in case there are still other uploads
 	// for same object name outstanding
-	os.Remove(objdir)
+	os.Remove(filepath.Join(bucket, objdir))
 
 	return &s3.CompleteMultipartUploadOutput{
 		Bucket: &bucket,
@@ -364,6 +425,125 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}, nil
 }
 
+func (s *ScoutFS) storeObjectMetadata(f *os.File, bucket, object string, m objectMetadata) error {
+	if getString(m.ContentType) != "" {
+		err := s.meta.StoreAttribute(f, bucket, object, contentTypeHdr, []byte(*m.ContentType))
+		if err != nil {
+			return fmt.Errorf("set content-type: %w", err)
+		}
+	}
+	if getString(m.ContentEncoding) != "" {
+		err := s.meta.StoreAttribute(f, bucket, object, contentEncHdr, []byte(*m.ContentEncoding))
+		if err != nil {
+			return fmt.Errorf("set content-encoding: %w", err)
+		}
+	}
+	if getString(m.ContentDisposition) != "" {
+		err := s.meta.StoreAttribute(f, bucket, object, contentDispHdr, []byte(*m.ContentDisposition))
+		if err != nil {
+			return fmt.Errorf("set content-disposition: %w", err)
+		}
+	}
+	if getString(m.ContentLanguage) != "" {
+		err := s.meta.StoreAttribute(f, bucket, object, contentLangHdr, []byte(*m.ContentLanguage))
+		if err != nil {
+			return fmt.Errorf("set content-language: %w", err)
+		}
+	}
+	if getString(m.CacheControl) != "" {
+		err := s.meta.StoreAttribute(f, bucket, object, cacheCtrlHdr, []byte(*m.CacheControl))
+		if err != nil {
+			return fmt.Errorf("set cache-control: %w", err)
+		}
+	}
+	if getString(m.Expires) != "" {
+		err := s.meta.StoreAttribute(f, bucket, object, expiresHdr, []byte(*m.Expires))
+		if err != nil {
+			return fmt.Errorf("set cache-control: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart) error {
+	n := numberOfChecksums(part)
+	if n > 1 {
+		return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
+	}
+	if checksum.Algorithm == "" {
+		if n != 0 {
+			return s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		return nil
+	}
+
+	algo := checksum.Algorithm
+	if n == 0 {
+		return s3err.APIError{
+			Code:           "InvalidRequest",
+			Description:    fmt.Sprintf("The upload was created using a %v checksum. The complete request must include the checksum for each part. It was missing for part %v in the request.", strings.ToLower(string(algo)), *part.PartNumber),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+
+	for _, cs := range []struct {
+		checksum         *string
+		expectedChecksum string
+		algo             types.ChecksumAlgorithm
+	}{
+		{part.ChecksumCRC32, getString(checksum.CRC32), types.ChecksumAlgorithmCrc32},
+		{part.ChecksumCRC32C, getString(checksum.CRC32C), types.ChecksumAlgorithmCrc32c},
+		{part.ChecksumSHA1, getString(checksum.SHA1), types.ChecksumAlgorithmSha1},
+		{part.ChecksumSHA256, getString(checksum.SHA256), types.ChecksumAlgorithmSha256},
+		{part.ChecksumCRC64NVME, getString(checksum.CRC64NVME), types.ChecksumAlgorithmCrc64nvme},
+	} {
+		if cs.checksum == nil {
+			continue
+		}
+
+		if !utils.IsValidChecksum(*cs.checksum, cs.algo, false) {
+			return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
+		}
+
+		if *cs.checksum != cs.expectedChecksum {
+			if algo == cs.algo {
+				return s3err.GetAPIError(s3err.ErrInvalidPart)
+			}
+
+			return s3err.APIError{
+				Code:           "BadDigest",
+				Description:    fmt.Sprintf("The %v you specified for part %v did not match what we received.", strings.ToLower(string(cs.algo)), *part.PartNumber),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		}
+	}
+
+	return nil
+}
+
+func numberOfChecksums(part types.CompletedPart) int {
+	counter := 0
+	if getString(part.ChecksumCRC32) != "" {
+		counter++
+	}
+	if getString(part.ChecksumCRC32C) != "" {
+		counter++
+	}
+	if getString(part.ChecksumSHA1) != "" {
+		counter++
+	}
+	if getString(part.ChecksumSHA256) != "" {
+		counter++
+	}
+	if getString(part.ChecksumCRC64NVME) != "" {
+		counter++
+	}
+
+	return counter
+}
+
 func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) {
 	sum := sha256.Sum256([]byte(object))
 	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
@@ -378,12 +558,21 @@ func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte
 	return sum, nil
 }
 
+type objectMetadata struct {
+	ContentType        *string
+	ContentEncoding    *string
+	ContentDisposition *string
+	ContentLanguage    *string
+	CacheControl       *string
+	Expires            *string
+}
+
 // fll out the user metadata map with the metadata for the object
 // and return the content type and encoding
-func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
+func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) objectMetadata {
 	ents, err := s.meta.ListAttributes(bucket, object)
 	if err != nil || len(ents) == 0 {
-		return "", ""
+		return objectMetadata{}
 	}
 	for _, e := range ents {
 		if !isValidMeta(e) {
@@ -400,20 +589,39 @@ func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (
 		m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
 	}
 
-	var contentType, contentEncoding string
-	b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
-	contentType = string(b)
-	if contentType != "" {
-		m[contentTypeHdr] = contentType
+	var result objectMetadata
+
+	b, err := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
+	if err == nil {
+		result.ContentType = backend.GetPtrFromString(string(b))
 	}
 
-	b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
-	contentEncoding = string(b)
-	if contentEncoding != "" {
-		m[contentEncHdr] = contentEncoding
+	b, err = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
+	if err == nil {
+		result.ContentEncoding = backend.GetPtrFromString(string(b))
 	}
 
-	return contentType, contentEncoding
+	b, err = s.meta.RetrieveAttribute(nil, bucket, object, contentDispHdr)
+	if err == nil {
+		result.ContentDisposition = backend.GetPtrFromString(string(b))
+	}
+
+	b, err = s.meta.RetrieveAttribute(nil, bucket, object, contentLangHdr)
+	if err == nil {
+		result.ContentLanguage = backend.GetPtrFromString(string(b))
+	}
+
+	b, err = s.meta.RetrieveAttribute(nil, bucket, object, cacheCtrlHdr)
+	if err == nil {
+		result.CacheControl = backend.GetPtrFromString(string(b))
+	}
+
+	b, err = s.meta.RetrieveAttribute(nil, bucket, object, expiresHdr)
+	if err == nil {
+		result.Expires = backend.GetPtrFromString(string(b))
+	}
+
+	return result
 }
 
 func isValidMeta(val string) bool {
@@ -427,99 +635,17 @@ func isValidMeta(val string) bool {
 }
 
 func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	bucket := *input.Bucket
-	object := *input.Key
-
-	if input.PartNumber != nil {
-		uploadId, sum, err := s.retrieveUploadId(bucket, object)
-		if err != nil {
-			return nil, err
-		}
-
-		ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("read parts: %w", err)
-		}
-
-		partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
-
-		part, err := os.Stat(filepath.Join(bucket, partPath))
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
-		if errors.Is(err, syscall.ENAMETOOLONG) {
-			return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("stat part: %w", err)
-		}
-
-		b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
-		etag := string(b)
-		if err != nil {
-			etag = ""
-		}
-		partsCount := int32(len(ents))
-		size := part.Size()
-
-		return &s3.HeadObjectOutput{
-			LastModified:  backend.GetTimePtr(part.ModTime()),
-			ETag:          &etag,
-			PartsCount:    &partsCount,
-			ContentLength: &size,
-		}, nil
-	}
-
-	_, err := os.Stat(bucket)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
-	}
+	res, err := s.Posix.HeadObject(ctx, input)
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return nil, err
 	}
 
-	objPath := filepath.Join(bucket, object)
-
-	fi, err := os.Stat(objPath)
-	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if errors.Is(err, syscall.ENAMETOOLONG) {
-		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("stat object: %w", err)
-	}
-	if strings.HasSuffix(object, "/") && !fi.IsDir() {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
-	userMetaData := make(map[string]string)
-	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
-
-	if fi.IsDir() {
-		// this is the media type for directories in AWS and Nextcloud
-		contentType = "application/x-directory"
-	}
-
-	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
-	etag := string(b)
-	if err != nil {
-		etag = ""
-	}
-
-	stclass := types.StorageClassStandard
-	requestOngoing := ""
 	if s.glaciermode {
+		objPath := filepath.Join(*input.Bucket, *input.Key)
+
+		stclass := types.StorageClassStandard
+		requestOngoing := ""
+
 		requestOngoing = stageComplete
 
 		// Check if there are any offline exents associated with this file.
@@ -546,62 +672,17 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 				requestOngoing = stageInProgress
 			}
 		}
+
+		res.Restore = &requestOngoing
+		res.StorageClass = stclass
 	}
 
-	contentLength := fi.Size()
-
-	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
-	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
-	if err == nil {
-		if *status {
-			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
-		} else {
-			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
-		}
-	}
-
-	var objectLockMode types.ObjectLockMode
-	var objectLockRetainUntilDate *time.Time
-	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
-	if err == nil {
-		var config types.ObjectLockRetention
-		if err := json.Unmarshal(retention, &config); err == nil {
-			objectLockMode = types.ObjectLockMode(config.Mode)
-			objectLockRetainUntilDate = config.RetainUntilDate
-		}
-	}
-
-	return &s3.HeadObjectOutput{
-		ContentLength:             &contentLength,
-		ContentType:               &contentType,
-		ContentEncoding:           &contentEncoding,
-		ETag:                      &etag,
-		LastModified:              backend.GetTimePtr(fi.ModTime()),
-		Metadata:                  userMetaData,
-		StorageClass:              stclass,
-		Restore:                   &requestOngoing,
-		ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
-		ObjectLockMode:            objectLockMode,
-		ObjectLockRetainUntilDate: objectLockRetainUntilDate,
-	}, nil
+	return res, nil
 }
 
-func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
-	sum := sha256.Sum256([]byte(object))
-	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
-
-	entries, err := os.ReadDir(objdir)
-	if err != nil || len(entries) == 0 {
-		return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
-	return entries[0].Name(), sum, nil
-}
-
-func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
-	acceptRange := *input.Range
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -628,31 +709,6 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
-	if err != nil {
-		return nil, err
-	}
-
-	objSize := fi.Size()
-	if fi.IsDir() {
-		// directory objects are always 0 len
-		objSize = 0
-		length = 0
-	}
-
-	if length == -1 {
-		length = fi.Size() - startOffset + 1
-	}
-
-	if startOffset+length > fi.Size() {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
-	}
-
-	var contentRange string
-	if acceptRange != "" {
-		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
-	}
-
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
@@ -668,67 +724,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 		}
 	}
 
-	f, err := os.Open(objPath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("open object: %w", err)
-	}
-
-	rdr := io.NewSectionReader(f, startOffset, length)
-
-	userMetaData := make(map[string]string)
-
-	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
-
-	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
-	etag := string(b)
-	if err != nil {
-		etag = ""
-	}
-
-	tags, err := s.getXattrTags(bucket, object)
-	if err != nil {
-		return nil, fmt.Errorf("get object tags: %w", err)
-	}
-
-	tagCount := int32(len(tags))
-
-	return &s3.GetObjectOutput{
-		AcceptRanges:    &acceptRange,
-		ContentLength:   &length,
-		ContentEncoding: &contentEncoding,
-		ContentType:     &contentType,
-		ETag:            &etag,
-		LastModified:    backend.GetTimePtr(fi.ModTime()),
-		Metadata:        userMetaData,
-		TagCount:        &tagCount,
-		StorageClass:    types.StorageClassStandard,
-		ContentRange:    &contentRange,
-		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
-	}, nil
-}
-
-func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) {
-	tags := make(map[string]string)
-	b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if isNoAttr(err) {
-		return tags, nil
-	}
-	if err != nil {
-		return nil, fmt.Errorf("get tags: %w", err)
-	}
-
-	err = json.Unmarshal(b, &tags)
-	if err != nil {
-		return nil, fmt.Errorf("unmarshal tags: %w", err)
-	}
-
-	return tags, nil
+	return s.Posix.GetObject(ctx, input)
 }
 
 func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
@@ -854,17 +850,24 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 				return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 			}
 
-			key := path + "/"
+			size := int64(0)
 			mtime := fi.ModTime()
 
 			return s3response.Object{
 				ETag:         &etag,
-				Key:          &key,
+				Key:          &path,
 				LastModified: &mtime,
+				Size:         &size,
 				StorageClass: types.ObjectStorageClassStandard,
 			}, nil
 		}
 
+		// Retreive the object checksum algorithm
+		checksums, err := s.retrieveChecksums(nil, bucket, path)
+		if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+			return s3response.Object{}, backend.ErrSkipObj
+		}
+
 		// file object, get object info and fill out object data
 		b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
@@ -906,15 +909,27 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		mtime := fi.ModTime()
 
 		return s3response.Object{
-			ETag:         &etag,
-			Key:          &path,
-			LastModified: &mtime,
-			Size:         &size,
-			StorageClass: sc,
+			ETag:              &etag,
+			Key:               &path,
+			LastModified:      &mtime,
+			Size:              &size,
+			StorageClass:      sc,
+			ChecksumAlgorithm: []types.ChecksumAlgorithm{checksums.Algorithm},
+			ChecksumType:      checksums.Type,
 		}, nil
 	}
 }
 
+func (s *ScoutFS) retrieveChecksums(f *os.File, bucket, object string) (checksums s3response.Checksum, err error) {
+	checksumsAtr, err := s.meta.RetrieveAttribute(f, bucket, object, checksumsKey)
+	if err != nil {
+		return checksums, err
+	}
+
+	err = json.Unmarshal(checksumsAtr, &checksums)
+	return checksums, err
+}
+
 // RestoreObject will set stage request on file if offline and do nothing if
 // file is online
 func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
@@ -940,28 +955,11 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput)
 	return nil
 }
 
-func setStaging(objname string) error {
-	b, err := xattr.Get(objname, flagskey)
-	if err != nil && !isNoAttr(err) {
-		return err
+func getString(str *string) string {
+	if str == nil {
+		return ""
 	}
-
-	var oldflags uint64
-	if !isNoAttr(err) {
-		err = json.Unmarshal(b, &oldflags)
-		if err != nil {
-			return err
-		}
-	}
-
-	newflags := oldflags | Staging
-
-	if newflags == oldflags {
-		// no flags change, just return
-		return nil
-	}
-
-	return fSetNewGlobalFlags(objname, newflags)
+	return *str
 }
 
 func isStaging(objname string) (bool, error) {
@@ -981,8 +979,28 @@ func isStaging(objname string) (bool, error) {
 	return flags&Staging == Staging, nil
 }
 
-func fSetNewGlobalFlags(objname string, flags uint64) error {
-	b, err := json.Marshal(&flags)
+func setFlag(objname string, flag uint64) error {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return err
+	}
+
+	var oldflags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &oldflags)
+		if err != nil {
+			return err
+		}
+	}
+
+	newflags := oldflags | flag
+
+	if newflags == oldflags {
+		// no flags change, just return
+		return nil
+	}
+
+	b, err = json.Marshal(&newflags)
 	if err != nil {
 		return err
 	}
@@ -990,6 +1008,14 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 	return xattr.Set(objname, flagskey, b)
 }
 
+func setStaging(objname string) error {
+	return setFlag(objname, Staging)
+}
+
+func setNoArchive(objname string) error {
+	return setFlag(objname, NoArchive)
+}
+
 func isNoAttr(err error) bool {
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {

backend/scoutfs/scoutfs_compat.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 
 	"golang.org/x/sys/unix"
 
@@ -31,6 +32,7 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/s3err"
 )
 
 func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
@@ -52,14 +54,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:       p,
-		rootfd:      f,
-		rootdir:     rootdir,
-		meta:        metastore,
-		chownuid:    opts.ChownUID,
-		chowngid:    opts.ChownGID,
-		glaciermode: opts.GlacierMode,
-		newDirPerm:  opts.NewDirPerm,
+		Posix:            p,
+		rootfd:           f,
+		rootdir:          rootdir,
+		meta:             metastore,
+		chownuid:         opts.ChownUID,
+		chowngid:         opts.ChownGID,
+		glaciermode:      opts.GlacierMode,
+		newDirPerm:       opts.NewDirPerm,
+		disableNoArchive: opts.DisableNoArchive,
 	}, nil
 }
 
@@ -89,6 +92,9 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 	// file descriptor into the namespace.
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, err
 	}
 

backend/walk.go

@@ -44,12 +44,19 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 	cpmap := make(map[string]struct{})
 	var objects []s3response.Object
 
+	// if max is 0, it should return empty non-truncated result
+	if max == 0 {
+		return WalkResults{
+			Truncated: false,
+		}, nil
+	}
+
 	var pastMarker bool
 	if marker == "" {
 		pastMarker = true
 	}
 
-	pastMax := max == 0
+	var pastMax bool
 	var newMarker string
 	var truncated bool
 
@@ -76,14 +83,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipDir
 		}
 
-		if pastMax {
-			if len(objects) != 0 {
-				newMarker = *objects[len(objects)-1].Key
-				truncated = true
-			}
-			return fs.SkipAll
-		}
-
 		// After this point, return skipflag instead of nil
 		// so we can skip a directory without an early return
 		var skipflag error
@@ -108,13 +107,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
 				skipflag = fs.SkipDir
 			} else {
-				// TODO: can we do better here rather than a second readdir
-				// per directory?
-				ents, err := fs.ReadDir(fileSystem, path)
-				if err != nil {
-					return fmt.Errorf("readdir %q: %w", path, err)
-				}
-				if len(ents) == 0 && delimiter == "" {
+				if delimiter == "" {
 					dirobj, err := getObj(path+"/", d)
 					if err == ErrSkipObj {
 						return skipflag
@@ -122,11 +115,26 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 					if err != nil {
 						return fmt.Errorf("directory to object %q: %w", path, err)
 					}
+					if pastMax {
+						truncated = true
+						return fs.SkipAll
+					}
 					objects = append(objects, dirobj)
+					if (len(objects) + len(cpmap)) == int(max) {
+						newMarker = path
+						pastMax = true
+					}
 
 					return skipflag
 				}
 
+				// TODO: can we do better here rather than a second readdir
+				// per directory?
+				ents, err := fs.ReadDir(fileSystem, path)
+				if err != nil {
+					return fmt.Errorf("readdir %q: %w", path, err)
+				}
+
 				if len(ents) != 0 {
 					return skipflag
 				}
@@ -159,9 +167,15 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
 			}
+			if pastMax {
+				truncated = true
+				return fs.SkipAll
+			}
+
 			objects = append(objects, obj)
 
-			if max > 0 && (len(objects)+len(cpmap)) == int(max) {
+			if (len(objects) + len(cpmap)) == int(max) {
+				newMarker = path
 				pastMax = true
 			}
 
@@ -199,8 +213,13 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
 			}
+			if pastMax {
+				truncated = true
+				return fs.SkipAll
+			}
 			objects = append(objects, obj)
 			if (len(objects) + len(cpmap)) == int(max) {
+				newMarker = path
 				pastMax = true
 			}
 			return skipflag
@@ -221,11 +240,14 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return skipflag
 		}
 
+		if pastMax {
+			truncated = true
+			return fs.SkipAll
+		}
 		cpmap[cpref] = struct{}{}
 		if (len(objects) + len(cpmap)) == int(max) {
 			newMarker = cpref
-			truncated = true
-			return fs.SkipAll
+			pastMax = true
 		}
 
 		return skipflag
@@ -251,6 +273,10 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		})
 	}
 
+	if !truncated {
+		newMarker = ""
+	}
+
 	return WalkResults{
 		CommonPrefixes: commonPrefixes,
 		Objects:        objects,

cmd/versitygw/main.go

@@ -524,19 +524,19 @@ func initFlags() []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:        "ipa-user",
-			Usage:       "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
+			Usage:       "Username used to connect to FreeIPA (requires permissions to read user vault contents)",
 			EnvVars:     []string{"VGW_IPA_USER"},
 			Destination: &ipaUser,
 		},
 		&cli.StringFlag{
 			Name:        "ipa-password",
-			Usage:       "Password of the user used to connect to FreeIPA.",
+			Usage:       "Password of the user used to connect to FreeIPA",
 			EnvVars:     []string{"VGW_IPA_PASSWORD"},
 			Destination: &ipaPassword,
 		},
 		&cli.BoolFlag{
 			Name:        "ipa-insecure",
-			Usage:       "Verify TLS certificate of FreeIPA server. Default is 'true'.",
+			Usage:       "Disable verify TLS certificate of FreeIPA server",
 			EnvVars:     []string{"VGW_IPA_INSECURE"},
 			Destination: &ipaInsecure,
 		},

cmd/versitygw/posix.go

@@ -31,6 +31,7 @@ var (
 	dirPerms           uint
 	sidecar            string
 	nometa             bool
+	forceNoTmpFile     bool
 )
 
 func posixCommand() *cli.Command {
@@ -93,6 +94,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_META_NONE"},
 				Destination: &nometa,
 			},
+			&cli.BoolFlag{
+				Name:        "disableotmp",
+				Usage:       "disable O_TMPFILE support for new objects",
+				EnvVars:     []string{"VGW_DISABLE_OTMP"},
+				Destination: &forceNoTmpFile,
+			},
 		},
 	}
 }
@@ -113,11 +120,12 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	opts := posix.PosixOpts{
-		ChownUID:      chownuid,
-		ChownGID:      chowngid,
-		BucketLinks:   bucketlinks,
-		VersioningDir: versioningDir,
-		NewDirPerm:    fs.FileMode(dirPerms),
+		ChownUID:       chownuid,
+		ChownGID:       chowngid,
+		BucketLinks:    bucketlinks,
+		VersioningDir:  versioningDir,
+		NewDirPerm:     fs.FileMode(dirPerms),
+		ForceNoTmpFile: forceNoTmpFile,
 	}
 
 	var ms meta.MetadataStorer

cmd/versitygw/s3.go

@@ -26,6 +26,7 @@ var (
 	s3proxySecret          string
 	s3proxyEndpoint        string
 	s3proxyRegion          string
+	s3proxyMetaBucket      string
 	s3proxyDisableChecksum bool
 	s3proxySslSkipVerify   bool
 	s3proxyDebug           bool
@@ -71,6 +72,12 @@ to an s3 storage backend service.`,
 				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
+			&cli.StringFlag{
+				Name:        "meta-bucket",
+				Usage:       "s3 service meta bucket to store buckets acl/policy",
+				EnvVars:     []string{"VGW_S3_META_BUCKET"},
+				Destination: &s3proxyMetaBucket,
+			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
@@ -97,8 +104,8 @@ to an s3 storage backend service.`,
 }
 
 func runS3(ctx *cli.Context) error {
-	be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
-		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
 	if err != nil {
 		return fmt.Errorf("init s3 backend: %w", err)
 	}

cmd/versitygw/scoutfs.go

@@ -24,7 +24,8 @@ import (
 )
 
 var (
-	glacier bool
+	glacier          bool
+	disableNoArchive bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -79,6 +80,12 @@ move interfaces as well as support for tiered filesystems.`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.BoolFlag{
+				Name:        "disable-noarchive",
+				Usage:       "disable setting noarchive for multipart part uploads",
+				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
+				Destination: &disableNoArchive,
+			},
 		},
 	}
 }
@@ -98,6 +105,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
 	opts.NewDirPerm = fs.FileMode(dirPerms)
+	opts.DisableNoArchive = disableNoArchive
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

extra/example.conf

@@ -99,6 +99,10 @@ ROOT_SECRET_ACCESS_KEY=
 # endpoint is unauthenticated, and returns a 200 status for GET.
 #VGW_HEALTH=
 
+# Enable VGW_READ_ONLY to only allow read operations to the S3 server. No write
+# operations will be allowed.
+#VGW_READ_ONLY=false
+
 ###############
 # Access Logs #
 ###############
@@ -240,6 +244,24 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_LDAP_USER_ID_ATR=
 #VGW_IAM_LDAP_GROUP_ID_ATR=
 
+# The FreeIPA options will enable the FreeIPA IAM service with accounts stored
+# in an external FreeIPA service. Currently the FreeIPA IAM service only
+# supports account retrieval. Creating and modifying accounts must be done
+# outside of the versitygw service.
+# FreeIPA server url e.g. https://ipa.example.test
+#VGW_IPA_HOST=
+# A name of the user vault containing their secret
+#VGW_IPA_VAULT_NAME=
+# Username used to connect to FreeIPA (requires permissions to read user vault
+# contents)
+#VGW_IPA_USER=
+# Password of the user used to connect to FreeIPA
+#VGW_IPA_PASSWORD=
+# Disable verify TLS certificate of FreeIPA server
+#VGW_IPA_INSECURE=false
+# FreeIPA IAM debug output
+#VGW_IPA_DEBUG=false
+
 ###############
 # IAM caching #
 ###############
@@ -317,6 +339,40 @@ ROOT_SECRET_ACCESS_KEY=
 # as any parent directories automatically created with object uploads.
 #VGW_DIR_PERMS=0755
 
+# To enable object versions, the VGW_VERSIONING_DIR option must be set to the
+# directory that will be used to store the object versions. The version
+# directory must NOT be a subdirectory of the VGW_BACKEND_ARG directory.
+#VGW_VERSIONING_DIR=
+
+# The gateway uses xattrs to store metadata for objects by default. For systems
+# that do not support xattrs, the VGW_META_SIDECAR option can be set to a
+# directory that will be used to store the metadata for objects. This is
+# currently experimental, and may have issues for some edge cases.
+#VGW_META_SIDECAR=
+
+# The VGW_META_NONE option will disable the metadata functionality for the
+# gateway. This will cause the gateway to not store any metadata for objects
+# or buckets. This include bucket ACLs and Policy. This may be useful for
+# read only access to pre-existing data where the gateway should not modify
+# the data. It is recommened to enable VGW_READ_ONLY (Global Options) along
+# with this.
+#VGW_META_NONE=false
+
+# The gateway will use O_TMPFILE for writing objects while uploading and
+# link the file to the final object name when the upload is complete if the
+# filesystem supports O_TMPFILE. This creates an atomic object creation
+# that is not visible to other clients or racing uploads until the upload
+# is complete. This will not work if there is a different filesystem mounted
+# below the bucket level than where the bucket resides. The VGW_DISABLE_OTMP
+# option can be set to true to disable this functionality and force the fallback
+# mode when O_TMPFILE is not available. This fallback will create a temporary
+# file in the bucket directory and rename it to the final object name when
+# the upload is complete if the final location is in the same filesystem, or
+# copy the file to the final location if the final location is in a different
+# filesystem. This fallback mode is still atomic, but may be less efficient
+# than O_TMPFILE when the data needs to be copied into the final location.
+#VGW_DISABLE_OTMP=false
+
 ###########
 # scoutfs #
 ###########
@@ -358,6 +414,13 @@ ROOT_SECRET_ACCESS_KEY=
 # as any parent directories automatically created with object uploads.
 #VGW_DIR_PERMS=0755
 
+# The default behavior of the gateway is to automatically set the noarchive
+# flag on the multipart upload parts while the multipart upload is in progress.
+# This is to prevent the parts from being archived since they are temporary
+# and will be deleted after the multipart upload is completed or aborted. The
+# VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
+#VGW_DISABLE_NOARCHIVE=false
+
 ######
 # s3 #
 ######

go.mod

@@ -1,82 +1,83 @@
 module github.com/versity/versitygw
 
-go 1.21.0
+go 1.23.0
+
+toolchain go1.24.1
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
 	github.com/DataDog/datadog-go/v5 v5.6.0
-	github.com/aws/aws-sdk-go-v2 v1.34.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1
-	github.com/aws/smithy-go v1.22.2
-	github.com/go-ldap/ldap/v3 v3.4.10
+	github.com/aws/aws-sdk-go-v2 v1.36.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2
+	github.com/aws/smithy-go v1.22.3
+	github.com/go-ldap/ldap/v3 v3.4.11
 	github.com/gofiber/fiber/v2 v2.52.6
-	github.com/google/go-cmp v0.6.0
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.38.0
+	github.com/nats-io/nats.go v1.41.2
 	github.com/oklog/ulid/v2 v2.1.0
 	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.4
-	github.com/urfave/cli/v2 v2.27.5
-	github.com/valyala/fasthttp v1.58.0
+	github.com/urfave/cli/v2 v2.27.6
+	github.com/valyala/fasthttp v1.60.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sync v0.10.0
-	golang.org/x/sys v0.29.0
+	golang.org/x/sync v0.13.0
+	golang.org/x/sys v0.32.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.9 // indirect
+	github.com/nats-io/nkeys v0.4.11 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/net v0.34.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
-	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/crypto v0.37.0 // indirect
+	golang.org/x/net v0.39.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.29.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.55
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.14
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/tcplisten v1.0.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 )

go.sum

@@ -1,11 +1,11 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 h1:1mvYtZfWQAnwNah/C+Z+Jb9rQH95LPE2vlmMuWAHJk8=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1/go.mod h1:75I/mXtme1JyWFtz8GocPHVFyH421IBoZErnO16dd0k=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1 h1:Bk5uOhSAenHyR5P61D/NzeQCv+4fEVV8mOkJ82NqpWw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1/go.mod h1:QZ4pw3or1WPmRBxf0cHd1tknzrT54WPBOQoGutCPvSU=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
@@ -14,8 +14,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
 github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -25,44 +25,44 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU=
-github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg=
-github.com/aws/aws-sdk-go-v2/config v1.29.2 h1:JuIxOEPcSKpMB0J+khMjznG9LIhIBdmqNiEcPclnwqc=
-github.com/aws/aws-sdk-go-v2/config v1.29.2/go.mod h1:HktTHregOZwNSM/e7WTfVSu9RCX+3eOv+6ij27PtaYs=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.55 h1:CDhKnDEaGkLA5ZszV/qw5uwN5M8rbv9Cl0JRN+PRsaM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.55/go.mod h1:kPD/vj+RB5MREDUky376+zdnjZpR+WgdBBvwrmnlmKE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 h1:kU7tmXNaJ07LsyN3BUgGqAmVmQtq0w6duVIHAKfp0/w=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25/go.mod h1:OiC8+OiqrURb1wrwmr/UbOVLFSWEGxjinj5C299VQdo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54 h1:6BWOAho3Cgdy4cmNJ4HWY8VZgqODEU7Gw78XXireNZI=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54/go.mod h1:n+t/oyYErOV3jf/GxNTVlizSM9RMV1yH7jvcIvld3Do=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 h1:kznaW4f81mNMlREkU9w3jUuJvU5g/KsqDV43ab7Rp6s=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.12/go.mod h1:bZy9r8e0/s0P7BSDHgMLXK2KvdyRRBIQ2blKlvLt0IU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 h1:mUwIpAvILeKFnRx4h1dEgGEFGuV8KJ3pEScZWVFYuZA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11/go.mod h1:JDJtD+b8HNVv71axz8+S5492KM8wTzHRFpMKQbPlYxw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 h1:g9d+TOsu3ac7SgmY2dUf1qMgu/uJVTlQ4VCbH6hRxSw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.10/go.mod h1:WZfNmntu92HO44MVZAubQaz3qCuIdeOdog2sADfU6hU=
-github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
-github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
+github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
+github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
+github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 h1:PcKMOZfp+kNtJTw2HF2op6SjDvwPBYRvz0Y24PQLUR4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72/go.mod h1:vq7/m7dahFXcdzWVOvvjasDI9RcsD3RsTfHmDundJYg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 h1:tWUG+4wZqdMl/znThEk9tcCy8tTMxq8dW0JTgamohrY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
+github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
@@ -74,21 +74,19 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
-github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
-github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU=
+github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM=
 github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
 github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
+github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
@@ -99,7 +97,6 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
@@ -116,11 +113,11 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
+github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
+github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
@@ -131,10 +128,10 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.38.0 h1:A7P+g7Wjp4/NWqDOOP/K6hfhr54DvdDQUznt5JFg9XA=
-github.com/nats-io/nats.go v1.38.0/go.mod h1:IGUM++TwokGnXPs82/wCuiHS02/aKrdYUQkU8If6yjw=
-github.com/nats-io/nkeys v0.4.9 h1:qe9Faq2Gxwi6RZnZMXfmGMZkg3afLLOtrU+gDZJ35b0=
-github.com/nats-io/nkeys v0.4.9/go.mod h1:jcMqs+FLG+W5YO36OX6wFIFcmpdAns+w1Wm6D3I/evE=
+github.com/nats-io/nats.go v1.41.2 h1:5UkfLAtu/036s99AhFRlyNDI1Ieylb36qbGjJzHixos=
+github.com/nats-io/nats.go v1.41.2/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
+github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
+github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
@@ -150,8 +147,8 @@ github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
 github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
-github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
+github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
+github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -169,20 +166,17 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
-github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
+github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.58.0 h1:GGB2dWxSbEprU9j0iMJHgdKYJVDyjrOwF9RE59PbRuE=
-github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw=
-github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
-github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/valyala/fasthttp v1.60.0 h1:kBRYS0lOhVJ6V+bYN8PqAHELKHtXqwq9zNMLKx1MBsw=
+github.com/valyala/fasthttp v1.60.0/go.mod h1:iY4kDgV3Gc6EqhRZ8icqcmlG6bqhcDXfuHgTO4FXCvc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -200,45 +194,28 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
-golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
-golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -254,23 +231,14 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -278,24 +246,19 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
-golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

metrics/actions.go

@@ -72,6 +72,9 @@ var (
 	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
 	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
 	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
+	ActionPutBucketCors                 = "s3_PutBucketCors"
+	ActionGetBucketCors                 = "s3_GetBucketCors"
+	ActionDeleteBucketCors              = "s3_DeleteBucketCors"
 
 	// Admin actions
 	ActionAdminCreateUser        = "admin_CreateUser"
@@ -266,4 +269,16 @@ func init() {
 		Name:    "UploadPartCopy",
 		Service: "s3",
 	}
+	ActionMap[ActionPutBucketCors] = Action{
+		Name:    "PutBucketCors",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketCors] = Action{
+		Name:    "GetBucketCors",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucketCors] = Action{
+		Name:    "DeleteBucketCors",
+		Service: "s3",
+	}
 }

s3api/controllers/backend_moq_test.go

@@ -32,18 +32,21 @@ var _ backend.Backend = &BackendMock{}
 //			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 //				panic("mock out the CompleteMultipartUpload method")
 //			},
-//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 //				panic("mock out the CopyObject method")
 //			},
 //			CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
 //				panic("mock out the CreateBucket method")
 //			},
-//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 //				panic("mock out the CreateMultipartUpload method")
 //			},
 //			DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketCorsFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketCors method")
+//			},
 //			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketOwnershipControls method")
 //			},
@@ -65,6 +68,9 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketCorsFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+//				panic("mock out the GetBucketCors method")
+//			},
 //			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
 //				panic("mock out the GetBucketOwnershipControls method")
 //			},
@@ -128,6 +134,9 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketCorsFunc: func(contextMoqParam context.Context, bytes []byte) error {
+//				panic("mock out the PutBucketCors method")
+//			},
 //			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
 //				panic("mock out the PutBucketOwnershipControls method")
 //			},
@@ -140,7 +149,7 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 //				panic("mock out the PutBucketVersioning method")
 //			},
-//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 //				panic("mock out the PutObject method")
 //			},
 //			PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error {
@@ -170,10 +179,10 @@ var _ backend.Backend = &BackendMock{}
 //			StringFunc: func() string {
 //				panic("mock out the String method")
 //			},
-//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 //				panic("mock out the UploadPart method")
 //			},
-//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 //				panic("mock out the UploadPartCopy method")
 //			},
 //		}
@@ -193,17 +202,20 @@ type BackendMock struct {
 	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 
 	// CopyObjectFunc mocks the CopyObject method.
-	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
 
 	// CreateBucketFunc mocks the CreateBucket method.
 	CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error
 
 	// CreateMultipartUploadFunc mocks the CreateMultipartUpload method.
-	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
+	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error
 
+	// DeleteBucketCorsFunc mocks the DeleteBucketCors method.
+	DeleteBucketCorsFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
 	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -225,6 +237,9 @@ type BackendMock struct {
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketCorsFunc mocks the GetBucketCors method.
+	GetBucketCorsFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
+
 	// GetBucketOwnershipControlsFunc mocks the GetBucketOwnershipControls method.
 	GetBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error)
 
@@ -288,6 +303,9 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketCorsFunc mocks the PutBucketCors method.
+	PutBucketCorsFunc func(contextMoqParam context.Context, bytes []byte) error
+
 	// PutBucketOwnershipControlsFunc mocks the PutBucketOwnershipControls method.
 	PutBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error
 
@@ -301,7 +319,7 @@ type BackendMock struct {
 	PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error
 
 	// PutObjectFunc mocks the PutObject method.
-	PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error)
+	PutObjectFunc func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error)
 
 	// PutObjectAclFunc mocks the PutObjectAcl method.
 	PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error
@@ -331,10 +349,10 @@ type BackendMock struct {
 	StringFunc func() string
 
 	// UploadPartFunc mocks the UploadPart method.
-	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error)
+	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error)
 
 	// UploadPartCopyFunc mocks the UploadPartCopy method.
-	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// calls tracks calls to the methods.
 	calls struct {
@@ -366,7 +384,7 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// CopyObjectInput is the copyObjectInput argument value.
-			CopyObjectInput *s3.CopyObjectInput
+			CopyObjectInput s3response.CopyObjectInput
 		}
 		// CreateBucket holds details about calls to the CreateBucket method.
 		CreateBucket []struct {
@@ -382,7 +400,7 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// CreateMultipartUploadInput is the createMultipartUploadInput argument value.
-			CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+			CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 		}
 		// DeleteBucket holds details about calls to the DeleteBucket method.
 		DeleteBucket []struct {
@@ -391,6 +409,13 @@ type BackendMock struct {
 			// Bucket is the bucket argument value.
 			Bucket string
 		}
+		// DeleteBucketCors holds details about calls to the DeleteBucketCors method.
+		DeleteBucketCors []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
 		DeleteBucketOwnershipControls []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -442,6 +467,13 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketCors holds details about calls to the GetBucketCors method.
+		GetBucketCors []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetBucketOwnershipControls holds details about calls to the GetBucketOwnershipControls method.
 		GetBucketOwnershipControls []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -599,6 +631,13 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketCors holds details about calls to the PutBucketCors method.
+		PutBucketCors []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bytes is the bytes argument value.
+			Bytes []byte
+		}
 		// PutBucketOwnershipControls holds details about calls to the PutBucketOwnershipControls method.
 		PutBucketOwnershipControls []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -640,7 +679,7 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// PutObjectInput is the putObjectInput argument value.
-			PutObjectInput *s3.PutObjectInput
+			PutObjectInput s3response.PutObjectInput
 		}
 		// PutObjectAcl holds details about calls to the PutObjectAcl method.
 		PutObjectAcl []struct {
@@ -739,6 +778,7 @@ type BackendMock struct {
 	lockCreateBucket                  sync.RWMutex
 	lockCreateMultipartUpload         sync.RWMutex
 	lockDeleteBucket                  sync.RWMutex
+	lockDeleteBucketCors              sync.RWMutex
 	lockDeleteBucketOwnershipControls sync.RWMutex
 	lockDeleteBucketPolicy            sync.RWMutex
 	lockDeleteBucketTagging           sync.RWMutex
@@ -746,6 +786,7 @@ type BackendMock struct {
 	lockDeleteObjectTagging           sync.RWMutex
 	lockDeleteObjects                 sync.RWMutex
 	lockGetBucketAcl                  sync.RWMutex
+	lockGetBucketCors                 sync.RWMutex
 	lockGetBucketOwnershipControls    sync.RWMutex
 	lockGetBucketPolicy               sync.RWMutex
 	lockGetBucketTagging              sync.RWMutex
@@ -767,6 +808,7 @@ type BackendMock struct {
 	lockListObjectsV2                 sync.RWMutex
 	lockListParts                     sync.RWMutex
 	lockPutBucketAcl                  sync.RWMutex
+	lockPutBucketCors                 sync.RWMutex
 	lockPutBucketOwnershipControls    sync.RWMutex
 	lockPutBucketPolicy               sync.RWMutex
 	lockPutBucketTagging              sync.RWMutex
@@ -898,13 +940,13 @@ func (mock *BackendMock) CompleteMultipartUploadCalls() []struct {
 }
 
 // CopyObject calls CopyObjectFunc.
-func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	if mock.CopyObjectFunc == nil {
 		panic("BackendMock.CopyObjectFunc: method is nil but Backend.CopyObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
-		CopyObjectInput *s3.CopyObjectInput
+		CopyObjectInput s3response.CopyObjectInput
 	}{
 		ContextMoqParam: contextMoqParam,
 		CopyObjectInput: copyObjectInput,
@@ -921,11 +963,11 @@ func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectI
 //	len(mockedBackend.CopyObjectCalls())
 func (mock *BackendMock) CopyObjectCalls() []struct {
 	ContextMoqParam context.Context
-	CopyObjectInput *s3.CopyObjectInput
+	CopyObjectInput s3response.CopyObjectInput
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
-		CopyObjectInput *s3.CopyObjectInput
+		CopyObjectInput s3response.CopyObjectInput
 	}
 	mock.lockCopyObject.RLock()
 	calls = mock.calls.CopyObject
@@ -974,13 +1016,13 @@ func (mock *BackendMock) CreateBucketCalls() []struct {
 }
 
 // CreateMultipartUpload calls CreateMultipartUploadFunc.
-func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	if mock.CreateMultipartUploadFunc == nil {
 		panic("BackendMock.CreateMultipartUploadFunc: method is nil but Backend.CreateMultipartUpload was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam            context.Context
-		CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+		CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 	}{
 		ContextMoqParam:            contextMoqParam,
 		CreateMultipartUploadInput: createMultipartUploadInput,
@@ -997,11 +1039,11 @@ func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context,
 //	len(mockedBackend.CreateMultipartUploadCalls())
 func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
 	ContextMoqParam            context.Context
-	CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+	CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 } {
 	var calls []struct {
 		ContextMoqParam            context.Context
-		CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+		CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 	}
 	mock.lockCreateMultipartUpload.RLock()
 	calls = mock.calls.CreateMultipartUpload
@@ -1045,6 +1087,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketCors calls DeleteBucketCorsFunc.
+func (mock *BackendMock) DeleteBucketCors(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketCorsFunc == nil {
+		panic("BackendMock.DeleteBucketCorsFunc: method is nil but Backend.DeleteBucketCors was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketCors.Lock()
+	mock.calls.DeleteBucketCors = append(mock.calls.DeleteBucketCors, callInfo)
+	mock.lockDeleteBucketCors.Unlock()
+	return mock.DeleteBucketCorsFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketCorsCalls gets all the calls that were made to DeleteBucketCors.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketCorsCalls())
+func (mock *BackendMock) DeleteBucketCorsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketCors.RLock()
+	calls = mock.calls.DeleteBucketCors
+	mock.lockDeleteBucketCors.RUnlock()
+	return calls
+}
+
 // DeleteBucketOwnershipControls calls DeleteBucketOwnershipControlsFunc.
 func (mock *BackendMock) DeleteBucketOwnershipControls(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketOwnershipControlsFunc == nil {
@@ -1301,6 +1379,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketCors calls GetBucketCorsFunc.
+func (mock *BackendMock) GetBucketCors(contextMoqParam context.Context, bucket string) ([]byte, error) {
+	if mock.GetBucketCorsFunc == nil {
+		panic("BackendMock.GetBucketCorsFunc: method is nil but Backend.GetBucketCors was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketCors.Lock()
+	mock.calls.GetBucketCors = append(mock.calls.GetBucketCors, callInfo)
+	mock.lockGetBucketCors.Unlock()
+	return mock.GetBucketCorsFunc(contextMoqParam, bucket)
+}
+
+// GetBucketCorsCalls gets all the calls that were made to GetBucketCors.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketCorsCalls())
+func (mock *BackendMock) GetBucketCorsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketCors.RLock()
+	calls = mock.calls.GetBucketCors
+	mock.lockGetBucketCors.RUnlock()
+	return calls
+}
+
 // GetBucketOwnershipControls calls GetBucketOwnershipControlsFunc.
 func (mock *BackendMock) GetBucketOwnershipControls(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
 	if mock.GetBucketOwnershipControlsFunc == nil {
@@ -2077,6 +2191,42 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketCors calls PutBucketCorsFunc.
+func (mock *BackendMock) PutBucketCors(contextMoqParam context.Context, bytes []byte) error {
+	if mock.PutBucketCorsFunc == nil {
+		panic("BackendMock.PutBucketCorsFunc: method is nil but Backend.PutBucketCors was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bytes           []byte
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bytes:           bytes,
+	}
+	mock.lockPutBucketCors.Lock()
+	mock.calls.PutBucketCors = append(mock.calls.PutBucketCors, callInfo)
+	mock.lockPutBucketCors.Unlock()
+	return mock.PutBucketCorsFunc(contextMoqParam, bytes)
+}
+
+// PutBucketCorsCalls gets all the calls that were made to PutBucketCors.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketCorsCalls())
+func (mock *BackendMock) PutBucketCorsCalls() []struct {
+	ContextMoqParam context.Context
+	Bytes           []byte
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bytes           []byte
+	}
+	mock.lockPutBucketCors.RLock()
+	calls = mock.calls.PutBucketCors
+	mock.lockPutBucketCors.RUnlock()
+	return calls
+}
+
 // PutBucketOwnershipControls calls PutBucketOwnershipControlsFunc.
 func (mock *BackendMock) PutBucketOwnershipControls(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
 	if mock.PutBucketOwnershipControlsFunc == nil {
@@ -2238,13 +2388,13 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct {
 }
 
 // PutObject calls PutObjectFunc.
-func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 	if mock.PutObjectFunc == nil {
 		panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
-		PutObjectInput  *s3.PutObjectInput
+		PutObjectInput  s3response.PutObjectInput
 	}{
 		ContextMoqParam: contextMoqParam,
 		PutObjectInput:  putObjectInput,
@@ -2261,11 +2411,11 @@ func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInp
 //	len(mockedBackend.PutObjectCalls())
 func (mock *BackendMock) PutObjectCalls() []struct {
 	ContextMoqParam context.Context
-	PutObjectInput  *s3.PutObjectInput
+	PutObjectInput  s3response.PutObjectInput
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
-		PutObjectInput  *s3.PutObjectInput
+		PutObjectInput  s3response.PutObjectInput
 	}
 	mock.lockPutObject.RLock()
 	calls = mock.calls.PutObject
@@ -2620,7 +2770,7 @@ func (mock *BackendMock) StringCalls() []struct {
 }
 
 // UploadPart calls UploadPartFunc.
-func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 	if mock.UploadPartFunc == nil {
 		panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called")
 	}
@@ -2656,7 +2806,7 @@ func (mock *BackendMock) UploadPartCalls() []struct {
 }
 
 // UploadPartCopy calls UploadPartCopyFunc.
-func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 	if mock.UploadPartCopyFunc == nil {
 		panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called")
 	}

s3api/controllers/base_test.go

@@ -232,6 +232,9 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
+	invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
+	invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -329,6 +332,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Get-actions-get-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invalidChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Get-actions-get-object-success",
 			app:  app,
@@ -622,8 +634,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</VersioningConfiguration>
 	`
 
-	policyBody := `
-	{
+	policyBody := `{
 		"Statement": [
 			{
 				"Effect": "Allow",
@@ -963,22 +974,22 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
 				return nil
 			},
-			CopyObjectFunc: func(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+			CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 				return &s3.CopyObjectOutput{
 					CopyObjectResult: &types.CopyObjectResult{},
 				}, nil
 			},
-			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+			PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 				return s3response.PutObjectOutput{}, nil
 			},
-			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
-				return "hello", nil
+			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+				return &s3.UploadPartOutput{}, nil
 			},
 			PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
 				return nil
 			},
-			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-				return s3response.CopyObjectResult{}, nil
+			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+				return s3response.CopyPartResult{}, nil
 			},
 			PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
 				return nil
@@ -1012,6 +1023,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
 
+	// CopyObject invalid checksum algorithm
+	cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	// PutObjectAcl success
 	aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	aclReq.Header.Set("X-Amz-Acl", "private")
@@ -1033,6 +1049,40 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	// PutObject invalid checksum algorithm
+	invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
+	// PutObject invalid base64 checksum
+	invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
+
+	// PutObject invalid crc32
+	invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
+
+	// PutObject invalid crc32c
+	invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
+
+	// PutObject invalid sha1
+	invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
+
+	// PutObject invalid sha256
+	invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
+
+	// PutObject multiple checksum headers
+	mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
+	// PutObject checksum algorithm and header mismatch
+	checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -1175,6 +1225,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Copy-object-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: cpyInvChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Copy-object-success",
 			app:  app,
@@ -1642,6 +1701,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	})
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
+	invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
+	invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -1658,6 +1720,15 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Head-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Head-object-error",
 			app:  appErr,
@@ -1697,7 +1768,7 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 				return &s3.CompleteMultipartUploadOutput{}, nil
 			},
-			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+			CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 				return s3response.InitiateMultipartUploadResult{}, nil
 			},
 			SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
@@ -1735,6 +1806,9 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	})
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
+	invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -1796,6 +1870,15 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Create-multipart-upload-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: invChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Create-multipart-upload-success",
 			app:  app,

s3api/debuglogger/logger.go

@@ -0,0 +1,251 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package debuglogger
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+const (
+	reset      = "\033[0m"
+	green      = "\033[32m"
+	yellow     = "\033[33m"
+	blue       = "\033[34m"
+	borderChar = "─"
+	boxWidth   = 120
+)
+
+// Logs http request details: headers, body, params, query args
+func LogFiberRequestDetails(ctx *fiber.Ctx) {
+	// Log the full request url
+	fullURL := ctx.Protocol() + "://" + ctx.Hostname() + ctx.OriginalURL()
+	fmt.Printf("%s[URL]: %s%s\n", green, fullURL, reset)
+
+	// log request headers
+	wrapInBox(green, "REQUEST HEADERS", boxWidth, func() {
+		ctx.Request().Header.VisitAll(func(key, value []byte) {
+			printWrappedLine(yellow, string(key), string(value))
+		})
+	})
+	// skip request body log for PutObject and UploadPart
+	skipBodyLog := isLargeDataAction(ctx)
+	if !skipBodyLog {
+		body := ctx.Request().Body()
+		if len(body) != 0 {
+			printBoxTitleLine(blue, "REQUEST BODY", boxWidth, false)
+			prettyBody := prettyPrintXML(body)
+			fmt.Printf("%s%s%s\n", blue, prettyBody, reset)
+			printHorizontalBorder(blue, boxWidth, false)
+		}
+	}
+
+	if ctx.Request().URI().QueryArgs().Len() != 0 {
+		ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
+			log.Printf("%s: %s", key, val)
+		})
+	}
+}
+
+// Logs http response details: body, headers
+func LogFiberResponseDetails(ctx *fiber.Ctx) {
+	wrapInBox(green, "RESPONSE HEADERS", boxWidth, func() {
+		ctx.Response().Header.VisitAll(func(key, value []byte) {
+			printWrappedLine(yellow, string(key), string(value))
+		})
+	})
+
+	_, ok := ctx.Locals("skip-res-body-log").(bool)
+	if !ok {
+		body := ctx.Response().Body()
+		if len(body) != 0 {
+			printBoxTitleLine(blue, "RESPONSE BODY", boxWidth, false)
+			prettyBody := prettyPrintXML(body)
+			fmt.Printf("%s%s%s\n", blue, prettyBody, reset)
+			printHorizontalBorder(blue, boxWidth, false)
+		}
+	}
+}
+
+// Logf is the same as 'fmt.Printf' with debug prefix,
+// a color added and '\n' at the end
+func Logf(format string, v ...any) {
+	debugPrefix := "[DEBUG]: "
+	fmt.Printf(yellow+debugPrefix+format+reset+"\n", v...)
+}
+
+// prettyPrintXML takes raw XML input and returns a formatted (pretty-printed) version.
+func prettyPrintXML(input []byte) string {
+	b := &bytes.Buffer{}
+	decoder := xml.NewDecoder(bytes.NewReader(input))
+	encoder := xml.NewEncoder(b)
+	encoder.Indent("", "  ")
+
+	var depth int
+	for {
+		token, err := decoder.Token()
+		if err == io.EOF {
+			encoder.Flush()
+			return b.String()
+		}
+		if err != nil {
+			// Return the raw input if decoding fails
+			return string(input)
+		}
+
+		switch t := token.(type) {
+		case xml.StartElement:
+			if depth > 0 {
+				// Strip namespace from tag name
+				t.Name.Space = ""
+
+				// Filter out xmlns attributes to make it more readable
+				newAttrs := make([]xml.Attr, 0, len(t.Attr))
+				for _, attr := range t.Attr {
+					if !(attr.Name.Space == "" && attr.Name.Local == "xmlns") {
+						newAttrs = append(newAttrs, attr)
+					}
+				}
+				t.Attr = newAttrs
+			}
+			depth++
+			err = encoder.EncodeToken(t)
+		case xml.EndElement:
+			if depth > 1 {
+				t.Name.Space = ""
+			}
+			depth--
+			err = encoder.EncodeToken(t)
+		default:
+			err = encoder.EncodeToken(t)
+		}
+
+		if err != nil {
+			// Return the raw input if decoding fails
+			return string(input)
+		}
+	}
+}
+
+// Prints out box title either with closing characters or not:  "┌", "┐"
+// e.g ┌────────────────[ RESPONSE HEADERS ]────────────────┐
+func printBoxTitleLine(color, title string, length int, closing bool) {
+	leftCorner, rightCorner := "┌", "┐"
+
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	// Calculate how many border characters are needed
+	titleFormatted := fmt.Sprintf("[ %s ]", title)
+	borderSpace := length - len(titleFormatted) - 2 // 2 for corners
+	leftLen := borderSpace / 2
+	rightLen := borderSpace - leftLen
+
+	// Build the line
+	line := leftCorner +
+		strings.Repeat(borderChar, leftLen) +
+		titleFormatted +
+		strings.Repeat(borderChar, rightLen) +
+		rightCorner
+
+	fmt.Println(color + line + reset)
+}
+
+// Prints out a horizontal line either with closing characters or not: "└", "┘"
+func printHorizontalBorder(color string, length int, closing bool) {
+	leftCorner, rightCorner := "└", "┘"
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	line := leftCorner + strings.Repeat(borderChar, length-2) + rightCorner + reset
+	fmt.Println(color + line)
+}
+
+// wrapInBox wraps the output of a function call (fn) inside a styled box with a title.
+func wrapInBox(color, title string, length int, fn func()) {
+	printBoxTitleLine(color, title, length, true)
+	fn()
+	printHorizontalBorder(color, length, true)
+}
+
+// returns the provided string length
+// defaulting to 13 for exceeding lengths
+func getLen(str string) int {
+	if len(str) < 13 {
+		return 13
+	}
+
+	return len(str)
+}
+
+// prints a formatted key-value pair within a box layout,
+// wrapping the value text if it exceeds the allowed width.
+func printWrappedLine(keyColor, key, value string) {
+	prefix := fmt.Sprintf("%s│%s %s%-13s%s : ", green, reset, keyColor, key, reset)
+	prefixLen := len(prefix) - len(green) - len(reset) - len(keyColor) - len(reset)
+	// the actual prefix size without colors
+	actualPrefixLen := getLen(key) + 5
+
+	lineWidth := boxWidth - prefixLen
+	valueLines := wrapText(value, lineWidth)
+
+	for i, line := range valueLines {
+		if i == 0 {
+			if len(line) < lineWidth {
+				line += strings.Repeat(" ", lineWidth-len(line))
+			}
+			fmt.Printf("%s%s%s %s│%s\n", prefix, reset, line, green, reset)
+		} else {
+			line = strings.Repeat(" ", actualPrefixLen-2) + line
+			if len(line) < boxWidth-4 {
+				line += strings.Repeat(" ", boxWidth-len(line)-4)
+			}
+			fmt.Printf("%s│ %s%s %s│%s\n", green, reset, line, green, reset)
+		}
+	}
+}
+
+// wrapText splits the input text into lines of at most `width` characters each.
+func wrapText(text string, width int) []string {
+	var lines []string
+	for len(text) > width {
+		lines = append(lines, text[:width])
+		text = text[width:]
+	}
+	if text != "" {
+		lines = append(lines, text)
+	}
+	return lines
+}
+
+// TODO: remove this and use utils.IsBidDataAction after refactoring
+// and creating 'internal' package
+func isLargeDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}

s3api/middlewares/acl-parser.go

@@ -51,7 +51,8 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			!ctx.Request().URI().QueryArgs().Has("versioning") &&
 			!ctx.Request().URI().QueryArgs().Has("policy") &&
 			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
-			!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+			!ctx.Request().URI().QueryArgs().Has("ownershipControls") &&
+			!ctx.Request().URI().QueryArgs().Has("cors") {
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
@@ -74,6 +75,11 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 		}
 
+		// if owner is not set, set default owner to root account
+		if parsedAcl.Owner == "" {
+			parsedAcl.Owner = ctx.Locals("rootAccess").(string)
+		}
+
 		ctx.Locals("parsedAcl", parsedAcl)
 		return ctx.Next()
 	}

s3api/middlewares/authentication.go

@@ -63,10 +63,6 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return sendResponse(ctx, err, logger, mm)
 		}
 
-		if authData.Algorithm != "AWS4-HMAC-SHA256" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger, mm)
-		}
-
 		if authData.Region != region {
 			return sendResponse(ctx, s3err.APIError{
 				Code:           "SignatureDoesNotMatch",
@@ -76,6 +72,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
+		ctx.Locals("rootAccess", root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
@@ -108,6 +105,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return sendResponse(ctx, err, logger, mm)
 		}
 
+		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if utils.IsBigDataAction(ctx) {
 			// for streaming PUT actions, authorization is deferred
 			// until end of stream due to need to get length and
@@ -115,10 +113,24 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
 				return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
 			})
+
+			// wrap the io.Reader with ChunkReader if x-amz-content-sha256
+			// provide chunk encoding value
+			if utils.IsStreamingPayload(hashPayload) {
+				var err error
+				wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+					var cr io.Reader
+					cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate, debug)
+					return cr
+				})
+				if err != nil {
+					return sendResponse(ctx, err, logger, mm)
+				}
+			}
+
 			return ctx.Next()
 		}
 
-		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if !utils.IsSpecialPayload(hashPayload) {
 			// Calculate the hash of the request payload
 			hashedPayload := sha256.Sum256(ctx.Body())

s3api/middlewares/chunk.go

@@ -1,62 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package middlewares
-
-import (
-	"io"
-	"time"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/metrics"
-	"github.com/versity/versitygw/s3api/utils"
-	"github.com/versity/versitygw/s3log"
-)
-
-// ProcessChunkedBody initializes the chunked upload stream if the
-// request appears to be a chunked upload
-func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
-	return func(ctx *fiber.Ctx) error {
-		decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
-		if decodedLength == "" {
-			return ctx.Next()
-		}
-		// TODO: validate content length
-
-		authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
-		if err != nil {
-			return sendResponse(ctx, err, logger, mm)
-		}
-
-		acct := ctx.Locals("account").(auth.Account)
-		amzdate := ctx.Get("X-Amz-Date")
-		date, _ := time.Parse(iso8601Format, amzdate)
-
-		if utils.IsBigDataAction(ctx) {
-			var err error
-			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
-				var cr io.Reader
-				cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
-				return cr
-			})
-			if err != nil {
-				return sendResponse(ctx, err, logger, mm)
-			}
-			return ctx.Next()
-		}
-
-		return ctx.Next()
-	}
-}

s3api/middlewares/logger.go

@@ -15,30 +15,15 @@
 package middlewares
 
 import (
-	"fmt"
-	"log"
-
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/debuglogger"
 )
 
-func RequestLogger(isDebug bool) fiber.Handler {
+func DebugLogger() fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		ctx.Locals("isDebug", isDebug)
-		if isDebug {
-			log.Println("Request headers: ")
-			ctx.Request().Header.VisitAll(func(key, val []byte) {
-				log.Printf("%s: %s", key, val)
-			})
-
-			if ctx.Request().URI().QueryArgs().Len() != 0 {
-				fmt.Println()
-				log.Println("Request query arguments: ")
-				ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
-					log.Printf("%s: %s", key, val)
-				})
-			}
-		}
-
-		return ctx.Next()
+		debuglogger.LogFiberRequestDetails(ctx)
+		err := ctx.Next()
+		debuglogger.LogFiberResponseDetails(ctx)
+		return err
 	}
 }

s3api/middlewares/md5.go

@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
 		}
 
 		sum := md5.Sum(ctx.Body())
-		calculatedSum := utils.Md5SumString(sum[:])
+		calculatedSum := utils.Base64SumString(sum[:])
 
 		if incomingSum != calculatedSum {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

s3api/middlewares/presign-auth.go

@@ -43,6 +43,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
+		ctx.Locals("rootAccess", root.Access)
+
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
 			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)

s3api/server.go

@@ -76,12 +76,13 @@ func New(
 		})
 	}
 	app.Use(middlewares.DecodeURL(l, mm))
-	app.Use(middlewares.RequestLogger(server.debug))
+	if server.debug {
+		app.Use(middlewares.DebugLogger())
+	}
 
 	// Authentication middlewares
 	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
-	app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l, server.readonly))
 

s3api/utils/auth-reader.go

@@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de
 	var hr *HashReader
 	hashPayload := ctx.Get("X-Amz-Content-Sha256")
 	if !IsSpecialPayload(hashPayload) {
-		hr, _ = NewHashReader(r, "", HashTypeSha256)
+		hr, _ = NewHashReader(r, "", HashTypeSha256Hex)
 	} else {
 		hr, _ = NewHashReader(r, "", HashTypeNone)
 	}
@@ -190,6 +190,10 @@ func ParseAuthorization(authorization string) (AuthData, error) {
 
 	algo := authParts[0]
 
+	if algo != "AWS4-HMAC-SHA256" {
+		return a, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported)
+	}
+
 	kvData := authParts[1]
 	kvPairs := strings.Split(kvData, ",")
 	// we are expecting at least Credential, SignedHeaders, and Signature

s3api/utils/chunk-reader.go

@@ -17,9 +17,12 @@ package utils
 import (
 	"fmt"
 	"io"
+	"net/http"
+	"strings"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
 )
 
 type payloadType string
@@ -33,6 +36,14 @@ const (
 	payloadTypeStreamingEcdsaTrailer    payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
 )
 
+func getPayloadTypeNotSupportedErr(p payloadType) error {
+	return s3err.APIError{
+		HTTPStatusCode: http.StatusNotImplemented,
+		Code:           "NotImplemented",
+		Description:    fmt.Sprintf("The chunk encoding algorithm %v is not supported.", p),
+	}
+}
+
 var (
 	specialValues = map[payloadType]bool{
 		payloadTypeUnsigned:                 true,
@@ -71,29 +82,46 @@ func (c checksumType) isValid() bool {
 		c == checksumTypeCrc64nvme
 }
 
-// IsSpecialPayload checks for streaming/unsigned authorization types
+// IsSpecialPayload checks for special authorization types
 func IsSpecialPayload(str string) bool {
 	return specialValues[payloadType(str)]
 }
 
-func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
+// IsChunkEncoding checks for streaming/unsigned authorization types
+func IsStreamingPayload(str string) bool {
+	pt := payloadType(str)
+	return pt == payloadTypeStreamingUnsignedTrailer ||
+		pt == payloadTypeStreamingSigned ||
+		pt == payloadTypeStreamingSignedTrailer
+}
+
+func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time, debug bool) (io.Reader, error) {
+	decContLength := ctx.Get("X-Amz-Decoded-Content-Length")
+	if decContLength == "" {
+		return nil, s3err.GetAPIError(s3err.ErrMissingDecodedContentLength)
+	}
 	contentSha256 := payloadType(ctx.Get("X-Amz-Content-Sha256"))
 	if !contentSha256.isValid() {
 		//TODO: Add proper APIError
 		return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
 	}
 
-	checksumType := checksumType(ctx.Get("X-Amz-Trailer"))
-	if checksumType != "" && !checksumType.isValid() {
-		//TODO: Add proper APIError
-		return nil, fmt.Errorf("invalid X-Amz-Trailer: %v", checksumType)
+	checksumType := checksumType(strings.ToLower(ctx.Get("X-Amz-Trailer")))
+	if contentSha256 != payloadTypeStreamingSigned && !checksumType.isValid() {
+		return nil, s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
 	}
 
 	switch contentSha256 {
 	case payloadTypeStreamingUnsignedTrailer:
-		return NewUnsignedChunkReader(r, checksumType)
-		//TODO: Add other chunk readers
+		return NewUnsignedChunkReader(r, checksumType, debug)
+	case payloadTypeStreamingSignedTrailer:
+		return NewSignedChunkReader(r, authdata, region, secret, date, checksumType, debug)
+	case payloadTypeStreamingSigned:
+		return NewSignedChunkReader(r, authdata, region, secret, date, "", debug)
+	// return not supported for:
+	// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
+	// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
+	default:
+		return nil, getPayloadTypeNotSupportedErr(contentSha256)
 	}
-
-	return NewSignedChunkReader(r, authdata, region, secret, date)
 }

s3api/utils/csum-reader.go

@@ -16,13 +16,19 @@ package utils
 
 import (
 	"crypto/md5"
+	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"hash"
+	"hash/crc32"
+	"hash/crc64"
 	"io"
+	"math/bits"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -31,11 +37,21 @@ type HashType string
 
 const (
 	// HashTypeMd5 generates MD5 checksum for the data stream
-	HashTypeMd5 = "md5"
-	// HashTypeSha256 generates SHA256 checksum for the data stream
-	HashTypeSha256 = "sha256"
+	HashTypeMd5 HashType = "md5"
+	// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
+	HashTypeSha256 HashType = "sha256"
+	// HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream
+	HashTypeSha256Hex HashType = "sha256-hex"
+	// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
+	HashTypeSha1 HashType = "sha1"
+	// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
+	HashTypeCRC32 HashType = "crc32"
+	// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
+	HashTypeCRC32C HashType = "crc32c"
+	// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
+	HashTypeCRC64NVME HashType = "crc64nvme"
 	// HashTypeNone is a no-op checksum for the data stream
-	HashTypeNone = "none"
+	HashTypeNone HashType = "none"
 )
 
 // HashReader is an io.Reader that calculates the checksum
@@ -62,8 +78,18 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
 	switch ht {
 	case HashTypeMd5:
 		hash = md5.New()
+	case HashTypeSha256Hex:
+		hash = sha256.New()
 	case HashTypeSha256:
 		hash = sha256.New()
+	case HashTypeSha1:
+		hash = sha1.New()
+	case HashTypeCRC32:
+		hash = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeCRC64NVME:
+		hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
 	case HashTypeNone:
 		hash = noop{}
 	default:
@@ -88,15 +114,40 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	if errors.Is(readerr, io.EOF) && hr.sum != "" {
 		switch hr.hashType {
 		case HashTypeMd5:
-			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
 			}
-		case HashTypeSha256:
-			sum := hex.EncodeToString(hr.hash.Sum(nil))
+		case HashTypeSha256Hex:
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
 			}
+		case HashTypeCRC32:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32)
+			}
+		case HashTypeCRC32C:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c)
+			}
+		case HashTypeSha1:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1)
+			}
+		case HashTypeSha256:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
+			}
+		case HashTypeCRC64NVME:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
+			}
 		default:
 			return n, errInvalidHashType
 		}
@@ -104,20 +155,38 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	return n, readerr
 }
 
+func (hr *HashReader) SetReader(r io.Reader) {
+	hr.r = r
+}
+
 // Sum returns the checksum hash of the data read so far
 func (hr *HashReader) Sum() string {
 	switch hr.hashType {
 	case HashTypeMd5:
-		return Md5SumString(hr.hash.Sum(nil))
-	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256Hex:
 		return hex.EncodeToString(hr.hash.Sum(nil))
+	case HashTypeCRC32:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC32C:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha1:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC64NVME:
+		return Base64SumString(hr.hash.Sum(nil))
 	default:
 		return ""
 	}
 }
 
+func (hr *HashReader) Type() HashType {
+	return hr.hashType
+}
+
 // Md5SumString converts the hash bytes to the string checksum value
-func Md5SumString(b []byte) string {
+func Base64SumString(b []byte) string {
 	return base64.StdEncoding.EncodeToString(b)
 }
 
@@ -128,3 +197,59 @@ func (n noop) Sum(b []byte) []byte         { return []byte{} }
 func (n noop) Reset()                      {}
 func (n noop) Size() int                   { return 0 }
 func (n noop) BlockSize() int              { return 1 }
+
+// NewCompositeChecksumReader initializes a composite checksum
+// processor, which decodes and validates the provided
+// checksums and returns the final checksum based on
+// the previous processings.
+//
+// The supported checksum types are:
+// - CRC32
+// - CRC32C
+// - SHA1
+// - SHA256
+func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
+	var hasher hash.Hash
+	switch ht {
+	case HashTypeSha256:
+		hasher = sha256.New()
+	case HashTypeSha1:
+		hasher = sha1.New()
+	case HashTypeCRC32:
+		hasher = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeNone:
+		hasher = noop{}
+	default:
+		return nil, errInvalidHashType
+	}
+
+	return &CompositeChecksumReader{
+		hasher: hasher,
+	}, nil
+}
+
+type CompositeChecksumReader struct {
+	hasher hash.Hash
+}
+
+// Decodes and writes the checksum in the hasher
+func (ccr *CompositeChecksumReader) Process(checksum string) error {
+	data, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		return fmt.Errorf("base64 decode: %w", err)
+	}
+
+	_, err = ccr.hasher.Write(data)
+	if err != nil {
+		return fmt.Errorf("hash write: %w", err)
+	}
+
+	return nil
+}
+
+// Returns the base64 encoded composite checksum
+func (ccr *CompositeChecksumReader) Sum() string {
+	return Base64SumString(ccr.hasher.Sum(nil))
+}

s3api/utils/logger.go

@@ -1,55 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package utils
-
-import (
-	"fmt"
-	"log"
-
-	"github.com/gofiber/fiber/v2"
-)
-
-func LogCtxDetails(ctx *fiber.Ctx, respBody []byte) {
-	isDebug, ok := ctx.Locals("isDebug").(bool)
-	_, notLogReqBody := ctx.Locals("logReqBody").(bool)
-	_, notLogResBody := ctx.Locals("logResBody").(bool)
-	if isDebug && ok {
-		// Log request body
-		if !notLogReqBody {
-			fmt.Println()
-			log.Printf("Request Body: %s", ctx.Request().Body())
-		}
-
-		// Log path parameters
-		fmt.Println()
-		log.Println("Path parameters: ")
-		for key, val := range ctx.AllParams() {
-			log.Printf("%s: %s", key, val)
-		}
-
-		// Log response headers
-		fmt.Println()
-		log.Println("Response Headers: ")
-		ctx.Response().Header.VisitAll(func(key, val []byte) {
-			log.Printf("%s: %s", key, val)
-		})
-
-		// Log response body
-		if !notLogResBody && len(respBody) > 0 {
-			fmt.Println()
-			log.Printf("Response body %s", ctx.Response().Body())
-		}
-	}
-}

s3api/utils/signed-chunk-reader.go

@@ -15,9 +15,11 @@
 package utils
 
 import (
+	"bufio"
 	"bytes"
 	"crypto/hmac"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -25,8 +27,10 @@ import (
 	"io"
 	"math"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -34,58 +38,84 @@ import (
 // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
 
 const (
-	chunkHdrStr       = ";chunk-signature="
-	chunkHdrDelim     = "\r\n"
-	zeroLenSig        = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-	awsV4             = "AWS4"
-	awsS3Service      = "s3"
-	awsV4Request      = "aws4_request"
-	streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
+	chunkHdrDelim            = "\r\n"
+	zeroLenSig               = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	awsV4                    = "AWS4"
+	awsS3Service             = "s3"
+	awsV4Request             = "aws4_request"
+	trailerSignatureHeader   = "x-amz-trailer-signature"
+	streamPayloadAlgo        = "AWS4-HMAC-SHA256-PAYLOAD"
+	streamPayloadTrailerAlgo = "AWS4-HMAC-SHA256-TRAILER"
 )
 
 // ChunkReader reads from chunked upload request body, and returns
 // object data stream
 type ChunkReader struct {
-	r                io.Reader
-	signingKey       []byte
-	prevSig          string
-	parsedSig        string
-	currentChunkSize int64
-	chunkDataLeft    int64
-	trailerExpected  int
-	stash            []byte
-	chunkHash        hash.Hash
-	strToSignPrefix  string
-	skipcheck        bool
+	r              io.Reader
+	signingKey     []byte
+	prevSig        string
+	parsedSig      string
+	chunkDataLeft  int64
+	trailer        checksumType
+	trailerSig     string
+	parsedChecksum string
+	stash          []byte
+	chunkHash      hash.Hash
+	checksumHash   hash.Hash
+	isEOF          bool
+	isFirstHeader  bool
+	//TODO: Add debug logging for the reader
+	debug  bool
+	region string
+	date   time.Time
 }
 
 // NewChunkReader reads from request body io.Reader and parses out the
 // chunk metadata in stream. The headers are validated for proper signatures.
 // Reading from the chunk reader will read only the object data stream
 // without the chunk headers/trailers.
-func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
-	return &ChunkReader{
+func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time, chType checksumType, debug bool) (io.Reader, error) {
+	chRdr := &ChunkReader{
 		r:          r,
 		signingKey: getSigningKey(secret, region, date),
 		// the authdata.Signature is validated in the auth-reader,
 		// so we can use that here without any other checks
-		prevSig:         authdata.Signature,
-		chunkHash:       sha256.New(),
-		strToSignPrefix: getStringToSignPrefix(date, region),
-	}, nil
+		prevSig:       authdata.Signature,
+		chunkHash:     sha256.New(),
+		isFirstHeader: true,
+		date:          date,
+		region:        region,
+		trailer:       chType,
+		debug:         debug,
+	}
+
+	if chType != "" {
+		checksumHasher, err := getHasher(chType)
+		if err != nil {
+			return nil, err
+		}
+
+		chRdr.checksumHash = checksumHasher
+	}
+	return chRdr, nil
 }
 
 // Read satisfies the io.Reader for this type
 func (cr *ChunkReader) Read(p []byte) (int, error) {
 	n, err := cr.r.Read(p)
 	if err != nil && err != io.EOF {
-		return n, err
+		return 0, err
 	}
 
+	cr.isEOF = err == io.EOF
+
 	if cr.chunkDataLeft < int64(n) {
 		chunkSize := cr.chunkDataLeft
 		if chunkSize > 0 {
 			cr.chunkHash.Write(p[:chunkSize])
+			if cr.checksumHash != nil {
+				cr.checksumHash.Write(p[:chunkSize])
+			}
 		}
 		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
 		n += int(chunkSize)
@@ -94,22 +124,25 @@ func (cr *ChunkReader) Read(p []byte) (int, error) {
 
 	cr.chunkDataLeft -= int64(n)
 	cr.chunkHash.Write(p[:n])
+	if cr.checksumHash != nil {
+		cr.checksumHash.Write(p[:n])
+	}
 	return n, err
 }
 
 // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
 // This part is the same for all chunks,
 // only the previous signature and hash of current chunk changes
-func getStringToSignPrefix(date time.Time, region string) string {
+func (cr *ChunkReader) getStringToSignPrefix(algo string) string {
 	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
-		date.Format("20060102"),
-		region,
+		cr.date.Format("20060102"),
+		cr.region,
 		awsS3Service,
 		awsV4Request)
 
 	return fmt.Sprintf("%s\n%s\n%s",
-		streamPayloadAlgo,
-		date.Format("20060102T150405Z"),
+		algo,
+		cr.date.Format("20060102T150405Z"),
 		credentialScope)
 }
 
@@ -117,14 +150,70 @@ func getStringToSignPrefix(date time.Time, region string) string {
 // signature For each chunk, you calculate the signature using the following
 // string to sign. For the first chunk, you use the seed-signature as the
 // previous signature.
-func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
+func (cr *ChunkReader) getChunkStringToSign() string {
+	prefix := cr.getStringToSignPrefix(streamPayloadAlgo)
+	chunkHash := cr.chunkHash.Sum(nil)
 	return fmt.Sprintf("%s\n%s\n%s\n%s",
 		prefix,
-		prevSig,
+		cr.prevSig,
 		zeroLenSig,
 		hex.EncodeToString(chunkHash))
 }
 
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
+// Builds the final chunk trailing signature string to sign
+func (cr *ChunkReader) getTrailerChunkStringToSign() string {
+	trailer := fmt.Sprintf("%v:%v\n", cr.trailer, cr.parsedChecksum)
+	hsh := sha256.Sum256([]byte(trailer))
+	sig := hex.EncodeToString(hsh[:])
+
+	prefix := cr.getStringToSignPrefix(streamPayloadTrailerAlgo)
+
+	return fmt.Sprintf("%s\n%s\n%s",
+		prefix,
+		cr.prevSig,
+		sig,
+	)
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
+// Calculates and validates the final chunk trailer signature
+func (cr *ChunkReader) verifyTrailerSignature() error {
+	strToSign := cr.getTrailerChunkStringToSign()
+	sig := hex.EncodeToString(hmac256(cr.signingKey, []byte(strToSign)))
+
+	if sig != cr.trailerSig {
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+
+	return nil
+}
+
+// Verifies the object checksum
+func (cr *ChunkReader) verifyChecksum() error {
+	checksumHash := cr.checksumHash.Sum(nil)
+	checksum := base64.StdEncoding.EncodeToString(checksumHash)
+	if checksum != cr.parsedChecksum {
+		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(cr.trailer), "x-amz-checksum-")))
+		return s3err.GetChecksumBadDigestErr(algo)
+	}
+
+	return nil
+}
+
+// Calculates and verifies the chunk signature
+func (cr *ChunkReader) checkSignature() error {
+	sigstr := cr.getChunkStringToSign()
+	cr.chunkHash.Reset()
+	cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
+
+	if cr.prevSig != cr.parsedSig {
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+	cr.parsedSig = ""
+	return nil
+}
+
 // The provided p should have all of the previous chunk data and trailer
 // consumed already. The positioning here is expected that p[0] starts the
 // new chunk size with the ";chunk-signature=" following. The only exception
@@ -137,38 +226,14 @@ func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
 func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 	n := len(p)
 
-	if !cr.skipcheck && cr.parsedSig != "" {
-		chunkhash := cr.chunkHash.Sum(nil)
-		cr.chunkHash.Reset()
-
-		sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
-		cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
-
-		if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	if cr.parsedSig != "" {
+		err := cr.checkSignature()
+		if err != nil {
+			return 0, err
 		}
 	}
 
-	if cr.trailerExpected != 0 {
-		if len(p) < len(chunkHdrDelim) {
-			// This is the special case where we need to consume the
-			// trailer, but instead hit the end of the buffer. The
-			// subsequent call will finish consuming the trailer.
-			cr.chunkDataLeft = 0
-			cr.trailerExpected -= len(p)
-			cr.skipcheck = true
-			return 0, nil
-		}
-		// move data up to remove trailer
-		copy(p, p[cr.trailerExpected:])
-		n -= cr.trailerExpected
-	}
-
-	cr.skipcheck = false
-
 	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
-	cr.currentChunkSize = chunkSize
-	cr.parsedSig = sig
 	if err == errskipHeader {
 		cr.chunkDataLeft = 0
 		return 0, nil
@@ -176,12 +241,30 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 	if err != nil {
 		return 0, err
 	}
+	cr.parsedSig = sig
+	// If we hit the final chunk, calculate and validate the final
+	// chunk signature and finish reading
 	if chunkSize == 0 {
+		cr.chunkHash.Reset()
+		err := cr.checkSignature()
+		if err != nil {
+			return 0, err
+		}
+
+		if cr.trailer != "" {
+			err := cr.verifyChecksum()
+			if err != nil {
+				return 0, err
+			}
+			err = cr.verifyTrailerSignature()
+			if err != nil {
+				return 0, err
+			}
+		}
+
 		return 0, io.EOF
 	}
 
-	cr.trailerExpected = len(chunkHdrDelim)
-
 	// move data up to remove chunk header
 	copy(p, p[bufOffset:n])
 	n -= bufOffset
@@ -191,6 +274,9 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 	if int64(n) > chunkSize {
 		cr.chunkDataLeft = 0
 		cr.chunkHash.Write(p[:chunkSize])
+		if cr.checksumHash != nil {
+			cr.checksumHash.Write(p[:chunkSize])
+		}
 		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
 		if (chunkSize + int64(n)) > math.MaxInt {
 			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
@@ -200,6 +286,9 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 
 	cr.chunkDataLeft = chunkSize - int64(n)
 	cr.chunkHash.Write(p[:n])
+	if cr.checksumHash != nil {
+		cr.checksumHash.Write(p[:n])
+	}
 
 	return n, nil
 }
@@ -239,38 +328,167 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
 		return 0, "", 0, errInvalidChunkFormat
 	}
 	if cr.stash != nil {
-		tmp := make([]byte, maxHeaderSize)
+		tmp := make([]byte, stashLen+len(header))
 		copy(tmp, cr.stash)
 		copy(tmp[len(cr.stash):], header)
 		header = tmp
 		cr.stash = nil
 	}
 
-	semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
-	if semicolonIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
+	rdr := bufio.NewReader(bytes.NewReader(header))
+
+	// After the first chunk each chunk header should start
+	// with "\n\r\n"
+	if !cr.isFirstHeader {
+		err := readAndSkip(rdr, '\r', '\n')
+		if err != nil {
+			return cr.handleRdrErr(err, header)
+		}
 	}
 
-	sigIndex := semicolonIndex + len(chunkHdrStr)
-	sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
-	if sigEndIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
+	// read and parse the chunk size
+	chunkSizeStr, err := readAndTrim(rdr, ';')
+	if err != nil {
+		return cr.handleRdrErr(err, header)
 	}
-
-	chunkSizeBytes := header[:semicolonIndex]
-	chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
+	chunkSize, err := strconv.ParseInt(chunkSizeStr, 16, 64)
 	if err != nil {
 		return 0, "", 0, errInvalidChunkFormat
 	}
 
-	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
-	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
+	// read the chunk signature
+	err = readAndSkip(rdr, 'c', 'h', 'u', 'n', 'k', '-', 's', 'i', 'g', 'n', 'a', 't', 'u', 'r', 'e', '=')
+	if err != nil {
+		return cr.handleRdrErr(err, header)
+	}
+	sig, err := readAndTrim(rdr, '\r')
+	if err != nil {
+		return cr.handleRdrErr(err, header)
+	}
 
-	return chunkSize, signature, dataStartOffset - stashLen, nil
+	// read and parse the final chunk trailer and checksum
+	if chunkSize == 0 {
+		if cr.trailer != "" {
+			err = readAndSkip(rdr, '\n')
+			if err != nil {
+				return cr.handleRdrErr(err, header)
+			}
+			// parse and validate the trailing header
+			trailer, err := readAndTrim(rdr, ':')
+			if err != nil {
+				return cr.handleRdrErr(err, header)
+			}
+			if trailer != string(cr.trailer) {
+				return 0, "", 0, errInvalidChunkFormat
+			}
+
+			algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(trailer, "x-amz-checksum-")))
+
+			// parse the checksum
+			checksum, err := readAndTrim(rdr, '\r')
+			if err != nil {
+				return cr.handleRdrErr(err, header)
+			}
+
+			if !IsValidChecksum(checksum, algo, cr.debug) {
+				return 0, "", 0, s3err.GetInvalidTrailingChecksumHeaderErr(trailer)
+			}
+
+			err = readAndSkip(rdr, '\n')
+			if err != nil {
+				return cr.handleRdrErr(err, header)
+			}
+
+			// parse the trailing signature
+			trailerSigPrefix, err := readAndTrim(rdr, ':')
+			if err != nil {
+				return cr.handleRdrErr(err, header)
+			}
+
+			if trailerSigPrefix != trailerSignatureHeader {
+				return 0, "", 0, errInvalidChunkFormat
+			}
+
+			trailerSig, err := readAndTrim(rdr, '\r')
+			if err != nil {
+				return cr.handleRdrErr(err, header)
+			}
+
+			cr.trailerSig = trailerSig
+			cr.parsedChecksum = checksum
+		}
+
+		// "\r\n\r\n" is followed after the last chunk
+		err = readAndSkip(rdr, '\n', '\r', '\n')
+		if err != nil {
+			return cr.handleRdrErr(err, header)
+		}
+
+		return 0, sig, 0, nil
+	}
+
+	err = readAndSkip(rdr, '\n')
+	if err != nil {
+		return cr.handleRdrErr(err, header)
+	}
+
+	// find the index of chunk ending: '\r\n'
+	// skip the first 2 bytes as it is the starting '\r\n'
+	// the first chunk doesn't contain the starting '\r\n', but
+	// anyway, trimming the first 2 bytes doesn't pollute the logic.
+	ind := bytes.Index(header[2:], []byte{'\r', '\n'})
+	cr.isFirstHeader = false
+
+	// the offset is the found index + 4 - the stash length
+	// where:
+	// ind is the index of '\r\n'
+	// 4 specifies the trimmed 2 bytes plus 2 to shift the index at the end of '\r\n'
+	offset := ind + 4 - stashLen
+	return chunkSize, sig, offset, nil
+}
+
+// Stashes the header in cr.stash and returns "errskipHeader"
+func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, error) {
+	cr.stash = make([]byte, len(header))
+	copy(cr.stash, header)
+	return 0, "", 0, errskipHeader
+}
+
+// Returns "errInvalidChunkFormat" if the passed err is "io.EOF" and cr.rdr EOF is reached
+// calls "cr.stashAndSkipHeader" if the passed err is "io.EOF" and cr.isEOF is false
+// Returns the error otherwise
+func (cr *ChunkReader) handleRdrErr(err error, header []byte) (int64, string, int, error) {
+	if err == io.EOF {
+		if cr.isEOF {
+			return 0, "", 0, errInvalidChunkFormat
+		}
+		return cr.stashAndSkipHeader(header)
+	}
+	return 0, "", 0, err
+}
+
+// reads data from the "rdr" and validates the passed data bytes
+func readAndSkip(rdr *bufio.Reader, data ...byte) error {
+	for _, d := range data {
+		b, err := rdr.ReadByte()
+		if err != nil {
+			return err
+		}
+
+		if b != d {
+			return errMalformedEncoding
+		}
+	}
+
+	return nil
+}
+
+// reads string by "delim" and trims the delimiter at the end
+func readAndTrim(r *bufio.Reader, delim byte) (string, error) {
+	str, err := r.ReadString(delim)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSuffix(str, string(delim)), nil
 }

s3api/utils/unsigned-chunk-reader.go

@@ -44,9 +44,11 @@ type UnsignedChunkReader struct {
 	stash            []byte
 	chunkCounter     int
 	offset           int
+	//TODO: Add debug logging for the reader
+	debug bool
 }
 
-func NewUnsignedChunkReader(r io.Reader, ct checksumType) (*UnsignedChunkReader, error) {
+func NewUnsignedChunkReader(r io.Reader, ct checksumType, debug bool) (*UnsignedChunkReader, error) {
 	hasher, err := getHasher(ct)
 	if err != nil {
 		return nil, err
@@ -57,6 +59,7 @@ func NewUnsignedChunkReader(r io.Reader, ct checksumType) (*UnsignedChunkReader,
 		stash:        make([]byte, 0),
 		hasher:       hasher,
 		chunkCounter: 1,
+		debug:        debug,
 	}, nil
 }
 

s3api/utils/utils.go

@@ -16,6 +16,8 @@ package utils
 
 import (
 	"bytes"
+	"encoding/base64"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
@@ -30,6 +32,7 @@ import (
 	"github.com/aws/smithy-go/encoding/httpbinding"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3api/debuglogger"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
@@ -174,13 +177,25 @@ func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
 	ctx.Response().Header.EnableNormalizing()
 }
 
-func ParseUint(str string) (int32, error) {
+func ParseUint(str string, debug bool) (int32, error) {
 	if str == "" {
 		return 1000, nil
 	}
-	num, err := strconv.ParseUint(str, 10, 16)
+	num, err := strconv.ParseInt(str, 10, 32)
 	if err != nil {
-		return 1000, fmt.Errorf("invalid uint: %w", err)
+		if debug {
+			debuglogger.Logf("invalid intager provided: %v\n", err)
+		}
+		return 1000, fmt.Errorf("invalid int: %w", err)
+	}
+	if num < 0 {
+		if debug {
+			debuglogger.Logf("negative intager provided: %v\n", num)
+		}
+		return 1000, fmt.Errorf("negative uint: %v", num)
+	}
+	if num > 1000 {
+		num = 1000
 	}
 	return int32(num), nil
 }
@@ -197,40 +212,26 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
 }
 
 // Streams the response body by chunks
-func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser) error {
-	buf := make([]byte, 4096) // 4KB chunks
-	defer rdr.Close()
-	for {
-		n, err := rdr.Read(buf)
-		if n > 0 {
-			_, writeErr := ctx.Write(buf[:n])
-			if writeErr != nil {
-				return fmt.Errorf("write chunk: %w", writeErr)
-			}
-		}
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-
-			return fmt.Errorf("read chunk: %w", err)
-		}
-	}
-
-	return nil
+func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser, bodysize int) {
+	// SetBodyStream will call Close() on the reader when the stream is done
+	// since rdr is a ReadCloser
+	ctx.Context().SetBodyStream(rdr, bodysize)
 }
 
-func IsValidBucketName(bucket string) bool {
+func IsValidBucketName(bucket string, debug bool) bool {
 	if len(bucket) < 3 || len(bucket) > 63 {
+		debuglogger.Logf("bucket name length should be in 3-63 range, got: %v\n", len(bucket))
 		return false
 	}
 	// Checks to contain only digits, lowercase letters, dot, hyphen.
 	// Checks to start and end with only digits and lowercase letters.
 	if !bucketNameRegexp.MatchString(bucket) {
+		debuglogger.Logf("invalid bucket name: %v\n", bucket)
 		return false
 	}
 	// Checks not to be a valid IP address
 	if bucketNameIpRegexp.MatchString(bucket) {
+		debuglogger.Logf("bucket name is an ip address: %v\n", bucket)
 		return false
 	}
 	return true
@@ -296,20 +297,28 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp
 	if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
 		output.StorageClass = ""
 	}
-	fmt.Printf("%+v\n", output)
+	if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok {
+		output.Checksum = nil
+	}
 
 	return output
 }
 
-func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
+func ParseObjectAttributes(ctx *fiber.Ctx, debug bool) (map[s3response.ObjectAttributes]struct{}, error) {
 	attrs := map[s3response.ObjectAttributes]struct{}{}
 	var err error
 	ctx.Request().Header.VisitAll(func(key, value []byte) {
 		if string(key) == "X-Amz-Object-Attributes" {
+			if len(value) == 0 {
+				return
+			}
 			oattrs := strings.Split(string(value), ",")
 			for _, a := range oattrs {
 				attr := s3response.ObjectAttributes(a)
 				if !attr.IsValid() {
+					if debug {
+						debuglogger.Logf("invalid object attribute: %v\n", attr)
+					}
 					err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
 					break
 				}
@@ -318,11 +327,18 @@ func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]stru
 		}
 	})
 
+	if err != nil {
+		return nil, err
+	}
+
 	if len(attrs) == 0 {
+		if debug {
+			debuglogger.Logf("empty get object attributes")
+		}
 		return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
 	}
 
-	return attrs, err
+	return attrs, nil
 }
 
 type objLockCfg struct {
@@ -331,12 +347,15 @@ type objLockCfg struct {
 	LegalHoldStatus types.ObjectLockLegalHoldStatus
 }
 
-func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
+func ParsObjectLockHdrs(ctx *fiber.Ctx, debug bool) (*objLockCfg, error) {
 	legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold")
 	objLockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode")
 	objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")
 
 	if (objLockDate != "" && objLockModeHdr == "") || (objLockDate == "" && objLockModeHdr != "") {
+		if debug {
+			debuglogger.Logf("one of 2 required params is missing: (lock date): %v, (lock mode): %v\n", objLockDate, objLockModeHdr)
+		}
 		return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)
 	}
 
@@ -344,9 +363,15 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 	if objLockDate != "" {
 		rDate, err := time.Parse(time.RFC3339, objLockDate)
 		if err != nil {
+			if debug {
+				debuglogger.Logf("failed to parse retain until date: %v\n", err)
+			}
 			return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 		}
 		if rDate.Before(time.Now()) {
+			if debug {
+				debuglogger.Logf("expired retain until date: %v\n", rDate.Format(time.RFC3339))
+			}
 			return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
 		}
 		retainUntilDate = rDate
@@ -357,13 +382,19 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 	if objLockMode != "" &&
 		objLockMode != types.ObjectLockModeCompliance &&
 		objLockMode != types.ObjectLockModeGovernance {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		if debug {
+			debuglogger.Logf("invalid object lock mode: %v\n", objLockMode)
+		}
+		return nil, s3err.GetAPIError(s3err.ErrInvalidObjectLockMode)
 	}
 
 	legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)
 
 	if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		if debug {
+			debuglogger.Logf("invalid object lock legal hold status: %v\n", legalHold)
+		}
+		return nil, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus)
 	}
 
 	return &objLockCfg{
@@ -373,7 +404,7 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 	}, nil
 }
 
-func IsValidOwnership(val types.ObjectOwnership) bool {
+func IsValidOwnership(val types.ObjectOwnership, debug bool) bool {
 	switch val {
 	case types.ObjectOwnershipBucketOwnerEnforced:
 		return true
@@ -382,6 +413,9 @@ func IsValidOwnership(val types.ObjectOwnership) bool {
 	case types.ObjectOwnershipObjectWriter:
 		return true
 	default:
+		if debug {
+			debuglogger.Logf("invalid object ownership: %v\n", val)
+		}
 		return false
 	}
 }
@@ -456,3 +490,304 @@ func shouldEscape(c byte) bool {
 
 	return true
 }
+
+type ChecksumValues map[types.ChecksumAlgorithm]string
+
+// Headers concatinates checksum algorithm by prefixing each
+// with 'x-amz-checksum-'
+// e.g.
+// "x-amz-checksum-crc64nvme, x-amz-checksum-sha1"
+func (cv ChecksumValues) Headers() string {
+	result := ""
+	isFirst := false
+
+	for key := range cv {
+		if !isFirst {
+			result += ", "
+		}
+		result += fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(key)))
+	}
+	return result
+}
+
+func ParseChecksumHeaders(ctx *fiber.Ctx, debug bool) (types.ChecksumAlgorithm, ChecksumValues, error) {
+	sdkAlgorithm := types.ChecksumAlgorithm(strings.ToUpper(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")))
+
+	err := IsChecksumAlgorithmValid(sdkAlgorithm, debug)
+	if err != nil {
+		if debug {
+			debuglogger.Logf("invalid checksum algorithm: %v\n", sdkAlgorithm)
+		}
+		return "", nil, err
+	}
+
+	checksums := ChecksumValues{}
+
+	var hdrErr error
+	// Parse and validate checksum headers
+	ctx.Request().Header.VisitAll(func(key, value []byte) {
+		// Skip `X-Amz-Checksum-Type` as it's a special header
+		if hdrErr != nil || !strings.HasPrefix(string(key), "X-Amz-Checksum-") || string(key) == "X-Amz-Checksum-Type" {
+			return
+		}
+
+		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(key), "X-Amz-Checksum-")))
+		err := IsChecksumAlgorithmValid(algo, debug)
+		if err != nil {
+			if debug {
+				debuglogger.Logf("invalid checksum header: %s\n", key)
+			}
+			hdrErr = s3err.GetAPIError(s3err.ErrInvalidChecksumHeader)
+			return
+		}
+
+		checksums[algo] = string(value)
+	})
+
+	if hdrErr != nil {
+		return sdkAlgorithm, nil, hdrErr
+	}
+
+	if len(checksums) > 1 {
+		if debug {
+			debuglogger.Logf("multiple checksum headers provided: %v\n", checksums.Headers())
+		}
+		return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
+	}
+
+	for al, val := range checksums {
+		if !IsValidChecksum(val, al, debug) {
+			return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al))))
+		}
+		// If any other checksum value is provided,
+		// rather than x-amz-sdk-checksum-algorithm
+		if sdkAlgorithm != "" && sdkAlgorithm != al {
+			return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
+		}
+		sdkAlgorithm = al
+	}
+
+	return sdkAlgorithm, checksums, nil
+}
+
+var checksumLengths = map[types.ChecksumAlgorithm]int{
+	types.ChecksumAlgorithmCrc32:     4,
+	types.ChecksumAlgorithmCrc32c:    4,
+	types.ChecksumAlgorithmCrc64nvme: 8,
+	types.ChecksumAlgorithmSha1:      20,
+	types.ChecksumAlgorithmSha256:    32,
+}
+
+func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm, debug bool) bool {
+	decoded, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		if debug {
+			debuglogger.Logf("failed to parse checksum base64: %v\n", err)
+		}
+		return false
+	}
+
+	expectedLength, exists := checksumLengths[algorithm]
+	if !exists {
+		if debug {
+			debuglogger.Logf("unknown checksum algorithm: %v\n", algorithm)
+		}
+		return false
+	}
+
+	isValid := len(decoded) == expectedLength
+	if !isValid && debug {
+		debuglogger.Logf("decoded checksum length: (expected): %v, (got): %v\n", expectedLength, len(decoded))
+	}
+
+	return isValid
+}
+
+func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm, debug bool) error {
+	alg = types.ChecksumAlgorithm(strings.ToUpper(string(alg)))
+	if alg != "" &&
+		alg != types.ChecksumAlgorithmCrc32 &&
+		alg != types.ChecksumAlgorithmCrc32c &&
+		alg != types.ChecksumAlgorithmSha1 &&
+		alg != types.ChecksumAlgorithmSha256 &&
+		alg != types.ChecksumAlgorithmCrc64nvme {
+		if debug {
+			debuglogger.Logf("invalid checksum algorithm: %v\n", alg)
+		}
+		return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
+	}
+
+	return nil
+}
+
+// Validates the provided checksum type
+func IsChecksumTypeValid(t types.ChecksumType, debug bool) error {
+	if t != "" &&
+		t != types.ChecksumTypeComposite &&
+		t != types.ChecksumTypeFullObject {
+		if debug {
+			debuglogger.Logf("invalid checksum type: %v\n", t)
+		}
+		return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")
+	}
+	return nil
+}
+
+type checksumTypeSchema map[types.ChecksumType]struct{}
+type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema
+
+// A table defining the checksum algorithm/type support
+var checksumMap checksumSchema = checksumSchema{
+	types.ChecksumAlgorithmCrc32: checksumTypeSchema{
+		types.ChecksumTypeComposite:  struct{}{},
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	types.ChecksumAlgorithmCrc32c: checksumTypeSchema{
+		types.ChecksumTypeComposite:  struct{}{},
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	types.ChecksumAlgorithmSha1: checksumTypeSchema{
+		types.ChecksumTypeComposite: struct{}{},
+		"":                          struct{}{},
+	},
+	types.ChecksumAlgorithmSha256: checksumTypeSchema{
+		types.ChecksumTypeComposite: struct{}{},
+		"":                          struct{}{},
+	},
+	types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	// Both could be empty
+	"": checksumTypeSchema{
+		"": struct{}{},
+	},
+}
+
+// Checks if checksum type and algorithm are supported together
+func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType, debug bool) error {
+	typeSchema := checksumMap[algo]
+	_, ok := typeSchema[t]
+	if !ok {
+		if debug {
+			debuglogger.Logf("checksum type and algorithm mismatch: (type): %v, (algorithm): %v\n", t, algo)
+		}
+		return s3err.GetChecksumSchemaMismatchErr(algo, t)
+	}
+
+	return nil
+}
+
+// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers
+func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx, debug bool) (types.ChecksumAlgorithm, types.ChecksumType, error) {
+	algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
+	if err := IsChecksumAlgorithmValid(algo, debug); err != nil {
+		return "", "", err
+	}
+
+	chType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
+	if err := IsChecksumTypeValid(chType, debug); err != nil {
+		return "", "", err
+	}
+
+	// Verify if checksum algorithm is provided, if
+	// checksum type is specified
+	if chType != "" && algo == "" {
+		if debug {
+			debuglogger.Logf("checksum type can only be used with checksum algorithm: (type): %v\n", chType)
+		}
+		return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)
+	}
+
+	// Verify if the checksum type is supported for
+	// the provided checksum algorithm
+	if err := checkChecksumTypeAndAlgo(algo, chType, debug); err != nil {
+		return algo, chType, err
+	}
+
+	// x-amz-checksum-type defaults to COMPOSITE
+	// if x-amz-checksum-algorithm is set except
+	// for the CRC64NVME algorithm: it defaults to FULL_OBJECT
+	if algo != "" && chType == "" {
+		if algo == types.ChecksumAlgorithmCrc64nvme {
+			chType = types.ChecksumTypeFullObject
+		} else {
+			chType = types.ChecksumTypeComposite
+		}
+	}
+
+	return algo, chType, nil
+}
+
+// TagLimit specifies the allowed tag count in a tag set
+type TagLimit int
+
+const (
+	// Tag limit for bucket tagging
+	TagLimitBucket TagLimit = 50
+	// Tag limit for object tagging
+	TagLimitObject TagLimit = 10
+)
+
+// Parses and validates tagging
+func ParseTagging(data []byte, limit TagLimit, debug bool) (map[string]string, error) {
+	var tagging s3response.TaggingInput
+	err := xml.Unmarshal(data, &tagging)
+	if err != nil {
+		if debug {
+			debuglogger.Logf("invalid taggging: %s", data)
+		}
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	tLen := len(tagging.TagSet.Tags)
+	if tLen > int(limit) {
+		switch limit {
+		case TagLimitObject:
+			if debug {
+				debuglogger.Logf("bucket tagging length exceeds %v: %v", limit, tLen)
+			}
+			return nil, s3err.GetAPIError(s3err.ErrObjectTaggingLimited)
+		case TagLimitBucket:
+			if debug {
+				debuglogger.Logf("object tagging length exceeds %v: %v", limit, tLen)
+			}
+			return nil, s3err.GetAPIError(s3err.ErrBucketTaggingLimited)
+		}
+	}
+
+	tagSet := make(map[string]string, tLen)
+
+	for _, tag := range tagging.TagSet.Tags {
+		// validate tag key
+		if len(tag.Key) == 0 || len(tag.Key) > 128 {
+			if debug {
+				debuglogger.Logf("tag key should 0 < tag.Key <= 128, key: %v", tag.Key)
+			}
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// validate tag value
+		if len(tag.Value) > 256 {
+			if debug {
+				debuglogger.Logf("invalid long tag value: (length): %v, (value): %v", len(tag.Value), tag.Value)
+			}
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// make sure there are no duplicate keys
+		_, ok := tagSet[tag.Key]
+		if ok {
+			if debug {
+				debuglogger.Logf("duplicate tag key: %v", tag.Key)
+			}
+			return nil, s3err.GetAPIError(s3err.ErrDuplicateTagKey)
+		}
+
+		tagSet[tag.Key] = tag.Value
+	}
+
+	return tagSet, nil
+}

s3api/utils/utils_test.go

@@ -16,6 +16,9 @@ package utils
 
 import (
 	"bytes"
+	"encoding/xml"
+	"errors"
+	"math/rand"
 	"net/http"
 	"reflect"
 	"testing"
@@ -25,6 +28,7 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
 
@@ -219,7 +223,7 @@ func TestIsValidBucketName(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := IsValidBucketName(tt.args.bucket); got != tt.want {
+			if got := IsValidBucketName(tt.args.bucket, false); got != tt.want {
 				t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want)
 			}
 		})
@@ -268,10 +272,18 @@ func TestParseUint(t *testing.T) {
 			want:    23,
 			wantErr: false,
 		},
+		{
+			name: "Parse-uint-greater-than-1000",
+			args: args{
+				str: "25000000",
+			},
+			want:    1000,
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := ParseUint(tt.args.str)
+			got, err := ParseUint(tt.args.str, false)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ParseMaxKeys() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -399,7 +411,7 @@ func TestIsValidOwnership(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := IsValidOwnership(tt.args.val); got != tt.want {
+			if got := IsValidOwnership(tt.args.val, false); got != tt.want {
 				t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want)
 			}
 		})
@@ -527,3 +539,485 @@ func Test_escapePath(t *testing.T) {
 		})
 	}
 }
+
+func TestIsChecksumAlgorithmValid(t *testing.T) {
+	type args struct {
+		alg types.ChecksumAlgorithm
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "empty",
+			args: args{
+				alg: "",
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc32",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc32,
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc32c",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc32c,
+			},
+			wantErr: false,
+		},
+		{
+			name: "sha1",
+			args: args{
+				alg: types.ChecksumAlgorithmSha1,
+			},
+			wantErr: false,
+		},
+		{
+			name: "sha256",
+			args: args{
+				alg: types.ChecksumAlgorithmSha256,
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc64nvme",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc64nvme,
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid",
+			args: args{
+				alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := IsChecksumAlgorithmValid(tt.args.alg, false); (err != nil) != tt.wantErr {
+				t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestIsValidChecksum(t *testing.T) {
+	type args struct {
+		checksum  string
+		algorithm types.ChecksumAlgorithm
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "invalid-base64",
+			args: args{
+				checksum:  "invalid_base64_string",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: false,
+		},
+		{
+			name: "invalid-crc32",
+			args: args{
+				checksum:  "YXNkZmFzZGZhc2Rm",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: false,
+		},
+		{
+			name: "valid-crc32",
+			args: args{
+				checksum:  "ww2FVQ==",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-crc32c",
+			args: args{
+				checksum:  "Zmdoa2doZmtnZmhr",
+				algorithm: types.ChecksumAlgorithmCrc32c,
+			},
+			want: false,
+		},
+		{
+			name: "valid-crc32c",
+			args: args{
+				checksum:  "DOsb4w==",
+				algorithm: types.ChecksumAlgorithmCrc32c,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-sha1",
+			args: args{
+				checksum:  "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm",
+				algorithm: types.ChecksumAlgorithmSha1,
+			},
+			want: false,
+		},
+		{
+			name: "valid-sha1",
+			args: args{
+				checksum:  "L4q6V59Zcwn12wyLIytoE2c1ugk=",
+				algorithm: types.ChecksumAlgorithmSha1,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-sha256",
+			args: args{
+				checksum:  "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==",
+				algorithm: types.ChecksumAlgorithmSha256,
+			},
+			want: false,
+		},
+		{
+			name: "valid-sha256",
+			args: args{
+				checksum:  "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
+				algorithm: types.ChecksumAlgorithmSha256,
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm, false); got != tt.want {
+				t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsChecksumTypeValid(t *testing.T) {
+	type args struct {
+		t types.ChecksumType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "valid_FULL_OBJECT",
+			args: args{
+				t: types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "valid_COMPOSITE",
+			args: args{
+				t: types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid",
+			args: args{
+				t: types.ChecksumType("invalid_checksum_type"),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := IsChecksumTypeValid(tt.args.t, false); (err != nil) != tt.wantErr {
+				t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_checkChecksumTypeAndAlgo(t *testing.T) {
+	type args struct {
+		algo types.ChecksumAlgorithm
+		t    types.ChecksumType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "full_object-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-crc32c",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32c,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-sha1",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-sha256",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-crc64nvme",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc64nvme,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc32c",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32c,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-sha1",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-sha256",
+			args: args{
+				algo: types.ChecksumAlgorithmSha256,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc64nvme",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc64nvme,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: true,
+		},
+		{
+			name: "composite-empty",
+			args: args{
+				t: types.ChecksumTypeComposite,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-empty",
+			args: args{
+				t: types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t, false); (err != nil) != tt.wantErr {
+				t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestParseTagging(t *testing.T) {
+	genRandStr := func(lgth int) string {
+		b := make([]byte, lgth)
+		for i := range b {
+			b[i] = byte(rand.Intn(95) + 32) // 126 - 32 + 1 = 95 printable characters
+		}
+		return string(b)
+	}
+	getTagSet := func(lgth int) s3response.TaggingInput {
+		res := s3response.TaggingInput{
+			TagSet: s3response.TagSet{
+				Tags: []s3response.Tag{},
+			},
+		}
+
+		for i := 0; i < lgth; i++ {
+			res.TagSet.Tags = append(res.TagSet.Tags, s3response.Tag{
+				Key:   genRandStr(10),
+				Value: genRandStr(20),
+			})
+		}
+
+		return res
+	}
+	type args struct {
+		data        s3response.TaggingInput
+		overrideXML []byte
+		limit       TagLimit
+		debug       bool
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    map[string]string
+		wantErr error
+	}{
+		{
+			name: "valid tags within limit",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{
+							{Key: "key1", Value: "value1"},
+							{Key: "key2", Value: "value2"},
+						},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    map[string]string{"key1": "value1", "key2": "value2"},
+			wantErr: nil,
+		},
+		{
+			name: "malformed XML",
+			args: args{
+				overrideXML: []byte("invalid xml"),
+				limit:       TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrMalformedXML),
+		},
+		{
+			name: "exceeds bucket tag limit",
+			args: args{
+				data:  getTagSet(51),
+				limit: TagLimitBucket,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrBucketTaggingLimited),
+		},
+		{
+			name: "exceeds object tag limit",
+			args: args{
+				data:  getTagSet(11),
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrObjectTaggingLimited),
+		},
+		{
+			name: "invalid 0 length tag key",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{{Key: "", Value: "value1"}},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrInvalidTagKey),
+		},
+		{
+			name: "invalid long tag key",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{{Key: genRandStr(130), Value: "value1"}},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrInvalidTagKey),
+		},
+		{
+			name: "invalid long tag value",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{{Key: "key", Value: genRandStr(257)}},
+					},
+				},
+				limit: TagLimitBucket,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrInvalidTagValue),
+		},
+		{
+			name: "duplicate tag key",
+			args: args{
+				data: s3response.TaggingInput{
+					TagSet: s3response.TagSet{
+						Tags: []s3response.Tag{
+							{Key: "key", Value: "value1"},
+							{Key: "key", Value: "value2"},
+						},
+					},
+				},
+				limit: TagLimitObject,
+			},
+			want:    nil,
+			wantErr: s3err.GetAPIError(s3err.ErrDuplicateTagKey),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var data []byte
+			if tt.args.overrideXML != nil {
+				data = tt.args.overrideXML
+			} else {
+				var err error
+				data, err = xml.Marshal(tt.args.data)
+				if err != nil {
+					t.Fatalf("error marshalling input: %v", err)
+				}
+			}
+			got, err := ParseTagging(data, tt.args.limit, tt.args.debug)
+
+			if !errors.Is(err, tt.wantErr) {
+				t.Errorf("expected error %v, got %v", tt.wantErr, err)
+			}
+			if err == nil && !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("expected result %v, got %v", tt.want, got)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -17,7 +17,11 @@ package s3err
 import (
 	"bytes"
 	"encoding/xml"
+	"fmt"
 	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 // APIError structure
@@ -74,10 +78,17 @@ const (
 	ErrInvalidPart
 	ErrEmptyParts
 	ErrInvalidPartNumber
+	ErrInvalidPartOrder
+	ErrInvalidCompleteMpPartNumber
 	ErrInternalError
 	ErrInvalidCopyDest
 	ErrInvalidCopySource
-	ErrInvalidTag
+	ErrInvalidCopySourceRange
+	ErrInvalidTagKey
+	ErrInvalidTagValue
+	ErrDuplicateTagKey
+	ErrBucketTaggingLimited
+	ErrObjectTaggingLimited
 	ErrAuthHeaderEmpty
 	ErrSignatureVersionNotSupported
 	ErrMalformedPOSTRequest
@@ -106,6 +117,7 @@ const (
 	ErrSignatureTerminationStr
 	ErrSignatureIncorrService
 	ErrContentSHA256Mismatch
+	ErrMissingDecodedContentLength
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
 	ErrMissingDateHeader
@@ -123,6 +135,8 @@ const (
 	ErrObjectLocked
 	ErrPastObjectLockRetainDate
 	ErrObjectLockInvalidRetentionPeriod
+	ErrInvalidLegalHoldStatus
+	ErrInvalidObjectLockMode
 	ErrNoSuchBucketPolicy
 	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
@@ -136,10 +150,17 @@ const (
 	ErrUnexpectedContent
 	ErrMissingSecurityHeader
 	ErrInvalidMetadataDirective
+	ErrInvalidTaggingDirective
 	ErrKeyTooLong
 	ErrInvalidVersionId
 	ErrNoSuchVersion
 	ErrSuspendedVersioningNotAllowed
+	ErrMultipleChecksumHeaders
+	ErrInvalidChecksumAlgorithm
+	ErrInvalidChecksumPart
+	ErrChecksumTypeWithAlgo
+	ErrInvalidChecksumHeader
+	ErrTrailerHeaderNotSupported
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
@@ -264,6 +285,16 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Part number must be an integer between 1 and 10000, inclusive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidPartOrder: {
+		Code:           "InvalidPartOrder",
+		Description:    "The list of parts was not in ascending order. Parts must be ordered by part number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCompleteMpPartNumber: {
+		Code:           "InvalidArgument",
+		Description:    "PartNumber must be >= 1",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -274,9 +305,34 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidTag: {
+	ErrInvalidCopySourceRange: {
 		Code:           "InvalidArgument",
-		Description:    "The Tag value you have provided is invalid",
+		Description:    "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidTagKey: {
+		Code:           "InvalidTag",
+		Description:    "The TagKey you have provided is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidTagValue: {
+		Code:           "InvalidTag",
+		Description:    "The TagValue you have provided is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrDuplicateTagKey: {
+		Code:           "InvalidTag",
+		Description:    "Cannot provide multiple Tags with the same key",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketTaggingLimited: {
+		Code:           "BadRequest",
+		Description:    "Bucket tag count cannot be greater than 50",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectTaggingLimited: {
+		Code:           "BadRequest",
+		Description:    "Object tags cannot be greater than 10",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMalformedXML: {
@@ -424,6 +480,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingDecodedContentLength: {
+		Code:           "MissingContentLength",
+		Description:    "You must provide the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
 	ErrMissingDateHeader: {
 		Code:           "AccessDenied",
 		Description:    "AWS authentication requires a valid Date or x-amz-date header.",
@@ -499,6 +560,16 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "the retention days/years must be positive integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidLegalHoldStatus: {
+		Code:           "InvalidArgument",
+		Description:    "Legal Hold must be either of 'ON' or 'OFF'",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidObjectLockMode: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown wormMode directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchBucketPolicy: {
 		Code:           "NoSuchBucketPolicy",
 		Description:    "The bucket policy does not exist.",
@@ -564,6 +635,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Unknown metadata directive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidTaggingDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown tagging directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidVersionId: {
 		Code:           "InvalidArgument",
 		Description:    "Invalid version id specified",
@@ -584,6 +660,36 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMultipleChecksumHeaders: {
+		Code:           "InvalidRequest",
+		Description:    "Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumAlgorithm: {
+		Code:           "InvalidRequest",
+		Description:    "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumPart: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid Base64 or multiple checksums present in request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrChecksumTypeWithAlgo: {
+		Code:           "InvalidRequest",
+		Description:    "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumHeader: {
+		Code:           "InvalidRequest",
+		Description:    "The algorithm type you specified in x-amz-checksum- header is invalid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrTrailerHeaderNotSupported: {
+		Code:           "InvalidRequest",
+		Description:    "The value specified in the x-amz-trailer header is not supported",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {
@@ -678,3 +784,80 @@ func encodeResponse(response interface{}) []byte {
 	e.Encode(response)
 	return bytesBuffer.Bytes()
 }
+
+// Returns invalid checksum error with the provided header in the error description
+func GetInvalidChecksumHeaderErr(header string) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Value for %v header is invalid.", header),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func GetInvalidTrailingChecksumHeaderErr(header string) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Value for %v trailing header is invalid.", header),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch APIError
+func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Checksum Type mismatch occurred, expected checksum Type: %v, actual checksum Type: %v", expected, actual),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns incorrect checksum APIError
+func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError {
+	return APIError{
+		Code:           "BadDigest",
+		Description:    fmt.Sprintf("The %v you specified did not match the calculated checksum.", algo),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch error with checksum algorithm
+func GetChecksumSchemaMismatchErr(algo types.ChecksumAlgorithm, t types.ChecksumType) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The %v checksum type cannot be used with the %v checksum algorithm.", algo, strings.ToLower(string(t))),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch error for multipart uploads
+func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The upload was created using the %v checksum mode. The complete request must use the same checksum mode.", t),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func GetIncorrectMpObjectSizeErr(expected, actual int64) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The provided 'x-amz-mp-object-size' header value %v does not match what was computed: %v", expected, actual),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func GetInvalidMpObjectSizeErr(val int64) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Value for x-amz-mp-object-size header is less than zero: '%v'", val),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func CreateExceedingRangeErr(objSize int64) APIError {
+	return APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}

s3response/s3response.go

@@ -16,6 +16,7 @@ package s3response
 
 import (
 	"encoding/xml"
+	"io"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -29,16 +30,27 @@ const (
 )
 
 type PutObjectOutput struct {
-	ETag      string
-	VersionID string
+	ETag              string
+	VersionID         string
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+	ChecksumType      types.ChecksumType
 }
 
 // Part describes part metadata.
 type Part struct {
-	PartNumber   int
-	LastModified time.Time
-	ETag         string
-	Size         int64
+	PartNumber        int
+	LastModified      time.Time
+	ETag              string
+	Size              int64
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
 }
 
 func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -59,9 +71,11 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
-	Bucket   string
-	Key      string
-	UploadID string `xml:"UploadId"`
+	Bucket            string
+	Key               string
+	UploadID          string `xml:"UploadId"`
+	ChecksumAlgorithm types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
 
 	Initiator Initiator
 	Owner     Owner
@@ -101,6 +115,7 @@ type GetObjectAttributesResponse struct {
 	ObjectSize   *int64
 	StorageClass types.StorageClass `xml:",omitempty"`
 	ObjectParts  *ObjectParts
+	Checksum     *types.Checksum
 
 	// Not included in the response body
 	VersionId    *string
@@ -169,13 +184,15 @@ type ListObjectsV2Result struct {
 }
 
 type Object struct {
-	ETag          *string
-	Key           *string
-	LastModified  *time.Time
-	Owner         *types.Owner
-	RestoreStatus *types.RestoreStatus
-	Size          *int64
-	StorageClass  types.ObjectStorageClass
+	ChecksumAlgorithm []types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
+	ETag              *string
+	Key               *string
+	LastModified      *time.Time
+	Owner             *types.Owner
+	RestoreStatus     *types.RestoreStatus
+	Size              *int64
+	StorageClass      types.ObjectStorageClass
 }
 
 func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -197,12 +214,14 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 // Upload describes in progress multipart upload
 type Upload struct {
-	Key          string
-	UploadID     string `xml:"UploadId"`
-	Initiator    Initiator
-	Owner        Owner
-	StorageClass types.StorageClass
-	Initiated    time.Time
+	Key               string
+	UploadID          string `xml:"UploadId"`
+	Initiator         Initiator
+	Owner             Owner
+	StorageClass      types.StorageClass
+	Initiated         time.Time
+	ChecksumAlgorithm types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
 }
 
 func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -332,6 +351,20 @@ type CopyObjectResult struct {
 	CopySourceVersionId string `xml:"-"`
 }
 
+type CopyPartResult struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
+	LastModified      time.Time
+	ETag              *string
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+
+	// not included in the body
+	CopySourceVersionId string `xml:"-"`
+}
+
 func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias CopyObjectResult
 	aux := &struct {
@@ -415,6 +448,134 @@ type PutObjectRetentionInput struct {
 	RetainUntilDate AmzDate
 }
 
+type PutObjectInput struct {
+	ContentLength             *int64
+	ObjectLockRetainUntilDate *time.Time
+
+	Bucket                  *string
+	Key                     *string
+	ContentType             *string
+	ContentEncoding         *string
+	ContentDisposition      *string
+	ContentLanguage         *string
+	CacheControl            *string
+	Expires                 *string
+	Tagging                 *string
+	ChecksumCRC32           *string
+	ChecksumCRC32C          *string
+	ChecksumSHA1            *string
+	ChecksumSHA256          *string
+	ChecksumCRC64NVME       *string
+	ContentMD5              *string
+	ExpectedBucketOwner     *string
+	GrantFullControl        *string
+	GrantRead               *string
+	GrantReadACP            *string
+	GrantWriteACP           *string
+	IfMatch                 *string
+	IfNoneMatch             *string
+	SSECustomerAlgorithm    *string
+	SSECustomerKey          *string
+	SSECustomerKeyMD5       *string
+	SSEKMSEncryptionContext *string
+	SSEKMSKeyId             *string
+	WebsiteRedirectLocation *string
+
+	ObjectLockMode            types.ObjectLockMode
+	ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	ChecksumAlgorithm         types.ChecksumAlgorithm
+
+	Metadata map[string]string
+	Body     io.Reader
+}
+
+type CreateMultipartUploadInput struct {
+	Bucket                    *string
+	Key                       *string
+	ExpectedBucketOwner       *string
+	CacheControl              *string
+	ContentDisposition        *string
+	ContentEncoding           *string
+	ContentLanguage           *string
+	ContentType               *string
+	Expires                   *string
+	SSECustomerAlgorithm      *string
+	SSECustomerKey            *string
+	SSECustomerKeyMD5         *string
+	SSEKMSEncryptionContext   *string
+	SSEKMSKeyId               *string
+	GrantFullControl          *string
+	GrantRead                 *string
+	GrantReadACP              *string
+	GrantWriteACP             *string
+	Tagging                   *string
+	WebsiteRedirectLocation   *string
+	BucketKeyEnabled          *bool
+	ObjectLockRetainUntilDate *time.Time
+	Metadata                  map[string]string
+
+	ACL                       types.ObjectCannedACL
+	ChecksumAlgorithm         types.ChecksumAlgorithm
+	ChecksumType              types.ChecksumType
+	ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	ObjectLockMode            types.ObjectLockMode
+	RequestPayer              types.RequestPayer
+	ServerSideEncryption      types.ServerSideEncryption
+	StorageClass              types.StorageClass
+}
+
+type CopyObjectInput struct {
+	Metadata                       map[string]string
+	Bucket                         *string
+	CopySource                     *string
+	Key                            *string
+	CacheControl                   *string
+	ContentDisposition             *string
+	ContentEncoding                *string
+	ContentLanguage                *string
+	ContentType                    *string
+	CopySourceIfMatch              *string
+	CopySourceIfNoneMatch          *string
+	CopySourceSSECustomerAlgorithm *string
+	CopySourceSSECustomerKey       *string
+	CopySourceSSECustomerKeyMD5    *string
+	ExpectedBucketOwner            *string
+	ExpectedSourceBucketOwner      *string
+	Expires                        *string
+	GrantFullControl               *string
+	GrantRead                      *string
+	GrantReadACP                   *string
+	GrantWriteACP                  *string
+	SSECustomerAlgorithm           *string
+	SSECustomerKey                 *string
+	SSECustomerKeyMD5              *string
+	SSEKMSEncryptionContext        *string
+	SSEKMSKeyId                    *string
+	Tagging                        *string
+	WebsiteRedirectLocation        *string
+
+	CopySourceIfModifiedSince   *time.Time
+	CopySourceIfUnmodifiedSince *time.Time
+	ObjectLockRetainUntilDate   *time.Time
+
+	BucketKeyEnabled *bool
+
+	ACL                       types.ObjectCannedACL
+	ChecksumAlgorithm         types.ChecksumAlgorithm
+	MetadataDirective         types.MetadataDirective
+	ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	ObjectLockMode            types.ObjectLockMode
+	RequestPayer              types.RequestPayer
+	ServerSideEncryption      types.ServerSideEncryption
+	StorageClass              types.StorageClass
+	TaggingDirective          types.TaggingDirective
+}
+
+type GetObjectLegalHoldResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LegalHold"`
+	Status  types.ObjectLockLegalHoldStatus
+}
+
 type AmzDate struct {
 	time.Time
 }
@@ -467,3 +628,14 @@ func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
 type ListBucketsResult struct {
 	Buckets []Bucket
 }
+
+type Checksum struct {
+	Algorithm types.ChecksumAlgorithm
+	Type      types.ChecksumType
+
+	CRC32     *string
+	CRC32C    *string
+	SHA1      *string
+	SHA256    *string
+	CRC64NVME *string
+}

tests/.env.docker.default

@@ -25,5 +25,8 @@ USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 RECREATE_BUCKETS=true
+DELETE_BUCKETS_AFTER_TEST=true
 REMOVE_TEST_FILE_FOLDER=true
+AUTOGENERATE_USERS=true
+USER_AUTOGENERATION_PREFIX=versitygw-docker-
 VERSIONING_DIR=/tmp/versioning

tests/README.md

@@ -110,6 +110,11 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 **ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**:  for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account.
 
+**USER_ID_{role}_{id}**, **USERNAME_{role}_{id}**, **PASSWORD_{role}_{id}**:  for setup_user_v2 non-autocreated users, the format for the user.
+* example:  USER_ID_USER_1={name}:  user ID corresponding to the first user with **user** permissions in the test.
+
+#### 
+
 ### Non-Secret
 
 **VERSITY_EXE**:  location of the versity executable relative to test folder.
@@ -150,7 +155,7 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 **COVERAGE_DB**:  database to store client command coverage info and usage counts, if using.
 
-**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  credentials for users created and tested for non-root user **versitygw** operations.
+**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  credentials for users created and tested for non-root user **versitygw** operations (non-setup_user_v2).
 
 **TEST_FILE_FOLDER**:  where to put temporary test files.
 
@@ -166,6 +171,10 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 **DELETE_BUCKETS_AFTER_TEST**:  whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked
 
+**AUTOCREATE_USERS**:  setup_user_v2, whether or not to autocreate users for tests.  If set to **false**, users must be pre-created (see `Secret` section above).
+
+**USER_AUTOCREATION_PREFIX**:  setup_user_v2, if **AUTOCREATE_USERS** is set to **true**, the prefix for the autocreated username. 
+
 ## REST Scripts
 
 REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder.  To call a script, the following parameters are needed:

tests/commands/create_bucket.sh

@@ -85,5 +85,8 @@ create_bucket_object_lock_enabled() {
     log 2 "error creating bucket: $error"
     return 1
   fi
+  if [ "$DIRECT" == "true" ]; then
+    sleep 15
+  fi
   return 0
 }

tests/commands/delete_object.sh

@@ -92,7 +92,7 @@ delete_object_with_user() {
   if [[ $1 == 's3' ]]; then
     delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
   else

tests/commands/get_bucket_ownership_controls.sh

@@ -26,6 +26,7 @@ get_bucket_ownership_controls() {
     return 1
   fi
 
+  raw_bucket_ownership_controls=""
   if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
     log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
     return 1

tests/commands/get_bucket_policy.sh

@@ -105,6 +105,34 @@ get_bucket_policy_s3cmd() {
   return 0
 }
 
+get_bucket_policy_rest() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "s3cmd 'get bucket policy' command requires bucket name"
+    return 1
+  fi
+  if ! get_bucket_policy_rest_expect_code "$1" "200"; then
+    log 2 "error getting REST bucket policy"
+    return 1
+  fi
+  return 0
+}
+
+get_bucket_policy_rest_expect_code() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "s3cmd 'get bucket policy' command requires bucket name, expected code"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then
+    log 2 "error attempting to get bucket policy response: $result"
+    return 1
+  fi
+  if [ "$result" != "$2" ]; then
+    log 2 "unexpected response code, expected '$2', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/policy.txt"))"
+    return 1
+  fi
+  bucket_policy="$(cat "$TEST_FILE_FOLDER/policy.txt")"
+}
+
 # return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading
 check_and_load_policy_info() {
   if [[ $policy_brackets == false ]]; then

tests/commands/get_object.sh

@@ -73,6 +73,8 @@ get_object_with_user() {
   elif [[ $1 == "mc" ]]; then
     log 5 "save location: $4"
     get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == "rest" ]]; then
+    get_object_rest_with_user "$5" "$6" "$2" "$3" "$4" || exit_code=$?
   else
     log 2 "'get_object_with_user' not implemented for client '$1'"
     return 1
@@ -91,37 +93,24 @@ get_object_rest() {
     log 2 "'get_object_rest' requires bucket name, object name, output file"
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1/$2
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
-    return 1
-  fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1/$2" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$3" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$3")"
+  if ! get_object_rest_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3"; then
+    log 2 "error getting REST object with root user"
+    return 1
+  fi
+  return 0
+}
+
+get_object_rest_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "'get_object_rest_with_user' requires username, password, bucket name, object name, output file"
+    return 1
+  fi
+  if ! result=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$3" OBJECT_KEY="$4" OUTPUT_FILE="$5" ./tests/rest_scripts/get_object.sh 2>&1); then
+    log 2 "error getting object: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200', was '$result' ($(cat "$3"))"
     return 1
   fi
   return 0

tests/commands/get_object_legal_hold.sh

@@ -42,4 +42,18 @@ get_object_legal_hold_rest() {
     return 1
   fi
   return 0
-}
+}
+
+get_object_legal_hold_version_id() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'get_object_legal_hold_version_id' command requires bucket, key, version id"
+    return 1
+  fi
+  record_command "get-object-legal-hold" "client:s3api"
+  if ! legal_hold=$(send_command aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" --version-id "$3" 2>&1); then
+    log 2 "error getting object legal hold w/version id: $legal_hold"
+    return 1
+  fi
+  echo "$legal_hold"
+  return 0
+}

tests/commands/get_object_lock_configuration.sh

@@ -36,35 +36,12 @@ get_object_lock_configuration_rest() {
     log 2 "'get_object_lock_configuration_rest' requires bucket name"
     return 1
   fi
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-object-lock=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object-lock-config.txt" ./tests/rest_scripts/get_object_lock_config.sh); then
+    log 2 "error getting lock configuration: $result"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
+  if [[ "$result" != "200" ]]; then
+    log 2 "expected '200', returned '$result': $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
     return 1
   fi
   return 0

tests/commands/head_bucket.sh

@@ -35,16 +35,40 @@ head_bucket() {
     bucket_info=$(send_command s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
     bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'rest' ]]; then
+    bucket_info=$(head_bucket_rest "$2") || exit_code=$?
+    return $exit_code
   else
     log 2 "invalid command type $1"
   fi
   if [ $exit_code -ne 0 ]; then
-    log 2 "error getting bucket info: $bucket_info"
     if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
       return 1
     fi
+    log 2 "error getting bucket info: $bucket_info"
     return 2
   fi
+  bucket_info="$(echo -n "$bucket_info" | grep -v "InsecureRequestWarning")"
   echo "$bucket_info"
   return 0
 }
+
+head_bucket_rest() {
+  if [ $# -ne 1 ]; then
+    log 2 "'head_bucket_rest' requires bucket name"
+    return 2
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/head_bucket.sh 2>&1); then
+    log 2 "error getting head bucket"
+    return 2
+  fi
+  if [ "$result" == "200" ]; then
+    bucket_info="$(cat "$TEST_FILE_FOLDER/result.txt")"
+    echo "$bucket_info"
+    return 0
+  elif [ "$result" == "404" ]; then
+    return 1
+  fi
+  log 2 "unexpected response code '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+  return 2
+}

tests/commands/list_objects.sh

@@ -138,35 +138,19 @@ list_objects_rest() {
     log 2 "'list_objects_rest' requires bucket name"
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  log 5 "canonical request: $canonical_request"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "bucket name: $1"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error listing objects: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))"
+    return 1
+  fi
+  # shellcheck disable=SC2034
+  reply=$(cat "$TEST_FILE_FOLDER/objects.txt")
+  if ! parse_objects_list_rest; then
+    log 2 "error parsing list objects"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" 2>&1)
-  log 5 "reply: $reply"
-  parse_objects_list_rest
 }

tests/commands/list_parts.sh

@@ -19,11 +19,11 @@ list_parts() {
     log 2 "'list-parts' command requires bucket, key, upload ID"
     return 1
   fi
-  record_command "list-parts" "client:s3api"
-  if ! listed_parts=$(send_command aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
-    log 2 "Error listing multipart upload parts: $listed_parts"
+  if ! list_parts_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3"; then
+    log 2 "error listing parts with user"
     return 1
   fi
+  return 0
 }
 
 list_parts_with_user() {
@@ -36,4 +36,7 @@ list_parts_with_user() {
     log 2 "Error listing multipart upload parts: $listed_parts"
     return 1
   fi
+  listed_parts="$(echo -n "$listed_parts" | grep -v "InsecureRequestWarning")"
+  log 5 "listed parts: $listed_parts"
+  return 0
 }

tests/commands/put_bucket_acl.sh

@@ -24,7 +24,7 @@ put_bucket_acl_s3api() {
     log 2 "put bucket acl command requires bucket name, acl file"
     return 1
   fi
-  log 5 "bucket name: $1, acls: $2"
+  log 5 "bucket name: $1, acls: $(cat "$2")"
   if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
     log 2 "error putting bucket acl: $error"
     return 1

tests/commands/put_bucket_policy.sh

@@ -37,6 +37,10 @@ put_bucket_policy() {
     export put_bucket_policy_error
     return 1
   fi
+  # direct can take some time to take effect
+  if [ "$DIRECT" == "true" ]; then
+    sleep 10
+  fi
   return 0
 }
 

tests/commands/put_object.sh

@@ -74,12 +74,69 @@ put_object_rest() {
     log 2 "'put_object_rest' requires local file, bucket name, key"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
-    log 2 "error sending object file: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "expected response code of '200', was '$result' (output: $(cat "$TEST_FILE_FOLDER/result.txt")"
+  if ! put_object_rest_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3"; then
+    log 2 "error putting object with REST with root user"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "'put_object_rest_with_user' requires username, password, local file, bucket name, key"
+    return 1
+  fi
+  if ! put_object_rest_with_user_and_code "$1" "$2" "$3" "$4" "$5" "200"; then
+    log 2 "error putting object with user '$1'"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_with_user_and_code() {
+  if [ $# -ne 6 ]; then
+    log 2 "'put_object_rest_with_user' requires username, password, local file, bucket name, key, expected response code"
+    return 1
+  fi
+  if ! result=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$3" BUCKET_NAME="$4" OBJECT_KEY="$5" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
+    log 2 "error sending object file: $result"
+    return 1
+  fi
+  if [ "$result" != "$6" ]; then
+    log 2 "expected response code of '$6', was '$result' (output: $(cat "$TEST_FILE_FOLDER/result.txt")"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_with_user_code_error() {
+  if [ $# -ne 7 ]; then
+    log 2 "'put_object_rest_with_user_code_error' requires username, password, lcoal file, bucket name, key, expected code, expected error"
+    return 1
+  fi
+  if ! result=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$3" BUCKET_NAME="$4" OBJECT_KEY="$5" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
+    log 2 "error sending object file: $result"
+    return 1
+  fi
+  if [ "$result" != "$6" ]; then
+    log 2 "expected response code of '$6', was '$result' (output: $(cat "$TEST_FILE_FOLDER/result.txt")"
+    return 1
+  fi
+  if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$7" "Error" "Code"; then
+    log 2 "error checking for error code '$7'"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_user_bad_signature() {
+  if [ $# -ne 5 ]; then
+    log 2 "'put_object_rest_user_bad_signature' requires username, password, local file, bucket name, key"
+    return 1
+  fi
+  export SIGNATURE="abcdefg"
+  if ! put_object_rest_with_user_code_error "$1" "$2" "$3" "$4" "$5" "403" "SignatureDoesNotMatch"; then
+    log 2 "error checking REST user bad signature error"
     return 1
   fi
   return 0

tests/commands/put_object_legal_hold.sh

@@ -20,11 +20,23 @@ put_object_legal_hold() {
     log 2 "'put object legal hold' command requires bucket, key, hold status ('ON' or 'OFF')"
     return 1
   fi
-  local error=""
-  error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1) || local put_hold_result=$?
-  if [[ $put_hold_result -ne 0 ]]; then
+  if ! error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1); then
     log 2 "error putting object legal hold: $error"
     return 1
   fi
   return 0
-}
+}
+
+put_object_legal_hold_version_id() {
+  record_command "put-object-legal-hold" "client:s3api"
+  if [[ $# -ne 4 ]]; then
+    log 2 "'put_object_legal_hold_version_id' command requires bucket, key, version ID, hold status ('ON' or 'OFF')"
+    return 1
+  fi
+  local error=""
+  if ! error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --version-id "$3" --legal-hold "{\"Status\": \"$4\"}" 2>&1); then
+    log 2 "error putting object legal hold w/version ID: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/upload_part.sh

@@ -19,9 +19,21 @@ upload_part() {
     log 2 "upload multipart part function must have bucket, key, upload ID, file name, part number"
     return 1
   fi
+  if ! upload_part_with_user "$1" "$2" "$3" "$4" "$5" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then
+    log 2 "error uploading part with user"
+    return 1
+  fi
+  return 0
+}
+
+upload_part_with_user() {
+  if [ $# -ne 7 ]; then
+    log 2 "upload multipart part function must have bucket, key, upload ID, file name, part number, username, password"
+    return 1
+  fi
   local etag_json
   record_command "upload-part" "client:s3api"
-  if ! etag_json=$(send_command aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then
+  if ! etag_json=$(AWS_ACCESS_KEY_ID="$6" AWS_SECRET_ACCESS_KEY="$7" send_command aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then
     log 2 "Error uploading part $5: $etag_json"
     return 1
   fi

tests/env.sh

@@ -58,29 +58,7 @@ source_config_file() {
   fi
 }
 
-check_universal_vars() {
-  if [[ $BYPASS_ENV_FILE != "true" ]]; then
-    source_config_file
-  fi
-  if [ -n "$COMMAND_LOG" ]; then
-    init_command_log
-  fi
-
-  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
-    # shellcheck source=./tests/.secrets
-    source "$SECRETS_FILE"
-  else
-    log 3 "Warning: no secrets file found"
-  fi
-
-  if [[ -n "$LOG_LEVEL" ]]; then
-    if [[ $LOG_LEVEL -lt 2 ]]; then
-      log 1 "log level must be 2 or greater"
-      exit 1
-    fi
-    export LOG_LEVEL_INT=$LOG_LEVEL
-  fi
-
+check_aws_vars() {
   if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     log 1 "AWS_ACCESS_KEY_ID missing"
     exit 1
@@ -103,10 +81,14 @@ check_universal_vars() {
       exit 1
     fi
   fi
-  if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
-    fail "RUN_VERSITYGW must be 'true' or 'false'"
+  # exporting these since they're needed for subshells
+  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
+  if [ -n "$AWS_CANONICAL_ID" ]; then
+    export AWS_CANONICAL_ID
   fi
+}
 
+check_bucket_vars() {
   if [ -z "$BUCKET_ONE_NAME" ]; then
     log 1 "BUCKET_ONE_NAME missing"
     exit 1
@@ -115,38 +97,60 @@ check_universal_vars() {
     log 1 "BUCKET_TWO_NAME missing"
     exit 1
   fi
-  if [ -z "$RECREATE_BUCKETS" ]; then
-    log 1 "RECREATE_BUCKETS missing"
-    exit 1
-  fi
   if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then
     log 1 "RECREATE_BUCKETS must be 'true' or 'false'"
     exit 1
   fi
-  if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then
-    log 1 "RECREATE_BUCKETS must be 'true' or 'false'"
+  if [ "$DELETE_BUCKETS_AFTER_TEST" != "true" ] && [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then
+    log 1 "DELETE_BUCKETS_AFTER_TEST must be 'true' or 'false'"
     exit 1
   fi
   if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then
     log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'"
-    return 1
+    exit 1
   fi
+}
+
+check_universal_vars() {
+  if [[ $BYPASS_ENV_FILE != "true" ]]; then
+    source_config_file
+  fi
+  if [ -n "$COMMAND_LOG" ]; then
+    init_command_log
+  fi
+  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
+    # shellcheck source=./tests/.secrets
+    source "$SECRETS_FILE"
+  else
+    log 3 "Warning: no secrets file found"
+  fi
+  if [[ -n "$LOG_LEVEL" ]]; then
+    if [[ $LOG_LEVEL -lt 2 ]]; then
+      log 1 "log level must be 2 or greater"
+      exit 1
+    fi
+    export LOG_LEVEL_INT=$LOG_LEVEL
+  fi
+
+  check_aws_vars
+
+  if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
+    log 1 "RUN_VERSITYGW must be 'true' or 'false'"
+    exit 1
+  fi
+
+  check_bucket_vars
+
   if [ -z "$TEST_FILE_FOLDER" ]; then
     log 1 "TEST_FILE_FOLDER missing"
     exit 1
   fi
   if [ ! -d "$TEST_FILE_FOLDER" ]; then
     if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then
-      log 2 "error creating test folder: $error"
+      log 1 "error creating test folder: $error"
       exit 1
     fi
   fi
-  # exporting these since they're needed for subshells
-  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
-  if [ -n "$AWS_CANONICAL_ID" ]; then
-    log 5 "canonical ID: $AWS_CANONICAL_ID"
-    export AWS_CANONICAL_ID
-  fi
 }
 
 delete_command_log() {
@@ -230,6 +234,18 @@ check_user_vars() {
     log 1 "PASSWORD_TWO missing"
     exit 1
   fi
+  if [ "$AUTOGENERATE_USERS" != "true" ] && [ "$AUTOGENERATE_USERS" != "false" ]; then
+    log 1 "AUTOGENERATE_USERS must be 'true' or 'false'"
+    return 1
+  fi
+  if [ "$AUTOGENERATE_USERS" == "true" ] && [ "$USER_AUTOGENERATION_PREFIX" == "" ]; then
+    log 1 "USER_AUTOGENERATION_PREFIX is required if AUTOGENERATE_USERS is 'true'"
+    return 1
+  fi
+  if [ "$AUTOGENERATE_USERS" == "false" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" != "false" ]; then
+    log 1 "If AUTOGENERATE_USERS is 'false', 'CREATE_STATIC_USERS_IF_NONEXISTENT' must be true or false"
+    return 1
+  fi
 
   if [[ -z "$IAM_TYPE" ]]; then
     export IAM_TYPE="folder"

tests/integration/utils.go

@@ -18,13 +18,19 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
+	"crypto/sha1"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"hash"
+	"hash/crc32"
+	"hash/crc64"
 	"io"
 	"math/big"
+	"math/bits"
 	rnd "math/rand"
 	"net/http"
 	"net/url"
@@ -362,10 +368,9 @@ func putObjects(client *s3.Client, objs []string, bucket string) ([]types.Object
 			return nil, err
 		}
 		k := key
-		etag := strings.Trim(*res.ETag, `"`)
 		contents = append(contents, types.Object{
 			Key:          &k,
-			ETag:         &etag,
+			ETag:         res.ETag,
 			StorageClass: types.ObjectStorageClassStandard,
 			Size:         &size,
 		})
@@ -475,11 +480,31 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client)
 	}, nil
 }
 
-func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
+type mpCfg struct {
+	checksumAlgorithm types.ChecksumAlgorithm
+	checksumType      types.ChecksumType
+}
+
+type mpOpt func(*mpCfg)
+
+func withChecksum(algo types.ChecksumAlgorithm) mpOpt {
+	return func(mc *mpCfg) { mc.checksumAlgorithm = algo }
+}
+func withChecksumType(t types.ChecksumType) mpOpt {
+	return func(mc *mpCfg) { mc.checksumType = t }
+}
+
+func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) {
+	cfg := new(mpCfg)
+	for _, opt := range opts {
+		opt(cfg)
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
 	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
-		Bucket: &bucket,
-		Key:    &key,
+		Bucket:            &bucket,
+		Key:               &key,
+		ChecksumAlgorithm: cfg.checksumAlgorithm,
+		ChecksumType:      cfg.checksumType,
 	})
 	cancel()
 	return out, err
@@ -513,6 +538,12 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
 		if item.StorageClass != list2[i].StorageClass {
 			return false
 		}
+		if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm {
+			return false
+		}
+		if item.ChecksumType != list2[i].ChecksumType {
+			return false
+		}
 	}
 
 	return true
@@ -520,16 +551,53 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
 
 func compareParts(parts1, parts2 []types.Part) bool {
 	if len(parts1) != len(parts2) {
+		fmt.Printf("list length are not equal: %v != %v\n", len(parts1), len(parts2))
 		return false
 	}
 
 	for i, prt := range parts1 {
 		if *prt.PartNumber != *parts2[i].PartNumber {
+			fmt.Printf("partNumbers are not equal, %v != %v\n", *prt.PartNumber, *parts2[i].PartNumber)
 			return false
 		}
 		if *prt.ETag != *parts2[i].ETag {
+			fmt.Printf("etags are not equal, %v != %v\n", *prt.ETag, *parts2[i].ETag)
 			return false
 		}
+		if *prt.Size != *parts2[i].Size {
+			fmt.Printf("sizes are not equal, %v != %v\n", *prt.Size, *parts2[i].Size)
+			return false
+		}
+		if prt.ChecksumCRC32 != nil {
+			if *prt.ChecksumCRC32 != getString(parts2[i].ChecksumCRC32) {
+				fmt.Printf("crc32 checksums are not equal, %v != %v\n", *prt.ChecksumCRC32, getString(parts2[i].ChecksumCRC32))
+				return false
+			}
+		}
+		if prt.ChecksumCRC32C != nil {
+			if *prt.ChecksumCRC32C != getString(parts2[i].ChecksumCRC32C) {
+				fmt.Printf("crc32c checksums are not equal, %v != %v\n", *prt.ChecksumCRC32C, getString(parts2[i].ChecksumCRC32C))
+				return false
+			}
+		}
+		if prt.ChecksumSHA1 != nil {
+			if *prt.ChecksumSHA1 != getString(parts2[i].ChecksumSHA1) {
+				fmt.Printf("sha1 checksums are not equal, %v != %v\n", *prt.ChecksumSHA1, getString(parts2[i].ChecksumSHA1))
+				return false
+			}
+		}
+		if prt.ChecksumSHA256 != nil {
+			if *prt.ChecksumSHA256 != getString(parts2[i].ChecksumSHA256) {
+				fmt.Printf("sha256 checksums are not equal, %v != %v\n", *prt.ChecksumSHA256, getString(parts2[i].ChecksumSHA256))
+				return false
+			}
+		}
+		if prt.ChecksumCRC64NVME != nil {
+			if *prt.ChecksumCRC64NVME != getString(parts2[i].ChecksumCRC64NVME) {
+				fmt.Printf("crc64nvme checksums are not equal, %v != %v\n", *prt.ChecksumCRC64NVME, getString(parts2[i].ChecksumCRC64NVME))
+				return false
+			}
+		}
 	}
 	return true
 }
@@ -647,6 +715,26 @@ func compareObjects(list1, list2 []types.Object) bool {
 				*obj.Key, *list2[i].Key, obj.StorageClass, list2[i].StorageClass)
 			return false
 		}
+		if len(obj.ChecksumAlgorithm) != 0 {
+			if obj.ChecksumAlgorithm[0] != list2[i].ChecksumAlgorithm[0] {
+				fmt.Printf("checksum algorithms are not equal: (%q %q) %v != %v\n",
+					*obj.Key, *list2[i].Key, obj.ChecksumAlgorithm[0], list2[i].ChecksumAlgorithm[0])
+				return false
+			}
+		}
+		if obj.ChecksumType != "" {
+			if obj.ChecksumType[0] != list2[i].ChecksumType[0] {
+				fmt.Printf("checksum types are not equal: (%q %q) %v != %v\n",
+					*obj.Key, *list2[i].Key, obj.ChecksumType, list2[i].ChecksumType)
+				return false
+			}
+		}
+		if obj.Owner != nil {
+			if *obj.Owner.ID != *list2[i].Owner.ID {
+				fmt.Printf("object owner IDs not equal: (%q %q) %v != %v\n",
+					*obj.Key, *list2[i].Key, *obj.Owner.ID, *list2[i].Owner.ID)
+			}
+		}
 	}
 
 	return true
@@ -711,10 +799,30 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool {
 	return true
 }
 
-func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string) (parts []types.Part, csum string, err error) {
+func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) {
 	partSize := size / partCount
 
-	hash := sha256.New()
+	var hash hash.Hash
+
+	cfg := new(mpCfg)
+	for _, opt := range opts {
+		opt(cfg)
+	}
+
+	switch cfg.checksumAlgorithm {
+	case types.ChecksumAlgorithmCrc32:
+		hash = crc32.NewIEEE()
+	case types.ChecksumAlgorithmCrc32c:
+		hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case types.ChecksumAlgorithmSha1:
+		hash = sha1.New()
+	case types.ChecksumAlgorithmSha256:
+		hash = sha256.New()
+	case types.ChecksumAlgorithmCrc64nvme:
+		hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
+	default:
+		hash = sha256.New()
+	}
 
 	for partNumber := int64(1); partNumber <= partCount; partNumber++ {
 		partStart := (partNumber - 1) * partSize
@@ -730,26 +838,46 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
 		pn := int32(partNumber)
 		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
-			Bucket:     &bucket,
-			Key:        &key,
-			UploadId:   &uploadId,
-			Body:       bytes.NewReader(partBuffer),
-			PartNumber: &pn,
+			Bucket:            &bucket,
+			Key:               &key,
+			UploadId:          &uploadId,
+			Body:              bytes.NewReader(partBuffer),
+			PartNumber:        &pn,
+			ChecksumAlgorithm: cfg.checksumAlgorithm,
 		})
 		cancel()
 		if err != nil {
 			return parts, "", err
 		}
 
-		parts = append(parts, types.Part{
+		part := types.Part{
 			ETag:       out.ETag,
 			PartNumber: &pn,
 			Size:       &partSize,
-		})
-	}
+		}
 
+		switch cfg.checksumAlgorithm {
+		case types.ChecksumAlgorithmCrc32:
+			part.ChecksumCRC32 = out.ChecksumCRC32
+		case types.ChecksumAlgorithmCrc32c:
+			part.ChecksumCRC32C = out.ChecksumCRC32C
+		case types.ChecksumAlgorithmSha1:
+			part.ChecksumSHA1 = out.ChecksumSHA1
+		case types.ChecksumAlgorithmSha256:
+			part.ChecksumSHA256 = out.ChecksumSHA256
+		case types.ChecksumAlgorithmCrc64nvme:
+			part.ChecksumCRC64NVME = out.ChecksumCRC64NVME
+		}
+
+		parts = append(parts, part)
+	}
 	sum := hash.Sum(nil)
-	csum = hex.EncodeToString(sum[:])
+
+	if cfg.checksumAlgorithm == "" {
+		csum = hex.EncodeToString(sum[:])
+	} else {
+		csum = base64.StdEncoding.EncodeToString(sum[:])
+	}
 
 	return parts, csum, err
 }
@@ -851,8 +979,7 @@ func changeAuthCred(uri, newVal string, index int) (string, error) {
 }
 
 func genPolicyDoc(effect, principal, action, resource string) string {
-	jsonTemplate := `
-	{
+	jsonTemplate := `{
 		"Statement": [
 			{
 				"Effect":  "%s",
@@ -989,7 +1116,22 @@ func pfxStrings(pfxs []types.CommonPrefix) []string {
 	return pfxStrs
 }
 
-func createObjVersions(client *s3.Client, bucket, object string, count int) ([]types.ObjectVersion, error) {
+type versCfg struct {
+	checksumAlgorithm types.ChecksumAlgorithm
+}
+
+type versOpt func(*versCfg)
+
+func withChecksumAlgo(algo types.ChecksumAlgorithm) versOpt {
+	return func(vc *versCfg) { vc.checksumAlgorithm = algo }
+}
+
+func createObjVersions(client *s3.Client, bucket, object string, count int, opts ...versOpt) ([]types.ObjectVersion, error) {
+	cfg := new(versCfg)
+	for _, o := range opts {
+		o(cfg)
+	}
+
 	versions := []types.ObjectVersion{}
 	for i := 0; i < count; i++ {
 		rNumber, err := rand.Int(rand.Reader, big.NewInt(100000))
@@ -1007,15 +1149,40 @@ func createObjVersions(client *s3.Client, bucket, object string, count int) ([]t
 		}
 
 		isLatest := i == count-1
-
-		versions = append(versions, types.ObjectVersion{
+		version := types.ObjectVersion{
 			ETag:         r.res.ETag,
 			IsLatest:     &isLatest,
 			Key:          &object,
 			Size:         &dataLength,
 			VersionId:    r.res.VersionId,
 			StorageClass: types.ObjectVersionStorageClassStandard,
-		})
+			ChecksumType: r.res.ChecksumType,
+		}
+
+		switch {
+		case r.res.ChecksumCRC32 != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmCrc32,
+			}
+		case r.res.ChecksumCRC32C != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmCrc32c,
+			}
+		case r.res.ChecksumCRC64NVME != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmCrc64nvme,
+			}
+		case r.res.ChecksumSHA1 != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmSha1,
+			}
+		case r.res.ChecksumSHA256 != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmSha256,
+			}
+		}
+
+		versions = append(versions, version)
 	}
 
 	versions = reverseSlice(versions)
@@ -1075,6 +1242,19 @@ func compareVersions(v1, v2 []types.ObjectVersion) bool {
 		if version.StorageClass != v2[i].StorageClass {
 			return false
 		}
+		if version.ChecksumType != "" {
+			if version.ChecksumType != v2[i].ChecksumType {
+				return false
+			}
+		}
+		if len(version.ChecksumAlgorithm) != 0 {
+			if len(v2[i].ChecksumAlgorithm) == 0 {
+				return false
+			}
+			if version.ChecksumAlgorithm[0] != v2[i].ChecksumAlgorithm[0] {
+				return false
+			}
+		}
 	}
 
 	return true
@@ -1111,6 +1291,64 @@ func compareDelMarkers(d1, d2 []types.DeleteMarkerEntry) bool {
 	return true
 }
 
+type ObjectMetaProps struct {
+	ContentLength      int64
+	ContentType        string
+	ContentEncoding    string
+	ContentDisposition string
+	ContentLanguage    string
+	CacheControl       string
+	ExpiresString      string
+	Metadata           map[string]string
+}
+
+func checkObjectMetaProps(client *s3.Client, bucket, object string, o ObjectMetaProps) error {
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	out, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	cancel()
+	if err != nil {
+		return err
+	}
+
+	if o.Metadata != nil {
+		if !areMapsSame(out.Metadata, o.Metadata) {
+			return fmt.Errorf("expected the object metadata to be %v, instead got %v", o.Metadata, out.Metadata)
+		}
+	}
+	if out.ContentLength == nil {
+		return fmt.Errorf("expected Content-Length %v, instead got nil", o.ContentLength)
+	}
+	if *out.ContentLength != o.ContentLength {
+		return fmt.Errorf("expected Content-Length %v, instead got %v", o.ContentLength, *out.ContentLength)
+	}
+	if o.ContentType != "" && getString(out.ContentType) != o.ContentType {
+		return fmt.Errorf("expected Content-Type %v, instead got %v", o.ContentType, getString(out.ContentType))
+	}
+	if o.ContentDisposition != "" && getString(out.ContentDisposition) != o.ContentDisposition {
+		return fmt.Errorf("expected Content-Disposition %v, instead got %v", o.ContentDisposition, getString(out.ContentDisposition))
+	}
+	if o.ContentEncoding != "" && getString(out.ContentEncoding) != o.ContentEncoding {
+		return fmt.Errorf("expected Content-Encoding %v, instead got %v", o.ContentEncoding, getString(out.ContentEncoding))
+	}
+	if o.ContentLanguage != "" && getString(out.ContentLanguage) != o.ContentLanguage {
+		return fmt.Errorf("expected Content-Language %v, instead got %v", o.ContentLanguage, getString(out.ContentLanguage))
+	}
+	if o.CacheControl != "" && getString(out.CacheControl) != o.CacheControl {
+		return fmt.Errorf("expected Cache-Control %v, instead got %v", o.CacheControl, getString(out.CacheControl))
+	}
+	if o.ExpiresString != "" && getString(out.ExpiresString) != o.ExpiresString {
+		return fmt.Errorf("expected Expires %v, instead got %v", o.ExpiresString, getString(out.ExpiresString))
+	}
+	if out.StorageClass != types.StorageClassStandard {
+		return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass)
+	}
+
+	return nil
+}
+
 func getBoolPtr(b bool) *bool {
 	return &b
 }

tests/logger.sh

@@ -16,15 +16,13 @@
 
 # levels:  1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace
 
-export LOG_LEVEL_INT=4
-
 log() {
   if [[ $# -ne 2 ]]; then
     echo "log function requires level, message"
     return 1
   fi
   # shellcheck disable=SC2153
-  if [[ $1 -gt $LOG_LEVEL_INT ]]; then
+  if [[ $1 -gt ${LOG_LEVEL_INT:=4} ]]; then
     return 0
   fi
   log_level=""
@@ -148,7 +146,7 @@ log_message() {
     echo "$now $1 $2" >&2
   fi
   if [[ -n "$TEST_LOG_FILE" ]]; then
-    echo "$now $1 $2" >> "$TEST_LOG_FILE.tmp"
+    echo "$now ${BASH_SOURCE[2]}:${BASH_LINENO[1]} $1 $2" >> "$TEST_LOG_FILE.tmp"
   fi
   sync
 }

tests/rest_scripts/abort_multipart_upload.sh

@@ -37,6 +37,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id"

tests/rest_scripts/calculate_checksum.py

@@ -0,0 +1,21 @@
+import sys
+from awscrt import checksums
+
+def get_call(checksum_type):
+    if checksum_type == "crc32c":
+        return checksums.crc32c
+    elif checksum_type == "crc64nvme":
+        return checksums.crc64nvme
+    sys.stderr.write("unrecognized checksum type " + checksum_type)
+    sys.exit(1)
+
+def main():
+    if len(sys.argv) != 3:
+        sys.stderr.write('Checksum type, data file path required')
+        sys.exit(1)
+    with open(sys.argv[2], 'rb') as f:
+        function = get_call(sys.argv[1])
+        print(function(f.read()))
+
+if __name__ == "__main__":
+    main()

tests/rest_scripts/calculate_checksum.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+calculate_checksum_python() {
+  if [ "$#" -ne 2 ]; then
+    log 2 "'calculate_checksum_python' requires checksum type, data file"
+    return 1
+  fi
+  if ! DEACTIVATE=false source ./tests/rest_scripts/init_python_env.sh; then
+    log_rest 2 "error initializing python environment"
+    return 1
+  fi
+  if ! checksum_decimal=$(python3 ./tests/rest_scripts/calculate_checksum.py "$1" "$2" 2>&1); then
+    log_rest 2 "error calculating checksum: $checksum_decimal"
+    return 1
+  fi
+  log 5 "decimal checksum: $checksum_decimal"
+  if ! deactivate 1>/dev/null; then
+    log_rest 2 "error deactivating virtual environment"
+    return 1
+  fi
+  if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then
+    hex_format="%016x"
+  else
+    hex_format="%08x"
+  fi
+  # shellcheck disable=SC2059
+  checksum=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64)
+  echo "$checksum"
+}
+
+case "$CHECKSUM_TYPE" in
+"crc32c")
+  if ! checksum=$(calculate_checksum_python "crc32c" "$DATA_FILE" 2>&1); then
+    log_rest 2 "error getting checksum: $checksum"
+    exit 1
+  fi
+  ;;
+"crc64nvme")
+  if ! checksum=$(calculate_checksum_python "crc64nvme" "$DATA_FILE" 2>&1); then
+    log 2 "error calculating checksum: $checksum"
+    exit 1
+  fi
+  ;;
+"sha256")
+  checksum="$(sha256sum "$DATA_FILE" | awk '{print $1}' | xxd -r -p | base64)"
+  ;;
+"sha1")
+  checksum="$(sha1sum "$DATA_FILE" | awk '{print $1}' | xxd -r -p | base64)"
+  ;;
+"crc32")
+  checksum="$(gzip -c -1 "$DATA_FILE" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
+  ;;
+*)
+  log_rest 2 "invalid checksum type: '$CHECKSUM_TYPE'"
+  exit 1
+  ;;
+esac
+echo "$checksum"

tests/rest_scripts/complete_multipart_upload.sh

@@ -41,6 +41,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id"

tests/rest_scripts/copy_object.sh

@@ -38,6 +38,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-copy-source;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key")

tests/rest_scripts/create_multipart_upload.sh

@@ -39,6 +39,7 @@ UNSIGNED-PAYLOAD"
 
 canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploads="

tests/rest_scripts/delete_object_tagging.sh

@@ -35,6 +35,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?tagging"

tests/rest_scripts/delete_objects.sh

@@ -49,6 +49,7 @@ fi
 canonical_request+="host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name?delete")

tests/rest_scripts/get_bucket_acl.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?acl="

tests/rest_scripts/get_bucket_ownership_controls.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?ownershipControls="

tests/rest_scripts/get_bucket_policy.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?policy="

tests/rest_scripts/get_bucket_tagging.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?tagging="

tests/rest_scripts/get_bucket_versioning.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?versioning="

tests/rest_scripts/get_object.sh

@@ -22,26 +22,26 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2154
 key="$OBJECT_KEY"
+# shellcheck disable=SC2154
+checksum_mode="${CHECKSUM_MODE:=false}"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
 
 #x-amz-object-attributes:ETag
-canonical_request="GET
-/$bucket_name/$key
+canonical_request_data+=("GET" "/$bucket_name/$key" "" "host:$host")
+if [ "$checksum_mode" == "true" ]; then
+  canonical_request_data+=("x-amz-checksum-mode:ENABLED")
+fi
+canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
 
-host:$host
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
+build_canonical_request "${canonical_request_data[@]}"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
--H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
--H "\"x-amz-date: $current_date_time\""
--o "$OUTPUT_FILE")
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
 # shellcheck disable=SC2154
 eval "${curl_command[*]}" 2>&1

tests/rest_scripts/get_object_attributes.sh

@@ -41,6 +41,7 @@ x-amz-object-attributes:$attributes
 host;x-amz-content-sha256;x-amz-date;x-amz-object-attributes
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key?attributes="

tests/rest_scripts/get_object_legal_hold.sh

@@ -35,6 +35,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key?legal-hold="

tests/rest_scripts/get_object_lock_config.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fields
+
+source ./tests/rest_scripts/rest.sh
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+canonical_request_data=("GET" "/$bucket_name" "object-lock=" "host:$host")
+canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi
+
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+# shellcheck disable=SC2154
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?object-lock")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$year_month_day/$AWS_REGION/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/get_public_access_block.sh

@@ -33,6 +33,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="

tests/rest_scripts/head_bucket.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+# shellcheck disable=SC2034
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request_data=("HEAD" "/$bucket_name" "" "host:$host")
+canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name"
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/head_object.sh

@@ -22,26 +22,36 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2154
 key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+version_id="$VERSION_ID"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
 
-#x-amz-object-attributes:ETag
-canonical_request="HEAD
-/$bucket_name/$key
-
-host:$host
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
+canonical_request_data=("HEAD" "/$bucket_name/$key")
+if [ "$version_id" != "" ]; then
+  canonical_request_data+=("versionId=$version_id")
+else
+  canonical_request_data+=("")
+fi
+canonical_request_data+=("host:$host")
+if [ "$CHECKSUM" == "true" ]; then
+  canonical_request_data+=("x-amz-checksum-mode:ENABLED")
+fi
+canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
--H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
--H "\"x-amz-date: $current_date_time\""
--o "$OUTPUT_FILE")
+url="$AWS_ENDPOINT_URL/$bucket_name/$key"
+if [ "$version_id" != "" ]; then
+  url+="?versionId=$version_id"
+fi
+curl_command+=(curl -ksI -w "\"%{http_code}\"" "$url"
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
 # shellcheck disable=SC2154
 eval "${curl_command[*]}" 2>&1

tests/rest_scripts/init_python_env.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+deactivate=${DEACTIVATE:=true}
+
+if [ -z "$PYTHON_ENV_FOLDER" ] && [ -z "$TEST_FILE_FOLDER" ]; then
+  log_rest 2 "python virtual env setup requires either PYTHON_ENV_FOLDER or TEST_FILE_FOLDER param"
+  exit 1
+fi
+python_env_folder=${PYTHON_ENV_FOLDER:=${TEST_FILE_FOLDER}/env}
+if [ ! -d "$python_env_folder" ] && ! python3 -m venv "$python_env_folder"; then
+  log_rest 2 "error creating python virtual environment"
+  exit 1
+fi
+if ! source "$python_env_folder/bin/activate"; then
+  log_rest 2 "error activating virtual environment"
+  exit 1
+fi
+if ! python3 -m pip list | grep awscrt 1>/dev/null; then
+  if ! python3 -m pip install awscrt 1>/dev/null; then
+    log_rest 2 "error installing awscrt"
+    exit 1
+  fi
+fi
+if [ "$deactivate" == "true" ] && ! deactivate; then
+  log_rest 2 "error deactivating"
+  exit 1
+fi

tests/rest_scripts/list_buckets.sh

@@ -28,6 +28,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host"

tests/rest_scripts/list_multipart_uploads.sh

@@ -36,6 +36,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name/$key?uploads="

tests/rest_scripts/list_object_versions.sh

@@ -34,6 +34,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name?versions"