Merge pull request #1121 from versity/test/rest_checksum_nomode

Test/rest checksum nomode

.github/workflows/betteralign.yml

@@ -1,23 +0,0 @@
-name: betteralign
-on: pull_request
-jobs:
-  build:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: "stable"
-        id: go
-
-      - name: Install betteralign
-        run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest
-
-      - name: Run betteralign
-        run: betteralign -test_files ./...

.github/workflows/docker-bats.yml

@@ -14,6 +14,7 @@ jobs:
         run: |
           cp tests/.env.docker.default tests/.env.docker
           cp tests/.secrets.default tests/.secrets
+          # see https://github.com/versity/versitygw/issues/1034
           docker build \
             --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
             --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \

.github/workflows/system.yml

@@ -73,16 +73,17 @@ jobs:
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
             BACKEND: "posix"
-          - set: "s3api, s3, multipart|object, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
-          - set: "s3api, s3, policy|user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy,s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
+          # TODO fix/debug s3 gateway
+          #- set: "s3api, s3, multipart|object, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
+          #- set: "s3api, s3, policy|user, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-policy,s3api-user"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
           - set: "s3cmd, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
             RUN_SET: "s3cmd-file-count"
@@ -132,6 +133,14 @@ jobs:
         run: |
           sudo apt-get install libxml2-utils
 
+      # see https://github.com/versity/versitygw/issues/1034
+      - name: Install AWS cli
+        run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" 
+          unzip -o awscliv2.zip 
+          ./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
       - name: Build and run
         env:
           IAM_TYPE: ${{ matrix.IAM_TYPE }}
@@ -163,6 +172,7 @@ jobs:
           VERSIONING_DIR: ${{ github.workspace }}/versioning
           COMMAND_LOG: command.log
           TIME_LOG: time.log
+          PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -170,6 +180,7 @@ jobs:
           export AWS_REGION=us-east-1
           export AWS_ACCESS_KEY_ID_TWO=user
           export AWS_SECRET_ACCESS_KEY_TWO=pass
+          export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity

README.md

@@ -40,7 +40,7 @@ Versity Gateway, a simple to use tool for seamless inline translation between AW
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
 
-The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versity’s open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.  
 
 The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
 

auth/acl.go

@@ -17,6 +17,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"strings"
@@ -33,46 +34,151 @@ type ACL struct {
 }
 
 type Grantee struct {
-	Permission types.Permission
+	Permission Permission
 	Access     string
 	Type       types.Type
 }
 
 type GetBucketAclOutput struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
 	Owner             *types.Owner
 	AccessControlList AccessControlList
 }
 
 type PutBucketAclInput struct {
 	Bucket              *string
+	ACL                 types.BucketCannedACL
 	AccessControlPolicy *AccessControlPolicy
 	GrantFullControl    *string
 	GrantRead           *string
 	GrantReadACP        *string
 	GrantWrite          *string
 	GrantWriteACP       *string
-	ACL                 types.BucketCannedACL
 }
 
 type AccessControlPolicy struct {
-	Owner             *types.Owner
 	AccessControlList AccessControlList `xml:"AccessControlList"`
+	Owner             *types.Owner
+}
+
+func (acp *AccessControlPolicy) Validate() error {
+	if !acp.AccessControlList.isValid() {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner can't be nil
+	if acp.Owner == nil {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner ID can't be empty
+	if acp.Owner.ID == nil || *acp.Owner.ID == "" {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	return nil
 }
 
 type AccessControlList struct {
 	Grants []Grant `xml:"Grant"`
 }
 
+// Validates the AccessControlList
+func (acl *AccessControlList) isValid() bool {
+	for _, el := range acl.Grants {
+		if !el.isValid() {
+			return false
+		}
+	}
+
+	return true
+}
+
+type Permission string
+
+const (
+	PermissionFullControl Permission = "FULL_CONTROL"
+	PermissionWrite       Permission = "WRITE"
+	PermissionWriteAcp    Permission = "WRITE_ACP"
+	PermissionRead        Permission = "READ"
+	PermissionReadAcp     Permission = "READ_ACP"
+)
+
+// Check if the permission is valid
+func (p Permission) isValid() bool {
+	return p == PermissionFullControl ||
+		p == PermissionRead ||
+		p == PermissionReadAcp ||
+		p == PermissionWrite ||
+		p == PermissionWriteAcp
+}
+
 type Grant struct {
-	Grantee    *Grt
-	Permission types.Permission
+	Grantee    *Grt       `xml:"Grantee"`
+	Permission Permission `xml:"Permission"`
+}
+
+// Checks if Grant is valid
+func (g *Grant) isValid() bool {
+	return g.Permission.isValid() && g.Grantee.isValid()
 }
 
 type Grt struct {
-	XMLNS  string     `xml:"xmlns:xsi,attr"`
-	XMLXSI types.Type `xml:"xsi:type,attr"`
-	Type   types.Type `xml:"Type"`
-	ID     string     `xml:"ID"`
+	XMLNS string     `xml:"xmlns:xsi,attr"`
+	Type  types.Type `xml:"xsi:type,attr"`
+	ID    string     `xml:"ID"`
+}
+
+// Custom Unmarshalling for Grt to parse xsi:type properly
+func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Iterate through the XML tokens to process the attributes
+	for _, attr := range start.Attr {
+		// Check if the attribute is xsi:type and belongs to the xsi namespace
+		if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
+			g.Type = types.Type(attr.Value)
+		}
+		// Handle xmlns:xsi
+		if attr.Name.Local == "xmlns:xsi" {
+			g.XMLNS = attr.Value
+		}
+	}
+
+	// Decode the inner XML elements like ID
+	for {
+		t, err := d.Token()
+		if err != nil {
+			return err
+		}
+
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "ID" {
+				if err := d.DecodeElement(&g.ID, &se); err != nil {
+					return err
+				}
+			}
+		case xml.EndElement:
+			if se.Name.Local == start.Name.Local {
+				return nil
+			}
+		}
+	}
+}
+
+// Validates Grt
+func (g *Grt) isValid() bool {
+	// Validate the Type
+	// Only these 2 types are supported in the gateway
+	if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
+		return false
+	}
+
+	// The ID prop shouldn't be empty
+	if g.ID == "" {
+		return false
+	}
+
+	return true
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -87,22 +193,32 @@ func ParseACL(data []byte) (ACL, error) {
 	return acl, nil
 }
 
-func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
+func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
+	grants := []Grant{}
+
+	if len(data) == 0 {
+		return GetBucketAclOutput{
+			Owner: &types.Owner{
+				ID: &owner,
+			},
+			AccessControlList: AccessControlList{
+				Grants: grants,
+			},
+		}, nil
+	}
+
 	var acl ACL
 	if err := json.Unmarshal(data, &acl); err != nil {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
-	grants := []Grant{}
-
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
 		grants = append(grants, Grant{
 			Grantee: &Grt{
-				XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
-				XMLXSI: elem.Type,
-				ID:     acs,
-				Type:   elem.Type,
+				XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
+				ID:    acs,
+				Type:  elem.Type,
 			},
 			Permission: elem.Permission,
 		})
@@ -125,7 +241,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 
 	defaultGrantees := []Grantee{
 		{
-			Permission: types.PermissionFullControl,
+			Permission: PermissionFullControl,
 			Access:     acl.Owner,
 			Type:       types.TypeCanonicalUser,
 		},
@@ -136,19 +252,19 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 		switch input.ACL {
 		case types.BucketCannedACLPublicRead:
 			defaultGrantees = append(defaultGrantees, Grantee{
-				Permission: types.PermissionRead,
+				Permission: PermissionRead,
 				Access:     "all-users",
 				Type:       types.TypeGroup,
 			})
 		case types.BucketCannedACLPublicReadWrite:
 			defaultGrantees = append(defaultGrantees, []Grantee{
 				{
-					Permission: types.PermissionRead,
+					Permission: PermissionRead,
 					Access:     "all-users",
 					Type:       types.TypeGroup,
 				},
 				{
-					Permission: types.PermissionWrite,
+					Permission: PermissionWrite,
 					Access:     "all-users",
 					Type:       types.TypeGroup,
 				},
@@ -165,7 +281,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range fullControlList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionFullControl,
+						Permission: PermissionFullControl,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -175,7 +291,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range readList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionRead,
+						Permission: PermissionRead,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -185,7 +301,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range readACPList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionReadAcp,
+						Permission: PermissionReadAcp,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -195,7 +311,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range writeList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionWrite,
+						Permission: PermissionWrite,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -205,7 +321,7 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 				for _, str := range writeACPList {
 					defaultGrantees = append(defaultGrantees, Grantee{
 						Access:     str,
-						Permission: types.PermissionWriteAcp,
+						Permission: PermissionWriteAcp,
 						Type:       types.TypeCanonicalUser,
 					})
 				}
@@ -262,8 +378,8 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
 				result = append(result, acc)
 				continue
 			}
-			if err == ErrNotSupported {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
+				return nil, err
 			}
 			return nil, fmt.Errorf("check user account: %w", err)
 		}
@@ -286,7 +402,7 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func verifyACL(acl ACL, access string, permission types.Permission) error {
+func verifyACL(acl ACL, access string, permission Permission) error {
 	grantee := Grantee{
 		Access:     access,
 		Permission: permission,
@@ -294,7 +410,7 @@ func verifyACL(acl ACL, access string, permission types.Permission) error {
 	}
 	granteeFullCtrl := Grantee{
 		Access:     access,
-		Permission: types.PermissionFullControl,
+		Permission: PermissionFullControl,
 		Type:       types.TypeCanonicalUser,
 	}
 	granteeAllUsers := Grantee{
@@ -352,19 +468,19 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 }
 
 type AccessOptions struct {
-	AclPermission types.Permission
+	Acl           ACL
+	AclPermission Permission
+	IsRoot        bool
+	Acc           Account
 	Bucket        string
 	Object        string
 	Action        Action
-	Acl           ACL
-	Acc           Account
-	IsRoot        bool
 	Readonly      bool
 }
 
 func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
 	if opts.Readonly {
-		if opts.AclPermission == types.PermissionWrite || opts.AclPermission == types.PermissionWriteAcp {
+		if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
 		}
 	}
@@ -422,7 +538,7 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource
 
 	if err := VerifyAccess(ctx, be, AccessOptions{
 		Acl:           srcBucketAcl,
-		AclPermission: types.PermissionRead,
+		AclPermission: PermissionRead,
 		IsRoot:        opts.IsRoot,
 		Acc:           opts.Acc,
 		Bucket:        srcBucket,

auth/bucket_policy.go

@@ -48,25 +48,26 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 }
 
 func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	var isAllowed bool
 	for _, statement := range bp.Statement {
 		if statement.findMatch(principal, action, resource) {
 			switch statement.Effect {
 			case BucketPolicyAccessTypeAllow:
-				return true
+				isAllowed = true
 			case BucketPolicyAccessTypeDeny:
 				return false
 			}
 		}
 	}
 
-	return false
+	return isAllowed
 }
 
 type BucketPolicyItem struct {
+	Effect     BucketPolicyAccessType `json:"Effect"`
 	Principals Principals             `json:"Principal"`
 	Actions    Actions                `json:"Action"`
 	Resources  Resources              `json:"Resource"`
-	Effect     BucketPolicyAccessType `json:"Effect"`
 }
 
 func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {

auth/bucket_policy_resources.go

@@ -102,21 +102,45 @@ func (r Resources) Validate(bucket string) error {
 
 func (r Resources) FindMatch(resource string) bool {
 	for res := range r {
-		if strings.HasSuffix(res, "*") {
-			pattern := strings.TrimSuffix(res, "*")
-			if strings.HasPrefix(resource, pattern) {
-				return true
-			}
-		} else {
-			if res == resource {
-				return true
-			}
+		if r.Match(res, resource) {
+			return true
 		}
 	}
 
 	return false
 }
 
+// Match checks if the input string matches the given pattern with wildcards (`*`, `?`).
+// - `?` matches exactly one occurrence of any character.
+// - `*` matches arbitrary many (including zero) occurrences of any character.
+func (r Resources) Match(pattern, input string) bool {
+	pIdx, sIdx := 0, 0
+	starIdx, matchIdx := -1, 0
+
+	for sIdx < len(input) {
+		if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
+			sIdx++
+			pIdx++
+		} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
+			starIdx = pIdx
+			matchIdx = sIdx
+			pIdx++
+		} else if starIdx != -1 {
+			pIdx = starIdx + 1
+			matchIdx++
+			sIdx = matchIdx
+		} else {
+			return false
+		}
+	}
+
+	for pIdx < len(pattern) && pattern[pIdx] == '*' {
+		pIdx++
+	}
+
+	return pIdx == len(pattern)
+}
+
 // Checks the resource to have arn prefix and not starting with /
 func isValidResource(rc string) (isValid bool, pattern string) {
 	if !strings.HasPrefix(rc, ResourceArnPrefix) {

auth/bucket_policy_resources_test.go

@@ -0,0 +1,182 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestUnmarshalJSON(t *testing.T) {
+	var r Resources
+
+	cases := []struct {
+		input    string
+		expected int
+		wantErr  bool
+	}{
+		{`"arn:aws:s3:::my-bucket/*"`, 1, false},
+		{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
+		{`""`, 0, true},
+		{`[]`, 0, true},
+		{`["invalid-bucket"]`, 0, true},
+	}
+
+	for _, tc := range cases {
+		r = Resources{}
+		err := json.Unmarshal([]byte(tc.input), &r)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+		if len(r) != tc.expected {
+			t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
+		}
+	}
+}
+
+func TestAdd(t *testing.T) {
+	r := Resources{}
+
+	cases := []struct {
+		input   string
+		wantErr bool
+	}{
+		{"arn:aws:s3:::valid-bucket/*", false},
+		{"arn:aws:s3:::valid-bucket/object", false},
+		{"invalid-bucket/*", true},
+		{"/invalid-start", true},
+	}
+
+	for _, tc := range cases {
+		err := r.Add(tc.input)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+	}
+}
+
+func TestContainsObjectPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, true},
+		{[]string{"arn:aws:s3:::my-bucket"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsObjectPattern() != tc.expected {
+			t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestContainsBucketPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsBucketPattern() != tc.expected {
+			t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestValidate(t *testing.T) {
+	cases := []struct {
+		resources []string
+		bucket    string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
+		{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
+		{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if (r.Validate(tc.bucket) == nil) != tc.expected {
+			t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
+		}
+	}
+}
+
+func TestFindMatch(t *testing.T) {
+	cases := []struct {
+		resources []string
+		input     string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
+		{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.FindMatch(tc.input) != tc.expected {
+			t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
+		}
+	}
+}
+
+func TestMatch(t *testing.T) {
+	r := Resources{}
+	cases := []struct {
+		pattern  string
+		input    string
+		expected bool
+	}{
+		{"my-bucket/*", "my-bucket/object", true},
+		{"my-bucket/?bject", "my-bucket/object", true},
+		{"my-bucket/*", "other-bucket/object", false},
+		{"*", "any-bucket/object", true},
+		{"my-bucket/*", "my-bucket/subdir/object", true},
+		{"my-bucket/*", "other-bucket", false},
+		{"my-bucket/*/*", "my-bucket/hello", false},
+		{"my-bucket/*/*", "my-bucket/hello/world", true},
+		{"foo/???/bar", "foo/qux/bar", true},
+		{"foo/???/bar", "foo/quxx/bar", false},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
+	}
+	for _, tc := range cases {
+		if r.Match(tc.pattern, tc.input) != tc.expected {
+			t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
+		}
+	}
+}

auth/iam.go

@@ -93,6 +93,7 @@ var (
 )
 
 type Opts struct {
+	RootAccount            Account
 	Dir                    string
 	LDAPServerURL          string
 	LDAPBindDN             string
@@ -118,12 +119,17 @@ type Opts struct {
 	S3Region               string
 	S3Bucket               string
 	S3Endpoint             string
-	RootAccount            Account
-	CacheTTL               int
-	CachePrune             int
 	S3DisableSSlVerfiy     bool
 	S3Debug                bool
 	CacheDisable           bool
+	CacheTTL               int
+	CachePrune             int
+	IpaHost                string
+	IpaVaultName           string
+	IpaUser                string
+	IpaPassword            string
+	IpaInsecure            bool
+	IpaDebug               bool
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -149,6 +155,9 @@ func New(o *Opts) (IAMService, error) {
 			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
 			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
 		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
+	case o.IpaHost != "":
+		svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure, o.IpaDebug)
+		fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")

auth/iam_cache.go

@@ -36,14 +36,14 @@ type IAMCache struct {
 var _ IAMService = &IAMCache{}
 
 type item struct {
-	exp   time.Time
 	value Account
+	exp   time.Time
 }
 
 type icache struct {
-	items  map[string]item
-	expire time.Duration
 	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
 }
 
 func (i *icache) set(k string, v Account) {

auth/iam_internal.go

@@ -33,8 +33,6 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	dir     string
-	rootAcc Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -42,6 +40,8 @@ type IAMServiceInternal struct {
 	// IAM service. All account updates should be sent to a single
 	// gateway instance if possible.
 	sync.RWMutex
+	dir     string
+	rootAcc Account
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored

auth/iam_ipa.go

@@ -0,0 +1,446 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+const IpaVersion = "2.254"
+
+type IpaIAMService struct {
+	client          http.Client
+	id              int
+	version         string
+	host            string
+	vaultName       string
+	username        string
+	password        string
+	kraTransportKey *rsa.PublicKey
+	debug           bool
+	rootAcc         Account
+}
+
+var _ IAMService = &IpaIAMService{}
+
+func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
+
+	ipa := IpaIAMService{
+		id:        0,
+		version:   IpaVersion,
+		host:      host,
+		vaultName: vaultName,
+		username:  username,
+		password:  password,
+		debug:     debug,
+		rootAcc:   rootAcc,
+	}
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		// this should never happen
+		return nil, fmt.Errorf("cookie jar creation: %w", err)
+	}
+
+	mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
+	tr := &http.Transport{
+		TLSClientConfig: mTLSConfig,
+	}
+	ipa.client = http.Client{Jar: jar, Transport: tr}
+
+	err = ipa.login()
+	if err != nil {
+		return nil, fmt.Errorf("ipa login failed: %w", err)
+	}
+
+	req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
+	if err != nil {
+		return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
+	}
+	vaultConfig := struct {
+		Kra_Server_Server             []string
+		Transport_Cert                Base64EncodedWrapped
+		Wrapping_default_algorithm    string
+		Wrapping_supported_algorithms []string
+	}{}
+	err = ipa.rpc(req, &vaultConfig)
+	if err != nil {
+		return nil, fmt.Errorf("ipa vault config: %w", err)
+	}
+
+	cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
+	if err != nil {
+		return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
+	}
+
+	ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
+
+	isSupported := false
+	for _, algo := range vaultConfig.Wrapping_supported_algorithms {
+		if algo == "aes-128-cbc" {
+			isSupported = true
+			break
+		}
+	}
+
+	if !isSupported {
+		return nil,
+			fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
+				vaultConfig.Wrapping_supported_algorithms)
+	}
+	return &ipa, nil
+}
+
+func (ipa *IpaIAMService) CreateAccount(account Account) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ipa.rootAcc.Access {
+		return ipa.rootAcc, nil
+	}
+
+	req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa user_show: %w", err)
+	}
+
+	userResult := struct {
+		Gidnumber []string
+		Uidnumber []string
+	}{}
+
+	err = ipa.rpc(req, &userResult)
+	if err != nil {
+		return Account{}, err
+	}
+
+	uid, err := strconv.Atoi(userResult.Uidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
+	}
+	gid, err := strconv.Atoi(userResult.Gidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
+	}
+
+	account := Account{
+		Access:  access,
+		Role:    RoleUser,
+		UserID:  uid,
+		GroupID: gid,
+	}
+
+	session_key := make([]byte, 16)
+
+	_, err = rand.Read(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot generate session key: %w", err)
+	}
+
+	encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
+	}
+
+	req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
+		map[string]any{"username": access,
+			"session_key":   Base64EncodedWrapped(encryptedKey),
+			"wrapping_algo": "aes-128-cbc"})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
+	}
+
+	data := struct {
+		Vault_data Base64EncodedWrapped
+		Nonce      Base64EncodedWrapped
+	}{}
+
+	err = ipa.rpc(req, &data)
+	if err != nil {
+		return account, err
+	}
+
+	aes, err := aes.NewCipher(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
+	}
+	cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
+	cbc.CryptBlocks(data.Vault_data, data.Vault_data)
+	secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
+	}
+
+	secret := struct {
+		Data Base64Encoded
+	}{}
+	json.Unmarshal(secretUnpaddedJson, &secret)
+	account.Secret = string(secret.Data)
+
+	return account, nil
+}
+
+func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
+	return []Account{}, fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) Shutdown() error {
+	return nil
+}
+
+// Implementation
+
+func (ipa *IpaIAMService) login() error {
+	form := url.Values{}
+	form.Set("user", ipa.username)
+	form.Set("password", ipa.password)
+
+	req, err := http.NewRequest(
+		"POST",
+		fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
+		strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := ipa.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode == 401 {
+		return errors.New("cannot login to FreeIPA: invalid credentials")
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+type rpcRequest = string
+
+type rpcResponse struct {
+	Result    json.RawMessage
+	Principal string
+	Id        int
+	Version   string
+}
+
+func (p rpcResponse) String() string {
+	return string(p.Result)
+}
+
+var errRpc = errors.New("IPA RPC error")
+
+func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
+	err := ipa.login()
+	if err != nil {
+		return err
+	}
+
+	res, err := ipa.rpcInternal(req)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(res.Result, value)
+}
+
+func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
+	httpReq, err := http.NewRequest("POST",
+		fmt.Sprintf("%s/ipa/session/json", ipa.host),
+		strings.NewReader(req))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	ipa.log(fmt.Sprintf("%v", req))
+	httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	httpResp, err := ipa.client.Do(httpReq)
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	bytes, err := io.ReadAll(httpResp.Body)
+	ipa.log(string(bytes))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	result := struct {
+		Result struct {
+			Json    json.RawMessage `json:"result"`
+			Value   string          `json:"value"`
+			Summary any             `json:"summary"`
+		} `json:"result"`
+		Error     json.RawMessage `json:"error"`
+		Id        int             `json:"id"`
+		Principal string          `json:"principal"`
+		Version   string          `json:"version"`
+	}{}
+
+	err = json.Unmarshal(bytes, &result)
+	if err != nil {
+		return rpcResponse{}, err
+	}
+	if string(result.Error) != "null" {
+		return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
+	}
+
+	return rpcResponse{
+		Result:    result.Result.Json,
+		Principal: result.Principal,
+		Id:        result.Id,
+		Version:   result.Version,
+	}, nil
+}
+
+func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
+
+	id := ipa.id
+	ipa.id++
+
+	dict["version"] = ipa.version
+
+	jmethod, errMethod := json.Marshal(method)
+	jargs, errArgs := json.Marshal(args)
+	jdict, errDict := json.Marshal(dict)
+
+	err := errors.Join(errMethod, errArgs, errDict)
+	if err != nil {
+		return "", fmt.Errorf("ipa request invalid: %w", err)
+	}
+
+	request := map[string]interface{}{
+		"id":     id,
+		"method": json.RawMessage(jmethod),
+		"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
+	}
+
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	return string(requestJSON), nil
+}
+
+// pkcs7Unpad validates and unpads data from the given bytes slice.
+// The returned value will be 1 to n bytes smaller depending on the
+// amount of padding, where n is the block size.
+func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
+	if blocksize <= 0 {
+		return nil, errors.New("invalid blocksize")
+	}
+	if len(b) == 0 {
+		return nil, errors.New("invalid PKCS7 data (empty or not padded)")
+	}
+	if len(b)%blocksize != 0 {
+		return nil, errors.New("invalid padding on input")
+	}
+	c := b[len(b)-1]
+	n := int(c)
+	if n == 0 || n > len(b) {
+		return nil, errors.New("invalid padding on input")
+	}
+	for i := 0; i < n; i++ {
+		if b[len(b)-n+i] != c {
+			return nil, errors.New("invalid padding on input")
+		}
+	}
+	return b[:len(b)-n], nil
+}
+
+/*
+e.g.
+
+	"value" {
+		"__base64__": "aGVsbG93b3JsZAo="
+	 }
+*/
+type Base64EncodedWrapped []byte
+
+func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{}
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
+	return err
+}
+
+func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{Base64: base64.StdEncoding.EncodeToString(*b)}
+	return json.Marshal(intermediate)
+}
+
+/*
+e.g.
+
+	"value": "aGVsbG93b3JsZAo="
+*/
+type Base64Encoded []byte
+
+func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
+	var intermediate string
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate)
+	return err
+}
+
+func (ipa *IpaIAMService) log(msg string) {
+	if ipa.debug {
+		log.Println(msg)
+	}
+}

auth/iam_s3_object.go

@@ -42,14 +42,6 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
-	client *s3.Client
-
-	access   string
-	secret   string
-	region   string
-	bucket   string
-	endpoint string
-	rootAcc  Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -58,8 +50,15 @@ type IAMServiceS3 struct {
 	// gateway instance if possible.
 	sync.RWMutex
 
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
 	sslSkipVerify bool
 	debug         bool
+	rootAcc       Account
+	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}

auth/iam_single.go

@@ -15,7 +15,7 @@
 package auth
 
 import (
-	"errors"
+	"github.com/versity/versitygw/s3err"
 )
 
 // IAMServiceSingle manages the single tenant (root-only) IAM service
@@ -23,31 +23,29 @@ type IAMServiceSingle struct{}
 
 var _ IAMService = &IAMServiceSingle{}
 
-var ErrNotSupported = errors.New("method is not supported")
-
 // CreateAccount not valid in single tenant mode
 func (IAMServiceSingle) CreateAccount(account Account) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // GetUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	return Account{}, ErrNoSuchUser
+	return Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // UpdateUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // ListUserAccounts no accounts in single tenant mode
 func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
-	return []Account{}, nil
+	return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // Shutdown graceful termination of service

auth/iam_vault.go

@@ -47,7 +47,7 @@ func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath,
 		tls.ServerCertificate.FromBytes = []byte(serverCert)
 		if clientCert != "" {
 			if clientCertKey == "" {
-				return nil, fmt.Errorf("client certificate and client certificate should both be specified")
+				return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
 			}
 
 			tls.ClientCertificate.FromBytes = []byte(clientCert)

auth/object_lock.go

@@ -29,9 +29,9 @@ import (
 )
 
 type BucketLockConfig struct {
+	Enabled          bool
 	DefaultRetention *types.DefaultRetention
 	CreatedAt        *time.Time
-	Enabled          bool
 }
 
 func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {

aws/signer/v4/v4.go

@@ -85,10 +85,6 @@ type keyDerivator interface {
 
 // SignerOptions is the SigV4 Signer options.
 type SignerOptions struct {
-
-	// The logger to send log messages to.
-	Logger logging.Logger
-
 	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
 	// request header to the request's query string. This is most commonly used
 	// with pre-signed requests preventing headers from being added to the
@@ -104,6 +100,9 @@ type SignerOptions struct {
 	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
 	DisableURIPathEscaping bool
 
+	// The logger to send log messages to.
+	Logger logging.Logger
+
 	// Enable logging of signed requests.
 	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
 	// presigned URL.
@@ -118,8 +117,8 @@ type SignerOptions struct {
 // Signer applies AWS v4 signing to given request. Use this to sign requests
 // that need to be signed with AWS V4 Signatures.
 type Signer struct {
-	keyDerivator keyDerivator
 	options      SignerOptions
+	keyDerivator keyDerivator
 }
 
 // NewSigner returns a new SigV4 Signer
@@ -134,19 +133,17 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
 }
 
 type httpSigner struct {
-	KeyDerivator keyDerivator
 	Request      *http.Request
-	Credentials  aws.Credentials
-	Time         v4Internal.SigningTime
 	ServiceName  string
 	Region       string
+	Time         v4Internal.SigningTime
+	Credentials  aws.Credentials
+	KeyDerivator keyDerivator
+	IsPreSign    bool
+	SignedHdrs   []string
 
 	PayloadHash string
 
-	SignedHdrs []string
-
-	IsPreSign bool
-
 	DisableHeaderHoisting  bool
 	DisableURIPathEscaping bool
 	DisableSessionToken    bool

backend/azure/azure.go

@@ -27,7 +27,6 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -196,7 +195,6 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 }
 
 func (az *Azure) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
-	fmt.Printf("%+v\n", input)
 	pager := az.client.NewListContainersPager(
 		&service.ListContainersOptions{
 			Include: service.ListContainersInclude{
@@ -391,17 +389,29 @@ func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error {
 }
 
 func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	client, err := az.getBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
 	var opts *azblob.DownloadStreamOptions
 	if *input.Range != "" {
-		offset, count, err := backend.ParseRange(0, *input.Range)
+		offset, count, isValid, err := backend.ParseGetObjectRange(*resp.ContentLength, *input.Range)
 		if err != nil {
 			return nil, err
 		}
-		opts = &azblob.DownloadStreamOptions{
-			Range: blob.HTTPRange{
-				Count:  count,
-				Offset: offset,
-			},
+		if isValid {
+			opts = &azblob.DownloadStreamOptions{
+				Range: blob.HTTPRange{
+					Count:  count,
+					Offset: offset,
+				},
+			}
 		}
 	}
 	blobDownloadResponse, err := az.client.DownloadStream(ctx, *input.Bucket, *input.Key, opts)
@@ -420,7 +430,7 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G
 	}
 
 	return &s3.GetObjectOutput{
-		AcceptRanges:    input.Range,
+		AcceptRanges:    backend.GetPtrFromString("bytes"),
 		ContentLength:   blobDownloadResponse.ContentLength,
 		ContentEncoding: blobDownloadResponse.ContentEncoding,
 		ContentType:     contentType,
@@ -522,7 +532,7 @@ func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAtt
 	}
 
 	return s3response.GetObjectAttributesResponse{
-		ETag:         data.ETag,
+		ETag:         backend.TrimEtag(data.ETag),
 		ObjectSize:   data.ContentLength,
 		StorageClass: data.StorageClass,
 		LastModified: data.LastModified,
@@ -565,7 +575,7 @@ Pager:
 				break Pager
 			}
 			objects = append(objects, s3response.Object{
-				ETag:         (*string)(v.Properties.ETag),
+				ETag:         backend.GetPtrFromString(fmt.Sprintf("%q", *v.Properties.ETag)),
 				Key:          v.Name,
 				LastModified: v.Properties.LastModified,
 				Size:         v.Properties.ContentLength,
@@ -647,7 +657,7 @@ Pager:
 				break Pager
 			}
 			objects = append(objects, s3response.Object{
-				ETag:         (*string)(v.Properties.ETag),
+				ETag:         backend.GetPtrFromString(fmt.Sprintf("%q", *v.Properties.ETag)),
 				Key:          v.Name,
 				LastModified: v.Properties.LastModified,
 				Size:         v.Properties.ContentLength,
@@ -906,9 +916,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
 }
 
 // Each part is translated into an uncommitted block in a newly created blob in staging area
-func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 	if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
-		return "", err
+		return nil, err
 	}
 
 	// TODO: request streamable version of StageBlock()
@@ -917,32 +927,34 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta
 	// the body in memory to create an io.ReadSeekCloser
 	rdr, err := getReadSeekCloser(input.Body)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	// block id serves as etag here
-	etag = blockIDInt32ToBase64(*input.PartNumber)
+	etag := blockIDInt32ToBase64(*input.PartNumber)
 	_, err = client.StageBlock(ctx, etag, rdr, nil)
 	if err != nil {
-		return "", parseMpError(err)
+		return nil, parseMpError(err)
 	}
 
-	return etag, nil
+	return &s3.UploadPartOutput{
+		ETag: &etag,
+	}, nil
 }
 
-func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
-		return s3response.CopyObjectResult{}, nil
+		return s3response.CopyPartResult{}, nil
 	}
 
 	if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
-		return s3response.CopyObjectResult{}, err
+		return s3response.CopyPartResult{}, err
 	}
 
 	eTag := blockIDInt32ToBase64(*input.PartNumber)
@@ -950,10 +962,10 @@ func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInp
 	//TODO: the action returns not implemented on azurite, maybe in production this will work?
 	_, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil)
 	if err != nil {
-		return s3response.CopyObjectResult{}, parseMpError(err)
+		return s3response.CopyPartResult{}, parseMpError(err)
 	}
 
-	return s3response.CopyObjectResult{}, nil
+	return s3response.CopyPartResult{}, nil
 }
 
 // Lists all uncommitted parts from the blob
@@ -1196,27 +1208,53 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 		return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 	}
 
-	slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int {
-		ptNumber, _ := decodeBlockId(*a.Name)
-		nextPtNumber, _ := decodeBlockId(*b.Name)
-		return ptNumber - nextPtNumber
-	})
-
-	for i, block := range blockList.UncommittedBlocks {
-		ptNumber, err := decodeBlockId(*block.Name)
+	uncommittedBlocks := map[int32]*blockblob.Block{}
+	for _, el := range blockList.UncommittedBlocks {
+		ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name))
 		if err != nil {
+			return nil, fmt.Errorf("invalid block name: %w", err)
+		}
+
+		uncommittedBlocks[int32(ptNumber)] = el
+	}
+
+	// The initialie values is the lower limit of partNumber: 0
+	var totalSize int64
+	var partNumber int32
+	last := len(blockList.UncommittedBlocks) - 1
+	for i, part := range input.MultipartUpload.Parts {
+		if part.PartNumber == nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
+		}
+		if *part.PartNumber <= partNumber {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder)
+		}
+		partNumber = *part.PartNumber
+
+		block, ok := uncommittedBlocks[*part.PartNumber]
+		if !ok {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
-		if *input.MultipartUpload.Parts[i].ETag != *block.Name {
+		if *part.ETag != *block.Name {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		// all parts except the last need to be greater, than
+		// the minimum allowed size (5 Mib)
+		if i < last && *block.Size < backend.MinPartSize {
+			return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
 		}
+		totalSize += *block.Size
 		blockIds = append(blockIds, *block.Name)
 	}
 
+	if input.MpuObjectSize != nil && totalSize != *input.MpuObjectSize {
+		return nil, s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize)
+	}
+
 	opts := &blockblob.CommitBlockListOptions{
 		Metadata: props.Metadata,
 		Tags:     parseAzTags(tags.BlobTagSet),
@@ -1459,7 +1497,10 @@ func (az *Azure) ChangeBucketOwner(ctx context.Context, bucket string, acl []byt
 // The action actually returns the containers owned by the user, who initialized the gateway
 // TODO: Not sure if there's a way to list all the containers and owners?
 func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
-	pager := az.client.NewListContainersPager(nil)
+	opts := &service.ListContainersOptions{
+		Include: service.ListContainersInclude{Metadata: true},
+	}
+	pager := az.client.NewListContainersPager(opts)
 
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
@@ -1735,9 +1776,11 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string
 }
 
 func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
+	var acl auth.ACL
+
 	data, ok := meta[string(key)]
 	if !ok {
-		return nil, s3err.GetAPIError(s3err.ErrInternalError)
+		return &acl, nil
 	}
 
 	value, err := decodeString(*data)
@@ -1745,7 +1788,6 @@ func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 		return nil, err
 	}
 
-	var acl auth.ACL
 	if len(value) == 0 {
 		return &acl, nil
 	}

backend/backend.go

@@ -53,8 +53,8 @@ type Backend interface {
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
-	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
-	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// standard object operations
 	PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
@@ -166,11 +166,11 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
 func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
 	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+	return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {

backend/common.go

@@ -19,7 +19,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -34,6 +33,9 @@ const (
 	// this is the media type for directories in AWS and Nextcloud
 	DirContentType     = "application/x-directory"
 	DefaultContentType = "binary/octet-stream"
+
+	// this is the minimum allowed size for mp parts
+	MinPartSize = 5 * 1024 * 1024
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -68,45 +70,118 @@ func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
 
+func TrimEtag(etag *string) *string {
+	if etag == nil {
+		return nil
+	}
+
+	return GetPtrFromString(strings.Trim(*etag, "\""))
+}
+
 var (
-	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidRange           = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
 )
 
-// ParseRange parses input range header and returns startoffset, length, and
-// error. If no endoffset specified, then length is set to -1.
-func ParseRange(size int64, acceptRange string) (int64, int64, error) {
+// ParseGetObjectRange parses input range header and returns startoffset, length, isValid
+// and error. If no endoffset specified, then length is set to the object size
+// for invalid inputs, it returns no error, but isValid=false
+// `InvalidRange` error is returnd, only if startoffset is greater than the object size
+func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
+	if acceptRange == "" {
+		return 0, size, false, nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) != 2 {
+		return 0, size, false, nil
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, size, false, nil
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) != 2 {
+		return 0, size, false, nil
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, size, false, nil
+	}
+
+	if startOffset >= size {
+		return 0, 0, false, errInvalidRange
+	}
+
+	if bRange[1] == "" {
+		return startOffset, size - startOffset, true, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, size, false, nil
+	}
+
+	if endOffset < startOffset {
+		return 0, size, false, nil
+	}
+
+	if endOffset >= size {
+		return startOffset, size - startOffset, true, nil
+	}
+
+	return startOffset, endOffset - startOffset + 1, true, nil
+}
+
+// ParseCopySourceRange parses input range header and returns startoffset, length
+// and error. If no endoffset specified, then length is set to the object size
+func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
 		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
 
-	if len(rangeKv) < 2 {
-		return 0, 0, errInvalidRange
+	if len(rangeKv) != 2 {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 1 || len(bRange) > 2 {
-		return 0, 0, errInvalidRange
+	if len(bRange) != 2 {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
-	endOffset := int64(-1)
-	if len(bRange) == 1 || bRange[1] == "" {
-		return startOffset, endOffset, nil
+	if startOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
-	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
+	if bRange[1] == "" {
+		return startOffset, size - startOffset + 1, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	if endOffset < startOffset {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if endOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
@@ -136,21 +211,13 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
 	return srcBucket, srcObject, versionId, nil
 }
 
-func CreateExceedingRangeErr(objSize int64) s3err.APIError {
-	return s3err.APIError{
-		Code:           "InvalidArgument",
-		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
-		HTTPStatusCode: http.StatusBadRequest,
-	}
-}
-
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
 		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
 	}
-	s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
-	return s3MD5
+
+	return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
 }
 
 func getEtagBytes(etag string) []byte {

backend/meta/none.go

@@ -0,0 +1,54 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"os"
+)
+
+// NoMeta is a metadata storer that does not store metadata.
+// This can be useful for read only mounts where attempting to store metadata
+// would fail.
+type NoMeta struct{}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+// always returns ErrNoSuchKey
+func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
+	return nil, ErrNoSuchKey
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+// always returns nil without storing the attribute
+func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+// always returns nil without deleting the attribute
+func (NoMeta) DeleteAttribute(_, _, _ string) error {
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+// always returns an empty list of attributes
+func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
+	return []string{}, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+// always returns nil without deleting any attributes
+func (NoMeta) DeleteAttributes(bucket, object string) error {
+	return nil
+}

backend/meta/sidecar.go

@@ -0,0 +1,139 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// SideCar is a metadata storer that uses sidecar files to store metadata.
+type SideCar struct {
+	dir string
+}
+
+const (
+	sidecarmeta = "meta"
+)
+
+// NewSideCar creates a new SideCar metadata storer.
+func NewSideCar(dir string) (SideCar, error) {
+	fi, err := os.Lstat(dir)
+	if err != nil {
+		return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
+	}
+	if !fi.IsDir() {
+		return SideCar{}, fmt.Errorf("not a directory")
+	}
+
+	return SideCar{dir: dir}, nil
+}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	value, err := os.ReadFile(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, ErrNoSuchKey
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attribute: %v", err)
+	}
+
+	return value, nil
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	err := os.MkdirAll(metadir, 0777)
+	if err != nil {
+		return fmt.Errorf("failed to create metadata directory: %v", err)
+	}
+
+	attr := filepath.Join(metadir, attribute)
+	err = os.WriteFile(attr, value, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to write attribute: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	err := os.Remove(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return ErrNoSuchKey
+	}
+	if err != nil {
+		return fmt.Errorf("failed to remove attribute: %v", err)
+	}
+
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	ents, err := os.ReadDir(metadir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []string{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to list attributes: %v", err)
+	}
+
+	var attrs []string
+	for _, ent := range ents {
+		attrs = append(attrs, ent.Name())
+	}
+
+	return attrs, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+func (s SideCar) DeleteAttributes(bucket, object string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	err := os.RemoveAll(metadir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove attributes: %v", err)
+	}
+	return nil
+}

backend/meta/xattr.go

@@ -23,6 +23,7 @@ import (
 	"syscall"
 
 	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/s3err"
 )
 
 const (
@@ -56,10 +57,18 @@ func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute strin
 // StoreAttribute stores the value of a specific attribute for an object in a bucket.
 func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
 	if f != nil {
-		return xattr.FSet(f, xattrPrefix+attribute, value)
+		err := xattr.FSet(f, xattrPrefix+attribute, value)
+		if errors.Is(err, syscall.EROFS) {
+			return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return err
 	}
 
-	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
+	return err
 }
 
 // DeleteAttribute removes the value of a specific attribute for an object in a bucket.
@@ -68,6 +77,9 @@ func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
 	if errors.Is(err, xattr.ENOATTR) {
 		return ErrNoSuchKey
 	}
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
 	return err
 }
 

backend/posix/with_otmpfile.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 	"golang.org/x/sys/unix"
 )
 
@@ -38,12 +39,12 @@ type tmpfile struct {
 	f          *os.File
 	bucket     string
 	objname    string
+	isOTmp     bool
 	size       int64
+	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
-	isOTmp     bool
-	needsChown bool
 }
 
 var (
@@ -62,6 +63,10 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	// this is not supported.
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+
 		// O_TMPFILE not supported, try fallback
 		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 		if err != nil {

backend/posix/without_otmpfile.go

@@ -24,9 +24,11 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"syscall"
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 )
 
 type tmpfile struct {
@@ -43,11 +45,17 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	var err error
 	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("create temp file: %w", err)
 	}
 

backend/s3proxy/s3.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -106,11 +107,29 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsI
 }
 
 func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
 	out, err := s.client.HeadBucket(ctx, input)
 	return out, handleError(err)
 }
 
 func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
+	if input.GrantFullControl != nil && *input.GrantFullControl == "" {
+		input.GrantFullControl = nil
+	}
+	if input.GrantRead != nil && *input.GrantRead == "" {
+		input.GrantRead = nil
+	}
+	if input.GrantReadACP != nil && *input.GrantReadACP == "" {
+		input.GrantReadACP = nil
+	}
+	if input.GrantWrite != nil && *input.GrantWrite == "" {
+		input.GrantWrite = nil
+	}
+	if input.GrantWriteACP != nil && *input.GrantWriteACP == "" {
+		input.GrantWriteACP = nil
+	}
 	_, err := s.client.CreateBucket(ctx, input)
 	if err != nil {
 		return handleError(err)
@@ -192,6 +211,25 @@ func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3res
 }
 
 func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	if input.Delimiter != nil && *input.Delimiter == "" {
+		input.Delimiter = nil
+	}
+	if input.Prefix != nil && *input.Prefix == "" {
+		input.Prefix = nil
+	}
+	if input.KeyMarker != nil && *input.KeyMarker == "" {
+		input.KeyMarker = nil
+	}
+	if input.VersionIdMarker != nil && *input.VersionIdMarker == "" {
+		input.VersionIdMarker = nil
+	}
+	if input.MaxKeys != nil && *input.MaxKeys == 0 {
+		input.MaxKeys = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+
 	out, err := s.client.ListObjectVersions(ctx, input)
 	if err != nil {
 		return s3response.ListVersionsResult{}, handleError(err)
@@ -214,7 +252,67 @@ func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVe
 	}, nil
 }
 
+var defTime = time.Time{}
+
 func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+	if input.CacheControl != nil && *input.CacheControl == "" {
+		input.CacheControl = nil
+	}
+	if input.ContentDisposition != nil && *input.ContentDisposition == "" {
+		input.ContentDisposition = nil
+	}
+	if input.ContentEncoding != nil && *input.ContentEncoding == "" {
+		input.ContentEncoding = nil
+	}
+	if input.ContentLanguage != nil && *input.ContentLanguage == "" {
+		input.ContentLanguage = nil
+	}
+	if input.ContentType != nil && *input.ContentType == "" {
+		input.ContentType = nil
+	}
+	if input.Expires != nil && *input.Expires == defTime {
+		input.Expires = nil
+	}
+	if input.GrantFullControl != nil && *input.GrantFullControl == "" {
+		input.GrantFullControl = nil
+	}
+	if input.GrantRead != nil && *input.GrantRead == "" {
+		input.GrantRead = nil
+	}
+	if input.GrantReadACP != nil && *input.GrantReadACP == "" {
+		input.GrantReadACP = nil
+	}
+	if input.GrantWriteACP != nil && *input.GrantWriteACP == "" {
+		input.GrantWriteACP = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.ObjectLockRetainUntilDate != nil && *input.ObjectLockRetainUntilDate == defTime {
+		input.ObjectLockRetainUntilDate = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+	if input.SSEKMSKeyId != nil && *input.SSEKMSKeyId == "" {
+		input.SSEKMSKeyId = nil
+	}
+	if input.SSEKMSEncryptionContext != nil && *input.SSEKMSEncryptionContext == "" {
+		input.SSEKMSEncryptionContext = nil
+	}
+	if input.Tagging != nil && *input.Tagging == "" {
+		input.Tagging = nil
+	}
+	if input.WebsiteRedirectLocation != nil && *input.WebsiteRedirectLocation == "" {
+		input.WebsiteRedirectLocation = nil
+	}
+
 	out, err := s.client.CreateMultipartUpload(ctx, input)
 	if err != nil {
 		return s3response.InitiateMultipartUploadResult{}, handleError(err)
@@ -228,16 +326,78 @@ func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMul
 }
 
 func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 == "" {
+		input.ChecksumCRC32 = nil
+	}
+	if input.ChecksumCRC32C != nil && *input.ChecksumCRC32C == "" {
+		input.ChecksumCRC32C = nil
+	}
+	if input.ChecksumCRC64NVME != nil && *input.ChecksumCRC64NVME == "" {
+		input.ChecksumCRC64NVME = nil
+	}
+	if input.ChecksumSHA1 != nil && *input.ChecksumSHA1 == "" {
+		input.ChecksumSHA1 = nil
+	}
+	if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
+		input.ChecksumSHA256 = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.IfMatch != nil && *input.IfMatch == "" {
+		input.IfMatch = nil
+	}
+	if input.IfNoneMatch != nil && *input.IfNoneMatch == "" {
+		input.IfNoneMatch = nil
+	}
+	if input.MpuObjectSize != nil && *input.MpuObjectSize == 0 {
+		input.MpuObjectSize = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+
 	out, err := s.client.CompleteMultipartUpload(ctx, input)
 	return out, handleError(err)
 }
 
 func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.IfMatchInitiatedTime != nil && *input.IfMatchInitiatedTime == defTime {
+		input.IfMatchInitiatedTime = nil
+	}
 	_, err := s.client.AbortMultipartUpload(ctx, input)
 	return handleError(err)
 }
 
 func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	if input.Delimiter != nil && *input.Delimiter == "" {
+		input.Delimiter = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.KeyMarker != nil && *input.KeyMarker == "" {
+		input.KeyMarker = nil
+	}
+	if input.MaxUploads != nil && *input.MaxUploads == 0 {
+		input.MaxUploads = nil
+	}
+	if input.Prefix != nil && *input.Prefix == "" {
+		input.Prefix = nil
+	}
+	if input.UploadIdMarker != nil && *input.UploadIdMarker == "" {
+		input.UploadIdMarker = nil
+	}
+
 	output, err := s.client.ListMultipartUploads(ctx, input)
 	if err != nil {
 		return s3response.ListMultipartUploadsResult{}, handleError(err)
@@ -256,8 +416,10 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
 				ID:          *u.Owner.ID,
 				DisplayName: *u.Owner.DisplayName,
 			},
-			StorageClass: u.StorageClass,
-			Initiated:    *u.Initiated,
+			StorageClass:      u.StorageClass,
+			Initiated:         *u.Initiated,
+			ChecksumAlgorithm: u.ChecksumAlgorithm,
+			ChecksumType:      u.ChecksumType,
 		})
 	}
 
@@ -285,6 +447,25 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
 }
 
 func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.MaxParts != nil && *input.MaxParts == 0 {
+		input.MaxParts = nil
+	}
+	if input.PartNumberMarker != nil && *input.PartNumberMarker == "" {
+		input.PartNumberMarker = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+
 	output, err := s.client.ListParts(ctx, input)
 	if err != nil {
 		return s3response.ListPartsResult{}, handleError(err)
@@ -293,10 +474,15 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
 	var parts []s3response.Part
 	for _, p := range output.Parts {
 		parts = append(parts, s3response.Part{
-			PartNumber:   int(*p.PartNumber),
-			LastModified: *p.LastModified,
-			ETag:         *p.ETag,
-			Size:         *p.Size,
+			PartNumber:        int(*p.PartNumber),
+			LastModified:      *p.LastModified,
+			ETag:              *p.ETag,
+			Size:              *p.Size,
+			ChecksumCRC32:     p.ChecksumCRC32,
+			ChecksumCRC32C:    p.ChecksumCRC32C,
+			ChecksumCRC64NVME: p.ChecksumCRC64NVME,
+			ChecksumSHA1:      p.ChecksumSHA1,
+			ChecksumSHA256:    p.ChecksumSHA256,
 		})
 	}
 	pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -329,35 +515,193 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
 		MaxParts:             int(*output.MaxParts),
 		IsTruncated:          *output.IsTruncated,
 		Parts:                parts,
+		ChecksumAlgorithm:    output.ChecksumAlgorithm,
+		ChecksumType:         output.ChecksumType,
 	}, nil
 }
 
-func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 == "" {
+		input.ChecksumCRC32 = nil
+	}
+	if input.ChecksumCRC32C != nil && *input.ChecksumCRC32C == "" {
+		input.ChecksumCRC32C = nil
+	}
+	if input.ChecksumCRC64NVME != nil && *input.ChecksumCRC64NVME == "" {
+		input.ChecksumCRC64NVME = nil
+	}
+	if input.ChecksumSHA1 != nil && *input.ChecksumSHA1 == "" {
+		input.ChecksumSHA1 = nil
+	}
+	if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
+		input.ChecksumSHA256 = nil
+	}
+	if input.ContentMD5 != nil && *input.ContentMD5 == "" {
+		input.ContentMD5 = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+
 	// streaming backend is not seekable,
 	// use unsigned payload for streaming ops
 	output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
 		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
 	))
-	if err != nil {
-		return "", handleError(err)
-	}
-
-	return *output.ETag, nil
+	return output, handleError(err)
 }
 
-func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	output, err := s.client.UploadPartCopy(ctx, input)
-	if err != nil {
-		return s3response.CopyObjectResult{}, handleError(err)
+func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+	if input.CopySourceIfMatch != nil && *input.CopySourceIfMatch == "" {
+		input.CopySourceIfMatch = nil
+	}
+	if input.CopySourceIfModifiedSince != nil && *input.CopySourceIfModifiedSince == defTime {
+		input.CopySourceIfModifiedSince = nil
+	}
+	if input.CopySourceIfNoneMatch != nil && *input.CopySourceIfNoneMatch == "" {
+		input.CopySourceIfNoneMatch = nil
+	}
+	if input.CopySourceIfUnmodifiedSince != nil && *input.CopySourceIfUnmodifiedSince == defTime {
+		input.CopySourceIfUnmodifiedSince = nil
+	}
+	if input.CopySourceRange != nil && *input.CopySourceRange == "" {
+		input.CopySourceRange = nil
+	}
+	if input.CopySourceSSECustomerAlgorithm != nil && *input.CopySourceSSECustomerAlgorithm == "" {
+		input.CopySourceSSECustomerAlgorithm = nil
+	}
+	if input.CopySourceSSECustomerKey != nil && *input.CopySourceSSECustomerKey == "" {
+		input.CopySourceSSECustomerKey = nil
+	}
+	if input.CopySourceSSECustomerKeyMD5 != nil && *input.CopySourceSSECustomerKeyMD5 == "" {
+		input.CopySourceSSECustomerKeyMD5 = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.ExpectedSourceBucketOwner != nil && *input.ExpectedSourceBucketOwner == "" {
+		input.ExpectedSourceBucketOwner = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
 	}
 
-	return s3response.CopyObjectResult{
-		LastModified: *output.CopyPartResult.LastModified,
-		ETag:         *output.CopyPartResult.ETag,
+	output, err := s.client.UploadPartCopy(ctx, input)
+	if err != nil {
+		return s3response.CopyPartResult{}, handleError(err)
+	}
+
+	return s3response.CopyPartResult{
+		LastModified:      *output.CopyPartResult.LastModified,
+		ETag:              output.CopyPartResult.ETag,
+		ChecksumCRC32:     output.CopyPartResult.ChecksumCRC32,
+		ChecksumCRC32C:    output.CopyPartResult.ChecksumCRC32C,
+		ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME,
+		ChecksumSHA1:      output.CopyPartResult.ChecksumSHA1,
+		ChecksumSHA256:    output.CopyPartResult.ChecksumSHA256,
 	}, nil
 }
 
 func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+	if input.CacheControl != nil && *input.CacheControl == "" {
+		input.CacheControl = nil
+	}
+	if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 == "" {
+		input.ChecksumCRC32 = nil
+	}
+	if input.ChecksumCRC32C != nil && *input.ChecksumCRC32C == "" {
+		input.ChecksumCRC32C = nil
+	}
+	if input.ChecksumCRC64NVME != nil && *input.ChecksumCRC64NVME == "" {
+		input.ChecksumCRC64NVME = nil
+	}
+	if input.ChecksumSHA1 != nil && *input.ChecksumSHA1 == "" {
+		input.ChecksumSHA1 = nil
+	}
+	if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
+		input.ChecksumSHA256 = nil
+	}
+	if input.ContentDisposition != nil && *input.ContentDisposition == "" {
+		input.ContentDisposition = nil
+	}
+	if input.ContentEncoding != nil && *input.ContentEncoding == "" {
+		input.ContentEncoding = nil
+	}
+	if input.ContentLanguage != nil && *input.ContentLanguage == "" {
+		input.ContentLanguage = nil
+	}
+	if input.ContentMD5 != nil && *input.ContentMD5 == "" {
+		input.ContentMD5 = nil
+	}
+	if input.ContentType != nil && *input.ContentType == "" {
+		input.ContentType = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.Expires != nil && *input.Expires == defTime {
+		input.Expires = nil
+	}
+	if input.GrantFullControl != nil && *input.GrantFullControl == "" {
+		input.GrantFullControl = nil
+	}
+	if input.GrantRead != nil && *input.GrantRead == "" {
+		input.GrantRead = nil
+	}
+	if input.GrantReadACP != nil && *input.GrantReadACP == "" {
+		input.GrantReadACP = nil
+	}
+	if input.GrantWriteACP != nil && *input.GrantWriteACP == "" {
+		input.GrantWriteACP = nil
+	}
+	if input.IfMatch != nil && *input.IfMatch == "" {
+		input.IfMatch = nil
+	}
+	if input.IfNoneMatch != nil && *input.IfNoneMatch == "" {
+		input.IfNoneMatch = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+	if input.SSEKMSEncryptionContext != nil && *input.SSEKMSEncryptionContext == "" {
+		input.SSEKMSEncryptionContext = nil
+	}
+	if input.SSEKMSKeyId != nil && *input.SSEKMSKeyId == "" {
+		input.SSEKMSKeyId = nil
+	}
+	if input.Tagging != nil && *input.Tagging == "" {
+		input.Tagging = nil
+	}
+	if input.WebsiteRedirectLocation != nil && *input.WebsiteRedirectLocation == "" {
+		input.WebsiteRedirectLocation = nil
+	}
+
+	// no object lock for backend
+	input.ObjectLockRetainUntilDate = nil
+	input.ObjectLockMode = ""
+	input.ObjectLockLegalHoldStatus = ""
+
 	// streaming backend is not seekable,
 	// use unsigned payload for streaming ops
 	output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
@@ -373,17 +717,126 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re
 	}
 
 	return s3response.PutObjectOutput{
-		ETag:      *output.ETag,
-		VersionID: versionID,
+		ETag:              *output.ETag,
+		VersionID:         versionID,
+		ChecksumCRC32:     output.ChecksumCRC32,
+		ChecksumCRC32C:    output.ChecksumCRC32C,
+		ChecksumCRC64NVME: output.ChecksumCRC64NVME,
+		ChecksumSHA1:      output.ChecksumSHA1,
+		ChecksumSHA256:    output.ChecksumSHA256,
 	}, nil
 }
 
 func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.IfMatch != nil && *input.IfMatch == "" {
+		input.IfMatch = nil
+	}
+	if input.IfModifiedSince != nil && *input.IfModifiedSince == defTime {
+		input.IfModifiedSince = nil
+	}
+	if input.IfNoneMatch != nil && *input.IfNoneMatch == "" {
+		input.IfNoneMatch = nil
+	}
+	if input.IfUnmodifiedSince != nil && *input.IfUnmodifiedSince == defTime {
+		input.IfUnmodifiedSince = nil
+	}
+	if input.PartNumber != nil && *input.PartNumber == 0 {
+		input.PartNumber = nil
+	}
+	if input.Range != nil && *input.Range == "" {
+		input.Range = nil
+	}
+	if input.ResponseCacheControl != nil && *input.ResponseCacheControl == "" {
+		input.ResponseCacheControl = nil
+	}
+	if input.ResponseContentDisposition != nil && *input.ResponseContentDisposition == "" {
+		input.ResponseContentDisposition = nil
+	}
+	if input.ResponseContentEncoding != nil && *input.ResponseContentEncoding == "" {
+		input.ResponseContentEncoding = nil
+	}
+	if input.ResponseContentLanguage != nil && *input.ResponseContentLanguage == "" {
+		input.ResponseContentLanguage = nil
+	}
+	if input.ResponseContentType != nil && *input.ResponseContentType == "" {
+		input.ResponseContentType = nil
+	}
+	if input.ResponseExpires != nil && *input.ResponseExpires == defTime {
+		input.ResponseExpires = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+	if input.VersionId != nil && *input.VersionId == "" {
+		input.VersionId = nil
+	}
+
 	out, err := s.client.HeadObject(ctx, input)
 	return out, handleError(err)
 }
 
 func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.IfMatch != nil && *input.IfMatch == "" {
+		input.IfMatch = nil
+	}
+	if input.IfModifiedSince != nil && *input.IfModifiedSince == defTime {
+		input.IfModifiedSince = nil
+	}
+	if input.IfNoneMatch != nil && *input.IfNoneMatch == "" {
+		input.IfNoneMatch = nil
+	}
+	if input.IfUnmodifiedSince != nil && *input.IfUnmodifiedSince == defTime {
+		input.IfUnmodifiedSince = nil
+	}
+	if input.PartNumber != nil && *input.PartNumber == 0 {
+		input.PartNumber = nil
+	}
+	if input.Range != nil && *input.Range == "" {
+		input.Range = nil
+	}
+	if input.ResponseCacheControl != nil && *input.ResponseCacheControl == "" {
+		input.ResponseCacheControl = nil
+	}
+	if input.ResponseContentDisposition != nil && *input.ResponseContentDisposition == "" {
+		input.ResponseContentDisposition = nil
+	}
+	if input.ResponseContentEncoding != nil && *input.ResponseContentEncoding == "" {
+		input.ResponseContentEncoding = nil
+	}
+	if input.ResponseContentLanguage != nil && *input.ResponseContentLanguage == "" {
+		input.ResponseContentLanguage = nil
+	}
+	if input.ResponseContentType != nil && *input.ResponseContentType == "" {
+		input.ResponseContentType = nil
+	}
+	if input.ResponseExpires != nil && *input.ResponseExpires == defTime {
+		input.ResponseExpires = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+	if input.VersionId != nil && *input.VersionId == "" {
+		input.VersionId = nil
+	}
+
 	output, err := s.client.GetObject(ctx, input)
 	if err != nil {
 		return nil, handleError(err)
@@ -393,6 +846,28 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
 }
 
 func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.MaxParts != nil && *input.MaxParts == 0 {
+		input.MaxParts = nil
+	}
+	if input.PartNumberMarker != nil && *input.PartNumberMarker == "" {
+		input.PartNumberMarker = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+	if input.VersionId != nil && *input.VersionId == "" {
+		input.VersionId = nil
+	}
+
 	out, err := s.client.GetObjectAttributes(ctx, input)
 
 	parts := s3response.ObjectParts{}
@@ -425,15 +900,114 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
 		ObjectSize:   out.ObjectSize,
 		StorageClass: out.StorageClass,
 		ObjectParts:  &parts,
+		Checksum:     out.Checksum,
 	}, handleError(err)
 }
 
 func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	if input.CacheControl != nil && *input.CacheControl == "" {
+		input.CacheControl = nil
+	}
+	if input.ContentDisposition != nil && *input.ContentDisposition == "" {
+		input.ContentDisposition = nil
+	}
+	if input.ContentEncoding != nil && *input.ContentEncoding == "" {
+		input.ContentEncoding = nil
+	}
+	if input.ContentLanguage != nil && *input.ContentLanguage == "" {
+		input.ContentLanguage = nil
+	}
+	if input.ContentType != nil && *input.ContentType == "" {
+		input.ContentType = nil
+	}
+	if input.CopySourceIfMatch != nil && *input.CopySourceIfMatch == "" {
+		input.CopySourceIfMatch = nil
+	}
+	if input.CopySourceIfModifiedSince != nil && *input.CopySourceIfModifiedSince == defTime {
+		input.CopySourceIfModifiedSince = nil
+	}
+	if input.CopySourceIfNoneMatch != nil && *input.CopySourceIfNoneMatch == "" {
+		input.CopySourceIfNoneMatch = nil
+	}
+	if input.CopySourceIfUnmodifiedSince != nil && *input.CopySourceIfUnmodifiedSince == defTime {
+		input.CopySourceIfUnmodifiedSince = nil
+	}
+	if input.CopySourceSSECustomerAlgorithm != nil && *input.CopySourceSSECustomerAlgorithm == "" {
+		input.CopySourceSSECustomerAlgorithm = nil
+	}
+	if input.CopySourceSSECustomerKey != nil && *input.CopySourceSSECustomerKey == "" {
+		input.CopySourceSSECustomerKey = nil
+	}
+	if input.CopySourceSSECustomerKeyMD5 != nil && *input.CopySourceSSECustomerKeyMD5 == "" {
+		input.CopySourceSSECustomerKeyMD5 = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.ExpectedSourceBucketOwner != nil && *input.ExpectedSourceBucketOwner == "" {
+		input.ExpectedSourceBucketOwner = nil
+	}
+	if input.Expires != nil && *input.Expires == defTime {
+		input.Expires = nil
+	}
+	if input.GrantFullControl != nil && *input.GrantFullControl == "" {
+		input.GrantFullControl = nil
+	}
+	if input.GrantRead != nil && *input.GrantRead == "" {
+		input.GrantRead = nil
+	}
+	if input.GrantReadACP != nil && *input.GrantReadACP == "" {
+		input.GrantReadACP = nil
+	}
+	if input.GrantWriteACP != nil && *input.GrantWriteACP == "" {
+		input.GrantWriteACP = nil
+	}
+	if input.ObjectLockRetainUntilDate != nil && *input.ObjectLockRetainUntilDate == defTime {
+		input.ObjectLockRetainUntilDate = nil
+	}
+	if input.SSECustomerAlgorithm != nil && *input.SSECustomerAlgorithm == "" {
+		input.SSECustomerAlgorithm = nil
+	}
+	if input.SSECustomerKey != nil && *input.SSECustomerKey == "" {
+		input.SSECustomerKey = nil
+	}
+	if input.SSECustomerKeyMD5 != nil && *input.SSECustomerKeyMD5 == "" {
+		input.SSECustomerKeyMD5 = nil
+	}
+	if input.SSEKMSEncryptionContext != nil && *input.SSEKMSEncryptionContext == "" {
+		input.SSEKMSEncryptionContext = nil
+	}
+	if input.SSEKMSKeyId != nil && *input.SSEKMSKeyId == "" {
+		input.SSEKMSKeyId = nil
+	}
+	if input.Tagging != nil && *input.Tagging == "" {
+		input.Tagging = nil
+	}
+	if input.WebsiteRedirectLocation != nil && *input.WebsiteRedirectLocation == "" {
+		input.WebsiteRedirectLocation = nil
+	}
+
 	out, err := s.client.CopyObject(ctx, input)
 	return out, handleError(err)
 }
 
 func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+	if input.Delimiter != nil && *input.Delimiter == "" {
+		input.Delimiter = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.Marker != nil && *input.Marker == "" {
+		input.Marker = nil
+	}
+	if input.MaxKeys != nil && *input.MaxKeys == 0 {
+		input.MaxKeys = nil
+	}
+	if input.Prefix != nil && *input.Prefix == "" {
+		input.Prefix = nil
+	}
+
 	out, err := s.client.ListObjects(ctx, input)
 	if err != nil {
 		return s3response.ListObjectsResult{}, handleError(err)
@@ -455,6 +1029,25 @@ func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
 }
 
 func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+	if input.ContinuationToken != nil && *input.ContinuationToken == "" {
+		input.ContinuationToken = nil
+	}
+	if input.Delimiter != nil && *input.Delimiter == "" {
+		input.Delimiter = nil
+	}
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.MaxKeys != nil && *input.MaxKeys == 0 {
+		input.MaxKeys = nil
+	}
+	if input.Prefix != nil && *input.Prefix == "" {
+		input.Prefix = nil
+	}
+	if input.StartAfter != nil && *input.StartAfter == "" {
+		input.StartAfter = nil
+	}
+
 	out, err := s.client.ListObjectsV2(ctx, input)
 	if err != nil {
 		return s3response.ListObjectsV2Result{}, handleError(err)
@@ -477,11 +1070,37 @@ func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
 }
 
 func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.IfMatch != nil && *input.IfMatch == "" {
+		input.IfMatch = nil
+	}
+	if input.IfMatchLastModifiedTime != nil && *input.IfMatchLastModifiedTime == defTime {
+		input.IfMatchLastModifiedTime = nil
+	}
+	if input.IfMatchSize != nil && *input.IfMatchSize == 0 {
+		input.IfMatchSize = nil
+	}
+	if input.MFA != nil && *input.MFA == "" {
+		input.MFA = nil
+	}
+	if input.VersionId != nil && *input.VersionId == "" {
+		input.VersionId = nil
+	}
+
 	res, err := s.client.DeleteObject(ctx, input)
 	return res, handleError(err)
 }
 
 func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+	if input.MFA != nil && *input.MFA == "" {
+		input.MFA = nil
+	}
+
 	if len(input.Delete.Objects) == 0 {
 		input.Delete.Objects = []types.ObjectIdentifier{}
 	}
@@ -498,10 +1117,25 @@ func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInpu
 }
 
 func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
+		input.ExpectedBucketOwner = nil
+	}
+
 	tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
 		Bucket: input.Bucket,
 	})
 	if err != nil {
+		var ae smithy.APIError
+		if errors.As(err, &ae) {
+			// sdk issue workaround for missing NoSuchTagSet error type
+			// https://github.com/aws/aws-sdk-go-v2/issues/2878
+			if strings.Contains(ae.ErrorCode(), "NoSuchTagSet") {
+				return []byte{}, nil
+			}
+			if strings.Contains(ae.ErrorCode(), "NotImplemented") {
+				return []byte{}, nil
+			}
+		}
 		return nil, handleError(err)
 	}
 
@@ -629,95 +1263,28 @@ func (s *S3Proxy) DeleteBucketPolicy(ctx context.Context, bucket string) error {
 }
 
 func (s *S3Proxy) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
-	cfg, err := auth.ParseBucketLockConfigurationOutput(config)
-	if err != nil {
-		return err
-	}
-
-	_, err = s.client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{
-		Bucket:                  &bucket,
-		ObjectLockConfiguration: cfg,
-	})
-
-	return handleError(err)
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (s *S3Proxy) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
-	resp, err := s.client.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
-		Bucket: &bucket,
-	})
-	if err != nil {
-		return nil, handleError(err)
-	}
-
-	config := auth.BucketLockConfig{
-		Enabled:          resp.ObjectLockConfiguration.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
-		DefaultRetention: resp.ObjectLockConfiguration.Rule.DefaultRetention,
-	}
-
-	return json.Marshal(config)
+	return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
 }
 
 func (s *S3Proxy) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	ret, err := auth.ParseObjectLockRetentionOutput(retention)
-	if err != nil {
-		return err
-	}
-
-	_, err = s.client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
-		Bucket:                    &bucket,
-		Key:                       &object,
-		VersionId:                 &versionId,
-		Retention:                 ret,
-		BypassGovernanceRetention: &bypass,
-	})
-	return handleError(err)
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (s *S3Proxy) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
-	resp, err := s.client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{
-		Bucket:    &bucket,
-		Key:       &object,
-		VersionId: &versionId,
-	})
-	if err != nil {
-		return nil, handleError(err)
-	}
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 
-	return json.Marshal(resp.Retention)
 }
 
 func (s *S3Proxy) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
-	var st types.ObjectLockLegalHoldStatus
-	if status {
-		st = types.ObjectLockLegalHoldStatusOn
-	} else {
-		st = types.ObjectLockLegalHoldStatusOff
-	}
-
-	_, err := s.client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{
-		Bucket:    &bucket,
-		Key:       &object,
-		VersionId: &versionId,
-		LegalHold: &types.ObjectLockLegalHold{
-			Status: st,
-		},
-	})
-	return handleError(err)
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
-	resp, err := s.client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{
-		Bucket:    &bucket,
-		Key:       &object,
-		VersionId: &versionId,
-	})
-	if err != nil {
-		return nil, handleError(err)
-	}
-
-	status := resp.LegalHold.Status == types.ObjectLockLegalHoldStatusOn
-	return &status, nil
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
@@ -833,17 +1400,19 @@ func base64Decode(encoded string) ([]byte, error) {
 }
 
 func convertObjects(objs []types.Object) []s3response.Object {
-	result := make([]s3response.Object, len(objs))
+	result := make([]s3response.Object, 0, len(objs))
 
 	for _, obj := range objs {
 		result = append(result, s3response.Object{
-			ETag:          obj.ETag,
-			Key:           obj.Key,
-			LastModified:  obj.LastModified,
-			Owner:         obj.Owner,
-			Size:          obj.Size,
-			RestoreStatus: obj.RestoreStatus,
-			StorageClass:  obj.StorageClass,
+			ETag:              obj.ETag,
+			Key:               obj.Key,
+			LastModified:      obj.LastModified,
+			Owner:             obj.Owner,
+			Size:              obj.Size,
+			RestoreStatus:     obj.RestoreStatus,
+			StorageClass:      obj.StorageClass,
+			ChecksumAlgorithm: obj.ChecksumAlgorithm,
+			ChecksumType:      obj.ChecksumType,
 		})
 	}
 

backend/scoutfs/scoutfs.go

@@ -40,29 +40,21 @@ import (
 )
 
 type ScoutfsOpts struct {
-	ChownUID    bool
-	ChownGID    bool
-	GlacierMode bool
-	BucketLinks bool
-	NewDirPerm  fs.FileMode
+	ChownUID         bool
+	ChownGID         bool
+	GlacierMode      bool
+	BucketLinks      bool
+	NewDirPerm       fs.FileMode
+	DisableNoArchive bool
 }
 
 type ScoutFS struct {
-
-	// bucket/object metadata storage facility
-	meta meta.MetadataStorer
-
 	*posix.Posix
 	rootfd  *os.File
 	rootdir string
 
-	// euid/egid are the effective uid/gid of the running versitygw process
-	// used to determine if chowning is needed
-	euid int
-	egid int
-
-	// newDirPerm is the permissions to use when creating new directories
-	newDirPerm fs.FileMode
+	// bucket/object metadata storage facility
+	meta meta.MetadataStorer
 
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
@@ -79,6 +71,19 @@ type ScoutFS struct {
 	// when objects are uploaded
 	chownuid bool
 	chowngid bool
+
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
+
+	// newDirPerm is the permissions to use when creating new directories
+	newDirPerm fs.FileMode
+
+	// disableNoArchive is used to disable setting scoutam noarchive flag
+	// on mutlipart parts. This is enabled by default to prevent archive
+	// copies of temporary multipart parts.
+	disableNoArchive bool
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -157,6 +162,31 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
 	return uid, gid, needsChown
 }
 
+func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	out, err := s.Posix.UploadPart(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	if !s.disableNoArchive {
+		sum := sha256.Sum256([]byte(*input.Key))
+		partPath := filepath.Join(
+			*input.Bucket,                        // bucket
+			metaTmpMultipartDir,                  // temp multipart dir
+			fmt.Sprintf("%x", sum),               // hashed objname
+			*input.UploadId,                      // upload id
+			fmt.Sprintf("%v", *input.PartNumber), // part number
+		)
+
+		err = setNoArchive(partPath)
+		if err != nil {
+			return nil, fmt.Errorf("set noarchive: %w", err)
+		}
+	}
+
+	return out, err
+}
+
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
@@ -353,10 +383,10 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}
 
 	// cleanup tmp dirs
-	os.RemoveAll(upiddir)
+	os.RemoveAll(filepath.Join(bucket, upiddir))
 	// use Remove for objdir in case there are still other uploads
 	// for same object name outstanding
-	os.Remove(objdir)
+	os.Remove(filepath.Join(bucket, objdir))
 
 	return &s3.CompleteMultipartUploadOutput{
 		Bucket: &bucket,
@@ -491,7 +521,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	objPath := filepath.Join(bucket, object)
 
 	fi, err := os.Stat(objPath)
-	if errors.Is(err, fs.ErrNotExist) {
+	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 	if errors.Is(err, syscall.ENAMETOOLONG) {
@@ -615,7 +645,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 	objPath := filepath.Join(bucket, object)
 
 	fi, err := os.Stat(objPath)
-	if errors.Is(err, fs.ErrNotExist) {
+	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 	if errors.Is(err, syscall.ENAMETOOLONG) {
@@ -629,28 +659,20 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
+	objSize := fi.Size()
+	startOffset, length, isValid, err := backend.ParseGetObjectRange(objSize, acceptRange)
 	if err != nil {
 		return nil, err
 	}
 
-	objSize := fi.Size()
 	if fi.IsDir() {
 		// directory objects are always 0 len
 		objSize = 0
 		length = 0
 	}
 
-	if length == -1 {
-		length = fi.Size() - startOffset + 1
-	}
-
-	if startOffset+length > fi.Size() {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
-	}
-
 	var contentRange string
-	if acceptRange != "" {
+	if isValid {
 		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
 	}
 
@@ -697,7 +719,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 	tagCount := int32(len(tags))
 
 	return &s3.GetObjectOutput{
-		AcceptRanges:    &acceptRange,
+		AcceptRanges:    backend.GetPtrFromString("bytes"),
 		ContentLength:   &length,
 		ContentEncoding: &contentEncoding,
 		ContentType:     &contentType,
@@ -941,30 +963,6 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput)
 	return nil
 }
 
-func setStaging(objname string) error {
-	b, err := xattr.Get(objname, flagskey)
-	if err != nil && !isNoAttr(err) {
-		return err
-	}
-
-	var oldflags uint64
-	if !isNoAttr(err) {
-		err = json.Unmarshal(b, &oldflags)
-		if err != nil {
-			return err
-		}
-	}
-
-	newflags := oldflags | Staging
-
-	if newflags == oldflags {
-		// no flags change, just return
-		return nil
-	}
-
-	return fSetNewGlobalFlags(objname, newflags)
-}
-
 func isStaging(objname string) (bool, error) {
 	b, err := xattr.Get(objname, flagskey)
 	if err != nil && !isNoAttr(err) {
@@ -982,8 +980,28 @@ func isStaging(objname string) (bool, error) {
 	return flags&Staging == Staging, nil
 }
 
-func fSetNewGlobalFlags(objname string, flags uint64) error {
-	b, err := json.Marshal(&flags)
+func setFlag(objname string, flag uint64) error {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return err
+	}
+
+	var oldflags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &oldflags)
+		if err != nil {
+			return err
+		}
+	}
+
+	newflags := oldflags | flag
+
+	if newflags == oldflags {
+		// no flags change, just return
+		return nil
+	}
+
+	b, err = json.Marshal(&newflags)
 	if err != nil {
 		return err
 	}
@@ -991,6 +1009,14 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 	return xattr.Set(objname, flagskey, b)
 }
 
+func setStaging(objname string) error {
+	return setFlag(objname, Staging)
+}
+
+func setNoArchive(objname string) error {
+	return setFlag(objname, NoArchive)
+}
+
 func isNoAttr(err error) bool {
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {

backend/scoutfs/scoutfs_compat.go

@@ -52,14 +52,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:       p,
-		rootfd:      f,
-		rootdir:     rootdir,
-		meta:        metastore,
-		chownuid:    opts.ChownUID,
-		chowngid:    opts.ChownGID,
-		glaciermode: opts.GlacierMode,
-		newDirPerm:  opts.NewDirPerm,
+		Posix:            p,
+		rootfd:           f,
+		rootdir:          rootdir,
+		meta:             metastore,
+		chownuid:         opts.ChownUID,
+		chowngid:         opts.ChownGID,
+		glaciermode:      opts.GlacierMode,
+		newDirPerm:       opts.NewDirPerm,
+		disableNoArchive: opts.DisableNoArchive,
 	}, nil
 }
 
@@ -70,10 +71,10 @@ type tmpfile struct {
 	bucket     string
 	objname    string
 	size       int64
+	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
-	needsChown bool
 }
 
 var (

backend/walk.go

@@ -21,16 +21,17 @@ import (
 	"io/fs"
 	"sort"
 	"strings"
+	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3response"
 )
 
 type WalkResults struct {
-	NextMarker     string
 	CommonPrefixes []types.CommonPrefix
 	Objects        []s3response.Object
 	Truncated      bool
+	NextMarker     string
 }
 
 type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
@@ -87,6 +88,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		// so we can skip a directory without an early return
 		var skipflag error
 		if d.IsDir() {
+			fmt.Println("path: ", path)
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
 			// match this prefix. Make sure to append the / at the end of
@@ -107,13 +109,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
 				skipflag = fs.SkipDir
 			} else {
-				// TODO: can we do better here rather than a second readdir
-				// per directory?
-				ents, err := fs.ReadDir(fileSystem, path)
-				if err != nil {
-					return fmt.Errorf("readdir %q: %w", path, err)
-				}
-				if len(ents) == 0 && delimiter == "" {
+				if delimiter == "" {
 					dirobj, err := getObj(path+"/", d)
 					if err == ErrSkipObj {
 						return skipflag
@@ -126,6 +122,13 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 					return skipflag
 				}
 
+				// TODO: can we do better here rather than a second readdir
+				// per directory?
+				ents, err := fs.ReadDir(fileSystem, path)
+				if err != nil {
+					return fmt.Errorf("readdir %q: %w", path, err)
+				}
+
 				if len(ents) != 0 {
 					return skipflag
 				}
@@ -231,7 +234,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 	})
 	if err != nil {
 		// suppress file not found caused by user's prefix
-		if errors.Is(err, fs.ErrNotExist) {
+		if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
 			return WalkResults{}, nil
 		}
 		return WalkResults{}, err
@@ -268,18 +271,18 @@ func contains(a string, strs []string) bool {
 }
 
 type WalkVersioningResults struct {
-	NextMarker          string
-	NextVersionIdMarker string
 	CommonPrefixes      []types.CommonPrefix
 	ObjectVersions      []types.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
 	Truncated           bool
+	NextMarker          string
+	NextVersionIdMarker string
 }
 
 type ObjVersionFuncResult struct {
-	NextVersionIdMarker string
 	ObjectVersions      []types.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
+	NextVersionIdMarker string
 	Truncated           bool
 }
 

backend/walk_test.go

@@ -41,8 +41,8 @@ type testcase struct {
 	prefix    string
 	delimiter string
 	marker    string
-	expected  backend.WalkResults
 	maxObjs   int32
+	expected  backend.WalkResults
 }
 
 func getObj(path string, d fs.DirEntry) (s3response.Object, error) {

cmd/versitygw/main.go

@@ -74,6 +74,9 @@ var (
 	metricsService                           string
 	statsdServers                            string
 	dogstatsServers                          string
+	ipaHost, ipaVaultName                    string
+	ipaUser, ipaPassword                     string
+	ipaInsecure, ipaDebug                    bool
 )
 
 var (
@@ -206,6 +209,7 @@ func initFlags() []cli.Flag {
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
+			Value:       false,
 			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
@@ -506,6 +510,42 @@ func initFlags() []cli.Flag {
 			Aliases:     []string{"mds"},
 			Destination: &dogstatsServers,
 		},
+		&cli.StringFlag{
+			Name:        "ipa-host",
+			Usage:       "FreeIPA server url e.g. https://ipa.example.test",
+			EnvVars:     []string{"VGW_IPA_HOST"},
+			Destination: &ipaHost,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-vault-name",
+			Usage:       "A name of the user vault containing their secret",
+			EnvVars:     []string{"VGW_IPA_VAULT_NAME"},
+			Destination: &ipaVaultName,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-user",
+			Usage:       "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
+			EnvVars:     []string{"VGW_IPA_USER"},
+			Destination: &ipaUser,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-password",
+			Usage:       "Password of the user used to connect to FreeIPA.",
+			EnvVars:     []string{"VGW_IPA_PASSWORD"},
+			Destination: &ipaPassword,
+		},
+		&cli.BoolFlag{
+			Name:        "ipa-insecure",
+			Usage:       "Verify TLS certificate of FreeIPA server. Default is 'true'.",
+			EnvVars:     []string{"VGW_IPA_INSECURE"},
+			Destination: &ipaInsecure,
+		},
+		&cli.BoolFlag{
+			Name:        "ipa-debug",
+			Usage:       "FreeIPA IAM debug output",
+			EnvVars:     []string{"VGW_IPA_DEBUG"},
+			Destination: &ipaDebug,
+		},
 	}
 }
 
@@ -623,6 +663,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		CacheDisable:           iamCacheDisable,
 		CacheTTL:               iamCacheTTL,
 		CachePrune:             iamCachePrune,
+		IpaHost:                ipaHost,
+		IpaVaultName:           ipaVaultName,
+		IpaUser:                ipaUser,
+		IpaPassword:            ipaPassword,
+		IpaInsecure:            ipaInsecure,
+		IpaDebug:               ipaDebug,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)

cmd/versitygw/posix.go

@@ -29,6 +29,8 @@ var (
 	bucketlinks        bool
 	versioningDir      string
 	dirPerms           uint
+	sidecar            string
+	nometa             bool
 )
 
 func posixCommand() *cli.Command {
@@ -79,6 +81,18 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.StringFlag{
+				Name:        "sidecar",
+				Usage:       "use provided sidecar directory to store metadata",
+				EnvVars:     []string{"VGW_META_SIDECAR"},
+				Destination: &sidecar,
+			},
+			&cli.BoolFlag{
+				Name:        "nometa",
+				Usage:       "disable metadata storage",
+				EnvVars:     []string{"VGW_META_NONE"},
+				Destination: &nometa,
+			},
 		},
 	}
 }
@@ -89,24 +103,45 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	gwroot := (ctx.Args().Get(0))
-	err := meta.XattrMeta{}.Test(gwroot)
-	if err != nil {
-		return fmt.Errorf("posix xattr check: %v", err)
-	}
 
 	if dirPerms > math.MaxUint32 {
 		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
 	}
 
-	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
+	if nometa && sidecar != "" {
+		return fmt.Errorf("cannot use both nometa and sidecar metadata")
+	}
+
+	opts := posix.PosixOpts{
 		ChownUID:      chownuid,
 		ChownGID:      chowngid,
 		BucketLinks:   bucketlinks,
 		VersioningDir: versioningDir,
 		NewDirPerm:    fs.FileMode(dirPerms),
-	})
+	}
+
+	var ms meta.MetadataStorer
+	switch {
+	case sidecar != "":
+		sc, err := meta.NewSideCar(sidecar)
+		if err != nil {
+			return fmt.Errorf("failed to init sidecar metadata: %w", err)
+		}
+		ms = sc
+		opts.SideCarDir = sidecar
+	case nometa:
+		ms = meta.NoMeta{}
+	default:
+		ms = meta.XattrMeta{}
+		err := meta.XattrMeta{}.Test(gwroot)
+		if err != nil {
+			return fmt.Errorf("xattr check failed: %w", err)
+		}
+	}
+
+	be, err := posix.New(gwroot, ms, opts)
 	if err != nil {
-		return fmt.Errorf("init posix: %v", err)
+		return fmt.Errorf("failed to init posix backend: %w", err)
 	}
 
 	return runGateway(ctx.Context, be)

cmd/versitygw/scoutfs.go

@@ -24,7 +24,8 @@ import (
 )
 
 var (
-	glacier bool
+	glacier          bool
+	disableNoArchive bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -79,6 +80,12 @@ move interfaces as well as support for tiered filesystems.`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.BoolFlag{
+				Name:        "disable-noarchive",
+				Usage:       "disable setting noarchive for multipart part uploads",
+				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
+				Destination: &disableNoArchive,
+			},
 		},
 	}
 }
@@ -98,6 +105,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
 	opts.NewDirPerm = fs.FileMode(dirPerms)
+	opts.DisableNoArchive = disableNoArchive
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

cmd/versitygw/test.go

@@ -38,6 +38,7 @@ var (
 	checksumDisable   bool
 	versioningEnabled bool
 	azureTests        bool
+	tlsStatus         bool
 )
 
 func testCommand() *cli.Command {
@@ -79,6 +80,12 @@ func initTestFlags() []cli.Flag {
 			Aliases:     []string{"d"},
 			Destination: &debug,
 		},
+		&cli.BoolFlag{
+			Name:        "allow-insecure",
+			Usage:       "skip tls verification",
+			Aliases:     []string{"ai"},
+			Destination: &tlsStatus,
+		},
 	}
 }
 
@@ -211,6 +218,7 @@ func initTestCommands() []*cli.Command {
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
 					integration.WithPartSize(partSize),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -271,6 +279,7 @@ func initTestCommands() []*cli.Command {
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -296,6 +305,7 @@ func getAction(tf testFunc) func(*cli.Context) error {
 			integration.WithSecret(awsSecret),
 			integration.WithRegion(region),
 			integration.WithEndpoint(endpoint),
+			integration.WithTLSStatus(tlsStatus),
 		}
 		if debug {
 			opts = append(opts, integration.WithDebug())
@@ -333,6 +343,7 @@ func extractIntTests() (commands []*cli.Command) {
 					integration.WithSecret(awsSecret),
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())

go.mod

@@ -1,42 +1,44 @@
 module github.com/versity/versitygw
 
-go 1.21.0
+go 1.23.0
+
+toolchain go1.23.6
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
-	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.32.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
-	github.com/aws/smithy-go v1.22.0
-	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/gofiber/fiber/v2 v2.52.5
-	github.com/google/go-cmp v0.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
+	github.com/DataDog/datadog-go/v5 v5.6.0
+	github.com/aws/aws-sdk-go-v2 v1.36.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1
+	github.com/aws/smithy-go v1.22.3
+	github.com/go-ldap/ldap/v3 v3.4.10
+	github.com/gofiber/fiber/v2 v2.52.6
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.37.0
+	github.com/nats-io/nats.go v1.39.1
 	github.com/oklog/ulid/v2 v2.1.0
 	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.4
-	github.com/urfave/cli/v2 v2.27.5
-	github.com/valyala/fasthttp v1.57.0
+	github.com/urfave/cli/v2 v2.27.6
+	github.com/valyala/fasthttp v1.59.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.26.0
+	golang.org/x/sync v0.12.0
+	golang.org/x/sys v0.31.0
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -45,38 +47,37 @@ require (
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nkeys v0.4.10 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.28.1
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.42
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.9
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.62
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/tcplisten v1.0.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 )

go.sum

@@ -1,23 +1,23 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2/go.mod h1:SqINnQ9lVVdRlyC8cd1lCI0SdX4n2paeABd2K8ggfnE=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
-github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.1 h1:8BKxhZZLX/WosEeoCvWysmKUscfa9v8LIPEEU0JjE2o=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
+github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
@@ -25,48 +25,48 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
-github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
-github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw=
-github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35 h1:ihPPdcCVSN0IvBByXwqVp28/l4VosBZ6sDulcvU2J7w=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35/go.mod h1:JkgEhs3SVF51Dj3m1Bj+yL8IznpxzkwlA3jLg3x7Kls=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58=
-github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE=
-github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
-github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
+github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
+github.com/aws/aws-sdk-go-v2/config v1.29.9 h1:Kg+fAYNaJeGXp1vmjtidss8O2uXIsXwaRqsQJKXVr+0=
+github.com/aws/aws-sdk-go-v2/config v1.29.9/go.mod h1:oU3jj2O53kgOU4TXq/yipt6ryiooYjlkqqVaZk7gY/U=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBajbBWGqjNTCa79ocU=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65 h1:03zF9oWZyXvw08Say761JGpE9PbeGPd4FAmdpgDAm/I=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65/go.mod h1:hBobvLKm46Igpcw6tkq9hFUmU14iAOrC5KL6EyYYckA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 h1:t/gZFyrijKuSU0elA5kRngP/oU3mc0I+Dvp8HwRE4c0=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1 h1:1M0gSbyP6q06gl3384wpoKPaH9G16NPqZFieEhLboSU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1/go.mod h1:4qzsZSzB/KiX2EzDjs9D7A8rI/WGJxZceVJIHqtJjIU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 h1:8JdC7Gr9NROg1Rusk25IcZeTO59zLxsKgE0gkh5O6h0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.1/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 h1:KwuLovgQPcdjNMfFt9OhUd9a2OwcOKhxfvF4glTzLuA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
+github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
-github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -74,18 +74,18 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
 github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
-github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
-github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
+github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
+github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
+github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
@@ -120,31 +120,30 @@ github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJk
 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
-github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
-github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
-github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nats.go v1.39.1 h1:oTkfKBmz7W047vRxV762M67ZdXeOtUgvbBaNoQ+3PPk=
+github.com/nats-io/nats.go v1.39.1/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM=
+github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc=
+github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
 github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
 github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
-github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -152,8 +151,8 @@ github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
 github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -175,16 +174,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
-github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
+github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
-github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE=
-github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
-github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/valyala/fasthttp v1.59.0 h1:Qu0qYHfXvPk1mSLNqcFtEk6DpxgA26hy6bmydotDpRI=
+github.com/valyala/fasthttp v1.59.0/go.mod h1:GTxNb9Bc6r2a9D0TWNSPwDz78UxnTGBViY3xZNEqyYU=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -203,14 +200,19 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -220,17 +222,23 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -242,23 +250,27 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -267,15 +279,19 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
-golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

metrics/metrics.go

@@ -43,14 +43,13 @@ type Tag struct {
 
 // Manager is a manager of metrics plugins
 type Manager struct {
+	wg  sync.WaitGroup
 	ctx context.Context
 
-	addDataChan chan datapoint
-
 	config Config
 
-	publishers []publisher
-	wg         sync.WaitGroup
+	publishers  []publisher
+	addDataChan chan datapoint
 }
 
 type Config struct {
@@ -221,6 +220,6 @@ func (m *Manager) addForwarder(addChan <-chan datapoint) {
 
 type datapoint struct {
 	key   string
-	tags  []Tag
 	value int64
+	tags  []Tag
 }

runtests.sh

@@ -10,6 +10,14 @@ mkdir /tmp/versioning.covdata
 rm -rf /tmp/versioningdir
 mkdir /tmp/versioningdir
 
+# setup tls certificate and key
+ECHO "Generating TLS certificate and key in the cert.pem and key.pem files"
+
+openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
+openssl req -new -x509 -key key.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+
+ECHO "Running the sdk test over http"
 # run server in background not versioning-enabled
 # port: 7070(default)
 GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
@@ -17,7 +25,7 @@ GW_PID=$!
 
 sleep 1
 
-# check if versioning-enabled gateway process is still running
+# check if gateway process is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -45,9 +53,48 @@ fi
 
 kill $GW_PID
 
+ECHO "Running the sdk test over https"
+
+# run server in background with TLS certificate
+# port: 7071(default)
+GOCOVERDIR=/tmp/https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7071 -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
+GW_HTTPS_PID=$!
+
+sleep 1
+
+# check if https gateway process is still running
+if ! kill -0 $GW_HTTPS_PID; then
+	echo "server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow; then
+	echo "full flow tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 posix; then
+	echo "posix tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+# iam tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 iam; then
+	echo "iam tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+
+kill $GW_HTTPS_PID
+
+
+ECHO "Running the sdk test over http against the versioning-enabled gateway"
 # run server in background versioning-enabled
-# port: 7071
-GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7071 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+# port: 7072
+GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7072 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
 GW_VS_PID=$!
 
 # wait a second for server to start up
@@ -61,13 +108,13 @@ fi
 
 # run tests
 # full flow tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 full-flow -vs; then
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs; then
 	echo "versioning-enabled full-flow tests failed"
 	kill $GW_VS_PID
 	exit 1
 fi
 # posix tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 posix -vs; then
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 posix -vs; then
 	echo "versiongin-enabled posix tests failed"
 	kill $GW_VS_PID
 	exit 1
@@ -76,6 +123,38 @@ fi
 # kill off server
 kill $GW_VS_PID
 
+ECHO "Running the sdk test over https against the versioning-enabled gateway"
+# run server in background versioning-enabled
+# port: 7073
+GOCOVERDIR=/tmp/versioning.https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7073 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+GW_VS_HTTPS_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if versioning-enabled gateway process is still running
+if ! kill -0 $GW_VS_HTTPS_PID; then
+	echo "versioning-enabled server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs; then
+	echo "versioning-enabled full-flow tests failed"
+	kill $GW_VS_HTTPS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 posix -vs; then
+	echo "versiongin-enabled posix tests failed"
+	kill $GW_VS_HTTPS_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_VS_HTTPS_PID
+
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-server.go

@@ -26,11 +26,11 @@ import (
 )
 
 type S3AdminServer struct {
-	backend backend.Backend
 	app     *fiber.App
+	backend backend.Backend
 	router  *S3AdminRouter
-	cert    *tls.Certificate
 	port    string
+	cert    *tls.Certificate
 }
 
 func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {

s3api/controllers/admin.go

@@ -167,7 +167,7 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		Owner: owner,
 		Grantees: []auth.Grantee{
 			{
-				Permission: types.PermissionFullControl,
+				Permission: auth.PermissionFullControl,
 				Access:     owner,
 				Type:       types.TypeCanonicalUser,
 			},

s3api/controllers/admin_test.go

@@ -64,11 +64,11 @@ func TestAdminController_CreateUser(t *testing.T) {
 	`
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-create-user-malformed-body",
@@ -149,11 +149,11 @@ func TestAdminController_UpdateUser(t *testing.T) {
 	`
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-update-user-success",
@@ -223,11 +223,11 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	app.Patch("/delete-user", adminController.DeleteUser)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-delete-user-success",
@@ -280,11 +280,11 @@ func TestAdminController_ListUsers(t *testing.T) {
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-list-users-iam-error",
@@ -361,11 +361,11 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -424,11 +424,11 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "List-buckets-success",

s3api/controllers/backend_moq_test.go

@@ -170,10 +170,10 @@ var _ backend.Backend = &BackendMock{}
 //			StringFunc: func() string {
 //				panic("mock out the String method")
 //			},
-//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 //				panic("mock out the UploadPart method")
 //			},
-//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 //				panic("mock out the UploadPartCopy method")
 //			},
 //		}
@@ -331,10 +331,10 @@ type BackendMock struct {
 	StringFunc func() string
 
 	// UploadPartFunc mocks the UploadPart method.
-	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error)
+	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error)
 
 	// UploadPartCopyFunc mocks the UploadPartCopy method.
-	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// calls tracks calls to the methods.
 	calls struct {
@@ -2620,7 +2620,7 @@ func (mock *BackendMock) StringCalls() []struct {
 }
 
 // UploadPart calls UploadPartFunc.
-func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 	if mock.UploadPartFunc == nil {
 		panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called")
 	}
@@ -2656,7 +2656,7 @@ func (mock *BackendMock) UploadPartCalls() []struct {
 }
 
 // UploadPartCopy calls UploadPartCopyFunc.
-func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 	if mock.UploadPartCopyFunc == nil {
 		panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called")
 	}

s3api/controllers/base.go

@@ -74,24 +74,28 @@ func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
 	maxBucketsStr := ctx.Query("max-buckets")
 	acct := ctx.Locals("account").(auth.Account)
 
-	maxBuckets, err := utils.ParseUint(maxBucketsStr)
-	if err != nil || maxBuckets > 10000 {
-		if c.debug {
-			log.Printf("error parsing max-buckets %q: %v\n", maxBucketsStr, err)
+	var maxBuckets int32 = 10000
+	if maxBucketsStr != "" {
+		maxBucketsParsed, err := strconv.ParseInt(maxBucketsStr, 10, 32)
+		if err != nil || maxBucketsParsed < 0 || maxBucketsParsed > 10000 {
+			if c.debug {
+				log.Printf("error parsing max-buckets %q: %v\n", maxBucketsStr, err)
+			}
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
+				&MetaOpts{
+					Logger:     c.logger,
+					MetricsMng: c.mm,
+					Action:     metrics.ActionListAllMyBuckets,
+				})
 		}
-		return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
-			&MetaOpts{
-				Logger:     c.logger,
-				MetricsMng: c.mm,
-				Action:     metrics.ActionListAllMyBuckets,
-			})
+		maxBuckets = int32(maxBucketsParsed)
 	}
 
 	res, err := c.be.ListBuckets(ctx.Context(),
 		s3response.ListBucketsInput{
 			Owner:             acct.Access,
 			IsAdmin:           acct.Role == auth.RoleAdmin,
-			MaxBuckets:        maxBuckets,
+			MaxBuckets:        int32(maxBuckets),
 			ContinuationToken: cToken,
 			Prefix:            prefix,
 		})
@@ -126,7 +130,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -175,7 +179,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -217,7 +221,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -282,7 +286,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -319,7 +323,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionReadAcp,
+			AclPermission: auth.PermissionReadAcp,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -352,7 +356,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -468,7 +472,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 		Readonly:      c.readonly,
 		Acl:           parsedAcl,
-		AclPermission: types.PermissionRead,
+		AclPermission: auth.PermissionRead,
 		IsRoot:        isRoot,
 		Acc:           acct,
 		Bucket:        bucket,
@@ -485,12 +489,27 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
+	if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
+		if c.debug {
+			log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode)
+		}
+		return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"),
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionGetObject,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	ctx.Locals("logResBody", false)
 	res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{
-		Bucket:    &bucket,
-		Key:       &key,
-		Range:     &acceptRange,
-		VersionId: &versionId,
+		Bucket:       &bucket,
+		Key:          &key,
+		Range:        &acceptRange,
+		VersionId:    &versionId,
+		ChecksumMode: checksumMode,
 	})
 	if err != nil {
 		if res != nil {
@@ -568,6 +587,42 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			Value: string(res.StorageClass),
 		})
 	}
+	if res.ChecksumCRC32 != nil {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc32",
+			Value: *res.ChecksumCRC32,
+		})
+	}
+	if res.ChecksumCRC32C != nil {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc32c",
+			Value: *res.ChecksumCRC32C,
+		})
+	}
+	if res.ChecksumSHA1 != nil {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-sha1",
+			Value: *res.ChecksumSHA1,
+		})
+	}
+	if res.ChecksumSHA256 != nil {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-sha256",
+			Value: *res.ChecksumSHA256,
+		})
+	}
+	if res.ChecksumCRC64NVME != nil {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc64nvme",
+			Value: *res.ChecksumCRC64NVME,
+		})
+	}
+	if res.ChecksumType != "" {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-type",
+			Value: string(res.ChecksumType),
+		})
+	}
 
 	// Set x-amz-meta-... headers
 	utils.SetMetaHeaders(ctx, res.Metadata)
@@ -589,7 +644,12 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if res.Body != nil {
-		ctx.Response().SetBodyStream(res.Body, int(getint64(res.ContentLength)))
+		// -1 will stream response body until EOF if content length not set
+		contentLen := -1
+		if res.ContentLength != nil {
+			contentLen = int(*res.ContentLength)
+		}
+		utils.StreamResponseBody(ctx, res.Body, contentLen)
 	}
 
 	return SendResponse(ctx, nil,
@@ -637,7 +697,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -685,7 +745,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -722,7 +782,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -762,7 +822,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -792,7 +852,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -845,7 +905,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -886,7 +946,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionReadAcp,
+			AclPermission: auth.PermissionReadAcp,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -912,7 +972,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		res, err := auth.ParseACLOutput(data)
+		res, err := auth.ParseACLOutput(data, parsedAcl.Owner)
 		return SendXMLResponse(ctx, res, err,
 			&MetaOpts{
 				Logger:      c.logger,
@@ -926,7 +986,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -977,7 +1037,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1027,7 +1087,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 		Readonly:      c.readonly,
 		Acl:           parsedAcl,
-		AclPermission: types.PermissionRead,
+		AclPermission: auth.PermissionRead,
 		IsRoot:        isRoot,
 		Acc:           acct,
 		Bucket:        bucket,
@@ -1127,7 +1187,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1180,7 +1240,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1210,7 +1270,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1272,7 +1332,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1313,7 +1373,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1384,7 +1444,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWriteAcp,
+				AclPermission: auth.PermissionWriteAcp,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -1415,13 +1475,12 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 					})
 			}
 
-			if accessControlPolicy.Owner == nil ||
-				accessControlPolicy.Owner.ID == nil ||
-				*accessControlPolicy.Owner.ID == "" {
+			err = accessControlPolicy.Validate()
+			if err != nil {
 				if c.debug {
-					log.Println("empty access control policy owner")
+					log.Printf("invalid access control policy: %v\n", err)
 				}
-				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedACL),
+				return SendResponse(ctx, err,
 					&MetaOpts{
 						Logger:      c.logger,
 						Action:      metrics.ActionPutBucketAcl,
@@ -1733,7 +1792,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1766,7 +1825,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1839,7 +1898,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -1903,7 +1962,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource,
 			auth.AccessOptions{
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -1954,7 +2013,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			if c.debug {
 				log.Printf("invalid part number: %d", partNumber)
 			}
-			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart),
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber),
 				&MetaOpts{
 					Logger:      c.logger,
 					MetricsMng:  c.mm,
@@ -1967,7 +2026,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -1999,6 +2058,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				})
 		}
 
+		algorithm, checksums, err := utils.ParseChecksumHeaders(ctx)
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutObject,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
 		var body io.Reader
 		bodyi := ctx.Locals("body-reader")
 		if bodyi != nil {
@@ -2008,16 +2078,59 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		}
 
 		ctx.Locals("logReqBody", false)
-		etag, err := c.be.UploadPart(ctx.Context(),
+		res, err := c.be.UploadPart(ctx.Context(),
 			&s3.UploadPartInput{
-				Bucket:        &bucket,
-				Key:           &keyStart,
-				UploadId:      &uploadId,
-				PartNumber:    &partNumber,
-				ContentLength: &contentLength,
-				Body:          body,
+				Bucket:            &bucket,
+				Key:               &keyStart,
+				UploadId:          &uploadId,
+				PartNumber:        &partNumber,
+				ContentLength:     &contentLength,
+				Body:              body,
+				ChecksumAlgorithm: algorithm,
+				ChecksumCRC32:     backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
+				ChecksumCRC32C:    backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
+				ChecksumSHA1:      backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
+				ChecksumSHA256:    backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
+				ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
 			})
-		ctx.Response().Header.Set("Etag", etag)
+		if err == nil {
+			headers := []utils.CustomHeader{}
+			if res.ETag != nil {
+				headers = append(headers, utils.CustomHeader{
+					Key:   "ETag",
+					Value: *res.ETag,
+				})
+			}
+			switch {
+			case res.ChecksumCRC32 != nil:
+				headers = append(headers, utils.CustomHeader{
+					Key:   "x-amz-checksum-crc32",
+					Value: *res.ChecksumCRC32,
+				})
+			case res.ChecksumCRC32C != nil:
+				headers = append(headers, utils.CustomHeader{
+					Key:   "x-amz-checksum-crc32c",
+					Value: *res.ChecksumCRC32C,
+				})
+			case res.ChecksumCRC64NVME != nil:
+				headers = append(headers, utils.CustomHeader{
+					Key:   "x-amz-checksum-crc64nvme",
+					Value: *res.ChecksumCRC64NVME,
+				})
+			case res.ChecksumSHA1 != nil:
+				headers = append(headers, utils.CustomHeader{
+					Key:   "x-amz-checksum-sha1",
+					Value: *res.ChecksumSHA1,
+				})
+			case res.ChecksumSHA256 != nil:
+				headers = append(headers, utils.CustomHeader{
+					Key:   "x-amz-checksum-sha256",
+					Value: *res.ChecksumSHA256,
+				})
+			}
+
+			utils.SetResponseHeaders(ctx, headers)
+		}
 		return SendResponse(ctx, err,
 			&MetaOpts{
 				Logger:        c.logger,
@@ -2073,7 +2186,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 						ID:   &grt.Grantee.ID,
 						Type: grt.Grantee.Type,
 					},
-					Permission: grt.Permission,
+					Permission: types.Permission(grt.Permission),
 				})
 			}
 
@@ -2174,7 +2287,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource,
 			auth.AccessOptions{
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -2248,6 +2361,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			metaDirective = types.MetadataDirectiveReplace
 		}
 
+		checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
+		err = utils.IsChecksumAlgorithmValid(checksumAlgorithm)
+		if err != nil {
+			if c.debug {
+				log.Printf("invalid checksum algorithm: %v", checksumAlgorithm)
+			}
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionCopyObject,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
 		res, err := c.be.CopyObject(ctx.Context(),
 			&s3.CopyObjectInput{
 				Bucket:                      &bucket,
@@ -2261,6 +2389,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				Metadata:                    metadata,
 				MetadataDirective:           metaDirective,
 				StorageClass:                types.StorageClass(storageClass),
+				ChecksumAlgorithm:           checksumAlgorithm,
 			})
 		if err == nil {
 			hdrs := []utils.CustomHeader{}
@@ -2306,7 +2435,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -2360,6 +2489,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	algorithm, checksums, err := utils.ParseChecksumHeaders(ctx)
+	if err != nil {
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionPutObject,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	var body io.Reader
 	bodyi := ctx.Locals("body-reader")
 	if bodyi != nil {
@@ -2382,6 +2522,12 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			ObjectLockRetainUntilDate: &objLock.RetainUntilDate,
 			ObjectLockMode:            objLock.ObjectLockMode,
 			ObjectLockLegalHoldStatus: objLock.LegalHoldStatus,
+			ChecksumAlgorithm:         algorithm,
+			ChecksumCRC32:             backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
+			ChecksumCRC32C:            backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
+			ChecksumSHA1:              backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
+			ChecksumSHA256:            backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
+			ChecksumCRC64NVME:         backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
 		})
 	if err != nil {
 		return SendResponse(ctx, err,
@@ -2409,6 +2555,39 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			Value: res.VersionID,
 		})
 	}
+	switch {
+	case res.ChecksumCRC32 != nil:
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc32",
+			Value: *res.ChecksumCRC32,
+		})
+	case res.ChecksumCRC32C != nil:
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc32c",
+			Value: *res.ChecksumCRC32C,
+		})
+	case res.ChecksumCRC64NVME != nil:
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc64nvme",
+			Value: *res.ChecksumCRC64NVME,
+		})
+	case res.ChecksumSHA1 != nil:
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-sha1",
+			Value: *res.ChecksumSHA1,
+		})
+	case res.ChecksumSHA256 != nil:
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-sha256",
+			Value: *res.ChecksumSHA256,
+		})
+	}
+	if res.ChecksumType != "" {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-checksum-type",
+			Value: string(res.ChecksumType),
+		})
+	}
 
 	utils.SetResponseHeaders(ctx, hdrs)
 
@@ -2437,7 +2616,7 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -2469,7 +2648,7 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -2501,7 +2680,7 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -2532,7 +2711,7 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 		auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -2585,7 +2764,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 		auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -2658,7 +2837,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -2696,7 +2875,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -2737,7 +2916,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 		auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -2826,7 +3005,7 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
 		auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -2909,7 +3088,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 		auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionRead,
+			AclPermission: auth.PermissionRead,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -2926,12 +3105,27 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 			})
 	}
 
+	checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
+	if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
+		if c.debug {
+			log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode)
+		}
+		return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"),
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionHeadObject,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	res, err := c.be.HeadObject(ctx.Context(),
 		&s3.HeadObjectInput{
-			Bucket:     &bucket,
-			Key:        &key,
-			PartNumber: partNumber,
-			VersionId:  &versionId,
+			Bucket:       &bucket,
+			Key:          &key,
+			PartNumber:   partNumber,
+			VersionId:    &versionId,
+			ChecksumMode: checksumMode,
 		})
 	if err != nil {
 		if res != nil {
@@ -3014,6 +3208,39 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 			Value: string(res.StorageClass),
 		})
 	}
+	switch {
+	case res.ChecksumCRC32 != nil:
+		headers = append(headers, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc32",
+			Value: *res.ChecksumCRC32,
+		})
+	case res.ChecksumCRC32C != nil:
+		headers = append(headers, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc32c",
+			Value: *res.ChecksumCRC32C,
+		})
+	case res.ChecksumCRC64NVME != nil:
+		headers = append(headers, utils.CustomHeader{
+			Key:   "x-amz-checksum-crc64nvme",
+			Value: *res.ChecksumCRC64NVME,
+		})
+	case res.ChecksumSHA1 != nil:
+		headers = append(headers, utils.CustomHeader{
+			Key:   "x-amz-checksum-sha1",
+			Value: *res.ChecksumSHA1,
+		})
+	case res.ChecksumSHA256 != nil:
+		headers = append(headers, utils.CustomHeader{
+			Key:   "x-amz-checksum-sha256",
+			Value: *res.ChecksumSHA256,
+		})
+	}
+	if res.ChecksumType != "" {
+		headers = append(headers, utils.CustomHeader{
+			Key:   "x-amz-checksum-type",
+			Value: string(res.ChecksumType),
+		})
+	}
 
 	contentType := getstring(res.ContentType)
 	if contentType == "" {
@@ -3080,7 +3307,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -3135,7 +3362,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionRead,
+				AclPermission: auth.PermissionRead,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -3189,11 +3416,55 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
+		if len(data.Parts) == 0 {
+			if c.debug {
+				log.Println("empty parts provided for complete multipart upload")
+			}
+			return SendXMLResponse(ctx, nil,
+				s3err.GetAPIError(s3err.ErrEmptyParts),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionCompleteMultipartUpload,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		var mpuObjectSize *int64
+		mpuObjSizeHdr := ctx.Get("X-Amz-Mp-Object-Size")
+		if mpuObjSizeHdr != "" {
+			val, err := strconv.ParseInt(mpuObjSizeHdr, 10, 64)
+			//TODO: Not sure if invalid request should be returned
+			if err != nil {
+				return SendXMLResponse(ctx, nil,
+					s3err.GetAPIError(s3err.ErrInvalidRequest),
+					&MetaOpts{
+						Logger:      c.logger,
+						MetricsMng:  c.mm,
+						Action:      metrics.ActionCompleteMultipartUpload,
+						BucketOwner: parsedAcl.Owner,
+					})
+			}
+
+			if val < 0 {
+				return SendXMLResponse(ctx, nil,
+					s3err.GetInvalidMpObjectSizeErr(val),
+					&MetaOpts{
+						Logger:      c.logger,
+						MetricsMng:  c.mm,
+						Action:      metrics.ActionCompleteMultipartUpload,
+						BucketOwner: parsedAcl.Owner,
+					})
+			}
+
+			mpuObjectSize = &val
+		}
+
 		err = auth.VerifyAccess(ctx.Context(), c.be,
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
-				AclPermission: types.PermissionWrite,
+				AclPermission: auth.PermissionWrite,
 				IsRoot:        isRoot,
 				Acc:           acct,
 				Bucket:        bucket,
@@ -3210,6 +3481,35 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
+		_, checksums, err := utils.ParseChecksumHeaders(ctx)
+		if err != nil {
+			if c.debug {
+				log.Printf("err parsing checksum headers: %v", err)
+			}
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionCompleteMultipartUpload,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
+		err = utils.IsChecksumTypeValid(checksumType)
+		if err != nil {
+			if c.debug {
+				log.Printf("invalid checksum type: %v", err)
+			}
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionCompleteMultipartUpload,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
 		res, err := c.be.CompleteMultipartUpload(ctx.Context(),
 			&s3.CompleteMultipartUploadInput{
 				Bucket:   &bucket,
@@ -3218,6 +3518,13 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				MultipartUpload: &types.CompletedMultipartUpload{
 					Parts: data.Parts,
 				},
+				MpuObjectSize:     mpuObjectSize,
+				ChecksumCRC32:     backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
+				ChecksumCRC32C:    backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
+				ChecksumSHA1:      backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
+				ChecksumSHA256:    backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
+				ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
+				ChecksumType:      checksumType,
 			})
 		if err == nil {
 			if getstring(res.VersionId) != "" {
@@ -3253,7 +3560,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 		auth.AccessOptions{
 			Readonly:      c.readonly,
 			Acl:           parsedAcl,
-			AclPermission: types.PermissionWrite,
+			AclPermission: auth.PermissionWrite,
 			IsRoot:        isRoot,
 			Acc:           acct,
 			Bucket:        bucket,
@@ -3283,6 +3590,20 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 
 	metadata := utils.GetUserMetaData(&ctx.Request().Header)
 
+	checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx)
+	if err != nil {
+		if c.debug {
+			log.Printf("err parsing checksum headers: %v", err)
+		}
+		return SendXMLResponse(ctx, nil, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionCreateMultipartUpload,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	res, err := c.be.CreateMultipartUpload(ctx.Context(),
 		&s3.CreateMultipartUploadInput{
 			Bucket:                    &bucket,
@@ -3294,7 +3615,19 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			ObjectLockMode:            objLockState.ObjectLockMode,
 			ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,
 			Metadata:                  metadata,
+			ChecksumAlgorithm:         checksumAlgorithm,
+			ChecksumType:              checksumType,
 		})
+	if err == nil {
+		if checksumAlgorithm != "" {
+			utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+				{
+					Key:   "x-amz-checksum-algorithm",
+					Value: string(checksumAlgorithm),
+				},
+			})
+		}
+	}
 	return SendXMLResponse(ctx, res, err,
 		&MetaOpts{
 			Logger:      c.logger,
@@ -3308,14 +3641,14 @@ type MetaOpts struct {
 	Logger        s3log.AuditLogger
 	EvSender      s3event.S3EventSender
 	MetricsMng    *metrics.Manager
-	ObjectETag    *string
-	VersionId     *string
+	ContentLength int64
 	Action        string
 	BucketOwner   string
-	EventName     s3event.EventType
-	ContentLength int64
 	ObjectSize    int64
 	ObjectCount   int64
+	EventName     s3event.EventType
+	ObjectETag    *string
+	VersionId     *string
 	Status        int
 }
 

s3api/controllers/base_test.go

@@ -123,11 +123,11 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
 
 	tests := []struct {
+		name       string
 		args       args
 		app        *fiber.App
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "List-bucket-method-not-allowed",
@@ -232,12 +232,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
+	invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
+	invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Get-actions-get-tags-success",
@@ -329,6 +332,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Get-actions-get-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invalidChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Get-actions-get-object-success",
 			app:  app,
@@ -435,11 +447,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Get-bucket-tagging-non-existing-bucket",
@@ -728,11 +740,11 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Put-bucket-tagging-invalid-body",
@@ -941,12 +953,12 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	</Tagging>
 	`
 
-	retentionBody := `
-	<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-		<Mode>GOVERNANCE</Mode>
-		<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
-	</Retention>
-	`
+	//retentionBody := `
+	//<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	//	<Mode>GOVERNANCE</Mode>
+	//	<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
+	//</Retention>
+	//`
 
 	legalHoldBody := `
 	<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
@@ -971,14 +983,14 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
 				return s3response.PutObjectOutput{}, nil
 			},
-			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
-				return "hello", nil
+			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+				return &s3.UploadPartOutput{}, nil
 			},
 			PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
 				return nil
 			},
-			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-				return s3response.CopyObjectResult{}, nil
+			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+				return s3response.CopyPartResult{}, nil
 			},
 			PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
 				return nil
@@ -1012,6 +1024,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
 
+	// CopyObject invalid checksum algorithm
+	cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	// PutObjectAcl success
 	aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	aclReq.Header.Set("X-Amz-Acl", "private")
@@ -1033,12 +1050,46 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	// PutObject invalid checksum algorithm
+	invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
+	// PutObject invalid base64 checksum
+	invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
+
+	// PutObject invalid crc32
+	invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
+
+	// PutObject invalid crc32c
+	invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
+
+	// PutObject invalid sha1
+	invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
+
+	// PutObject invalid sha256
+	invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
+
+	// PutObject multiple checksum headers
+	mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
+	// PutObject checksum algorithm and header mismatch
+	checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Put-object-part-error-case",
@@ -1076,15 +1127,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "put-object-retention-success",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
+		//{
+		//	name: "put-object-retention-success",
+		//	app:  app,
+		//	args: args{
+		//		req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
+		//	},
+		//	wantErr:    false,
+		//	statusCode: 200,
+		//},
 		{
 			name: "put-legal-hold-invalid-request",
 			app:  app,
@@ -1175,6 +1226,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Copy-object-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: cpyInvChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Copy-object-success",
 			app:  app,
@@ -1243,11 +1303,11 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app.Delete("/:bucket", s3ApiController.DeleteBucket)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Delete-bucket-success",
@@ -1334,11 +1394,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	request.Header.Set("Content-Type", "application/xml")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Delete-Objects-success",
@@ -1432,11 +1492,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Abort-multipart-upload-success",
@@ -1541,11 +1601,11 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Head-bucket-success",
@@ -1642,12 +1702,15 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	})
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
+	invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
+	invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Head-object-success",
@@ -1658,6 +1721,15 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Head-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Head-object-error",
 			app:  appErr,
@@ -1713,6 +1785,19 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 		</SelectObjectContentRequest>
 	`
 
+	completMpBody := `
+		<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Part>
+				<ETag>etag</ETag>
+				<PartNumber>1</PartNumber>
+			</Part>
+		</CompleteMultipartUpload>
+	`
+
+	completMpEmptyBody := `
+		<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></CompleteMultipartUpload>
+	`
+
 	app.Use(func(ctx *fiber.Ctx) error {
 		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
@@ -1722,12 +1807,15 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	})
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
+	invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Restore-object-success",
@@ -1765,15 +1853,33 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
+		{
+			name: "Complete-multipart-upload-empty-parts",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpEmptyBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Complete-multipart-upload-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(`<root><key>body</key></root>`)),
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpBody)),
 			},
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Create-multipart-upload-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: invChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Create-multipart-upload-success",
 			app:  app,
@@ -1808,10 +1914,10 @@ func Test_XMLresponse(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		args       args
 		name       string
-		statusCode int
+		args       args
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Internal-server-error",
@@ -1883,10 +1989,10 @@ func Test_response(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		args       args
 		name       string
-		statusCode int
+		args       args
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Internal-server-error",

s3api/middlewares/acl-parser.go

@@ -74,6 +74,11 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 		}
 
+		// if owner is not set, set default owner to root account
+		if parsedAcl.Owner == "" {
+			parsedAcl.Owner = ctx.Locals("rootAccess").(string)
+		}
+
 		ctx.Locals("parsedAcl", parsedAcl)
 		return ctx.Next()
 	}

s3api/middlewares/authentication.go

@@ -76,6 +76,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
+		ctx.Locals("rootAccess", root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
@@ -108,6 +109,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return sendResponse(ctx, err, logger, mm)
 		}
 
+		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if utils.IsBigDataAction(ctx) {
 			// for streaming PUT actions, authorization is deferred
 			// until end of stream due to need to get length and
@@ -115,10 +117,24 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
 				return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
 			})
+
+			// wrap the io.Reader with ChunkReader if x-amz-content-sha256
+			// provide chunk encoding value
+			if utils.IsStreamingPayload(hashPayload) {
+				var err error
+				wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+					var cr io.Reader
+					cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate)
+					return cr
+				})
+				if err != nil {
+					return sendResponse(ctx, err, logger, mm)
+				}
+			}
+
 			return ctx.Next()
 		}
 
-		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if !utils.IsSpecialPayload(hashPayload) {
 			// Calculate the hash of the request payload
 			hashedPayload := sha256.Sum256(ctx.Body())
@@ -149,8 +165,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 }
 
 type accounts struct {
-	iam  auth.IAMService
 	root RootUserConfig
+	iam  auth.IAMService
 }
 
 func (a accounts) getAccount(access string) (auth.Account, error) {

s3api/middlewares/chunk.go

@@ -1,62 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package middlewares
-
-import (
-	"io"
-	"time"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/metrics"
-	"github.com/versity/versitygw/s3api/utils"
-	"github.com/versity/versitygw/s3log"
-)
-
-// ProcessChunkedBody initializes the chunked upload stream if the
-// request appears to be a chunked upload
-func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
-	return func(ctx *fiber.Ctx) error {
-		decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
-		if decodedLength == "" {
-			return ctx.Next()
-		}
-		// TODO: validate content length
-
-		authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
-		if err != nil {
-			return sendResponse(ctx, err, logger, mm)
-		}
-
-		acct := ctx.Locals("account").(auth.Account)
-		amzdate := ctx.Get("X-Amz-Date")
-		date, _ := time.Parse(iso8601Format, amzdate)
-
-		if utils.IsBigDataAction(ctx) {
-			var err error
-			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
-				var cr *utils.ChunkReader
-				cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
-				return cr
-			})
-			if err != nil {
-				return sendResponse(ctx, err, logger, mm)
-			}
-			return ctx.Next()
-		}
-
-		return ctx.Next()
-	}
-}

s3api/middlewares/md5.go

@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
 		}
 
 		sum := md5.Sum(ctx.Body())
-		calculatedSum := utils.Md5SumString(sum[:])
+		calculatedSum := utils.Base64SumString(sum[:])
 
 		if incomingSum != calculatedSum {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

s3api/middlewares/presign-auth.go

@@ -43,6 +43,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
+		ctx.Locals("rootAccess", root.Access)
+
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
 			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)

s3api/router_test.go

@@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) {
 		iam auth.IAMService
 	}
 	tests := []struct {
-		args args
-		sa   *S3ApiRouter
 		name string
+		sa   *S3ApiRouter
+		args args
 	}{
 		{
 			name: "Initialize S3 api router",

s3api/server.go

@@ -29,15 +29,15 @@ import (
 )
 
 type S3ApiServer struct {
-	backend  backend.Backend
 	app      *fiber.App
+	backend  backend.Backend
 	router   *S3ApiRouter
-	cert     *tls.Certificate
 	port     string
-	health   string
+	cert     *tls.Certificate
 	quiet    bool
 	debug    bool
 	readonly bool
+	health   string
 }
 
 func New(
@@ -81,7 +81,6 @@ func New(
 	// Authentication middlewares
 	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
-	app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l, server.readonly))
 

s3api/server_test.go

@@ -39,9 +39,9 @@ func TestNew(t *testing.T) {
 	port := ":7070"
 
 	tests := []struct {
-		wantS3ApiServer *S3ApiServer
-		args            args
 		name            string
+		args            args
+		wantS3ApiServer *S3ApiServer
 		wantErr         bool
 	}{
 		{
@@ -78,8 +78,8 @@ func TestNew(t *testing.T) {
 
 func TestS3ApiServer_Serve(t *testing.T) {
 	tests := []struct {
-		sa      *S3ApiServer
 		name    string
+		sa      *S3ApiServer
 		wantErr bool
 	}{
 		{

s3api/utils/auth-reader.go

@@ -41,10 +41,10 @@ const (
 // the data is completely read.
 type AuthReader struct {
 	ctx    *fiber.Ctx
-	r      *HashReader
 	auth   AuthData
 	secret string
 	size   int
+	r      *HashReader
 	debug  bool
 }
 
@@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de
 	var hr *HashReader
 	hashPayload := ctx.Get("X-Amz-Content-Sha256")
 	if !IsSpecialPayload(hashPayload) {
-		hr, _ = NewHashReader(r, "", HashTypeSha256)
+		hr, _ = NewHashReader(r, "", HashTypeSha256Hex)
 	} else {
 		hr, _ = NewHashReader(r, "", HashTypeNone)
 	}
@@ -260,19 +260,3 @@ func removeSpace(str string) string {
 	}
 	return b.String()
 }
-
-var (
-	specialValues = map[string]bool{
-		"UNSIGNED-PAYLOAD":                                 true,
-		"STREAMING-UNSIGNED-PAYLOAD-TRAILER":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER":       true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD":         true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
-	}
-)
-
-// IsSpecialPayload checks for streaming/unsigned authorization types
-func IsSpecialPayload(str string) bool {
-	return specialValues[str]
-}

s3api/utils/chunk-reader.go

@@ -15,260 +15,98 @@
 package utils
 
 import (
-	"bytes"
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
 	"fmt"
-	"hash"
 	"io"
-	"math"
-	"strconv"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/s3err"
 )
 
-// chunked uploads described in:
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+type payloadType string
 
 const (
-	chunkHdrStr       = ";chunk-signature="
-	chunkHdrDelim     = "\r\n"
-	zeroLenSig        = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-	awsV4             = "AWS4"
-	awsS3Service      = "s3"
-	awsV4Request      = "aws4_request"
-	streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
+	payloadTypeUnsigned                 payloadType = "UNSIGNED-PAYLOAD"
+	payloadTypeStreamingUnsignedTrailer payloadType = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+	payloadTypeStreamingSigned          payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	payloadTypeStreamingSignedTrailer   payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
+	payloadTypeStreamingEcdsa           payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
+	payloadTypeStreamingEcdsaTrailer    payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
 )
 
-// ChunkReader reads from chunked upload request body, and returns
-// object data stream
-type ChunkReader struct {
-	r                io.Reader
-	chunkHash        hash.Hash
-	prevSig          string
-	parsedSig        string
-	strToSignPrefix  string
-	signingKey       []byte
-	stash            []byte
-	currentChunkSize int64
-	chunkDataLeft    int64
-	trailerExpected  int
-	skipcheck        bool
-}
-
-// NewChunkReader reads from request body io.Reader and parses out the
-// chunk metadata in stream. The headers are validated for proper signatures.
-// Reading from the chunk reader will read only the object data stream
-// without the chunk headers/trailers.
-func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (*ChunkReader, error) {
-	return &ChunkReader{
-		r:          r,
-		signingKey: getSigningKey(secret, region, date),
-		// the authdata.Signature is validated in the auth-reader,
-		// so we can use that here without any other checks
-		prevSig:         authdata.Signature,
-		chunkHash:       sha256.New(),
-		strToSignPrefix: getStringToSignPrefix(date, region),
-	}, nil
-}
-
-// Read satisfies the io.Reader for this type
-func (cr *ChunkReader) Read(p []byte) (int, error) {
-	n, err := cr.r.Read(p)
-	if err != nil && err != io.EOF {
-		return n, err
-	}
-
-	if cr.chunkDataLeft < int64(n) {
-		chunkSize := cr.chunkDataLeft
-		if chunkSize > 0 {
-			cr.chunkHash.Write(p[:chunkSize])
-		}
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		n += int(chunkSize)
-		return n, err
-	}
-
-	cr.chunkDataLeft -= int64(n)
-	cr.chunkHash.Write(p[:n])
-	return n, err
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
-// This part is the same for all chunks,
-// only the previous signature and hash of current chunk changes
-func getStringToSignPrefix(date time.Time, region string) string {
-	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
-		date.Format("20060102"),
-		region,
-		awsS3Service,
-		awsV4Request)
-
-	return fmt.Sprintf("%s\n%s\n%s",
-		streamPayloadAlgo,
-		date.Format("20060102T150405Z"),
-		credentialScope)
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
-// signature For each chunk, you calculate the signature using the following
-// string to sign. For the first chunk, you use the seed-signature as the
-// previous signature.
-func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
-	return fmt.Sprintf("%s\n%s\n%s\n%s",
-		prefix,
-		prevSig,
-		zeroLenSig,
-		hex.EncodeToString(chunkHash))
-}
-
-// The provided p should have all of the previous chunk data and trailer
-// consumed already. The positioning here is expected that p[0] starts the
-// new chunk size with the ";chunk-signature=" following. The only exception
-// is if we started consuming the trailer, but hit the end of the read buffer.
-// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
-// finish consuming the final trailer bytes.
-// This parses the chunk metadata in situ without allocating an extra buffer.
-// It will just read and validate the chunk metadata and then move the
-// following chunk data to overwrite the metadata in the provided buffer.
-func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
-	n := len(p)
-
-	if !cr.skipcheck && cr.parsedSig != "" {
-		chunkhash := cr.chunkHash.Sum(nil)
-		cr.chunkHash.Reset()
-
-		sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
-		cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
-
-		if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
-		}
-	}
-
-	if cr.trailerExpected != 0 {
-		if len(p) < len(chunkHdrDelim) {
-			// This is the special case where we need to consume the
-			// trailer, but instead hit the end of the buffer. The
-			// subsequent call will finish consuming the trailer.
-			cr.chunkDataLeft = 0
-			cr.trailerExpected -= len(p)
-			cr.skipcheck = true
-			return 0, nil
-		}
-		// move data up to remove trailer
-		copy(p, p[cr.trailerExpected:])
-		n -= cr.trailerExpected
-	}
-
-	cr.skipcheck = false
-
-	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
-	cr.currentChunkSize = chunkSize
-	cr.parsedSig = sig
-	if err == errskipHeader {
-		cr.chunkDataLeft = 0
-		return 0, nil
-	}
-	if err != nil {
-		return 0, err
-	}
-	if chunkSize == 0 {
-		return 0, io.EOF
-	}
-
-	cr.trailerExpected = len(chunkHdrDelim)
-
-	// move data up to remove chunk header
-	copy(p, p[bufOffset:n])
-	n -= bufOffset
-
-	// if remaining buffer larger than chunk data,
-	// parse next header in buffer
-	if int64(n) > chunkSize {
-		cr.chunkDataLeft = 0
-		cr.chunkHash.Write(p[:chunkSize])
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		if (chunkSize + int64(n)) > math.MaxInt {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
-		}
-		return n + int(chunkSize), err
-	}
-
-	cr.chunkDataLeft = chunkSize - int64(n)
-	cr.chunkHash.Write(p[:n])
-
-	return n, nil
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
-// Task 3: Calculate Signature
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
-func getSigningKey(secret, region string, date time.Time) []byte {
-	dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
-	dateRegionKey := hmac256(dateKey, []byte(region))
-	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
-	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
-	return signingKey
-}
-
-func hmac256(key []byte, data []byte) []byte {
-	hash := hmac.New(sha256.New, key)
-	hash.Write(data)
-	return hash.Sum(nil)
-}
-
 var (
-	errInvalidChunkFormat = errors.New("invalid chunk header format")
-	errskipHeader         = errors.New("skip to next header")
+	specialValues = map[payloadType]bool{
+		payloadTypeUnsigned:                 true,
+		payloadTypeStreamingUnsignedTrailer: true,
+		payloadTypeStreamingSigned:          true,
+		payloadTypeStreamingSignedTrailer:   true,
+		payloadTypeStreamingEcdsa:           true,
+		payloadTypeStreamingEcdsaTrailer:    true,
+	}
 )
 
+func (pt payloadType) isValid() bool {
+	return pt == payloadTypeUnsigned ||
+		pt == payloadTypeStreamingUnsignedTrailer ||
+		pt == payloadTypeStreamingSigned ||
+		pt == payloadTypeStreamingSignedTrailer ||
+		pt == payloadTypeStreamingEcdsa ||
+		pt == payloadTypeStreamingEcdsaTrailer
+}
+
+type checksumType string
+
 const (
-	maxHeaderSize = 1024
+	checksumTypeCrc32     checksumType = "x-amz-checksum-crc32"
+	checksumTypeCrc32c    checksumType = "x-amz-checksum-crc32c"
+	checksumTypeSha1      checksumType = "x-amz-checksum-sha1"
+	checksumTypeSha256    checksumType = "x-amz-checksum-sha256"
+	checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
 )
 
-// Theis returns the chunk payload size, signature, data start offset, and
-// error if any. See the AWS documentation for the chunk header format. The
-// header[0] byte is expected to be the first byte of the chunk size here.
-func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
-	stashLen := len(cr.stash)
-	if cr.stash != nil {
-		tmp := make([]byte, maxHeaderSize)
-		copy(tmp, cr.stash)
-		copy(tmp[len(cr.stash):], header)
-		header = tmp
-		cr.stash = nil
-	}
-
-	semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
-	if semicolonIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
-	}
-
-	sigIndex := semicolonIndex + len(chunkHdrStr)
-	sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
-	if sigEndIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
-	}
-
-	chunkSizeBytes := header[:semicolonIndex]
-	chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
-	if err != nil {
-		return 0, "", 0, errInvalidChunkFormat
-	}
-
-	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
-	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
-
-	return chunkSize, signature, dataStartOffset - stashLen, nil
+func (c checksumType) isValid() bool {
+	return c == checksumTypeCrc32 ||
+		c == checksumTypeCrc32c ||
+		c == checksumTypeSha1 ||
+		c == checksumTypeSha256 ||
+		c == checksumTypeCrc64nvme
+}
+
+// IsSpecialPayload checks for special authorization types
+func IsSpecialPayload(str string) bool {
+	return specialValues[payloadType(str)]
+}
+
+// IsChunkEncoding checks for streaming/unsigned authorization types
+func IsStreamingPayload(str string) bool {
+	pt := payloadType(str)
+	return pt == payloadTypeStreamingUnsignedTrailer ||
+		pt == payloadTypeStreamingSigned ||
+		pt == payloadTypeStreamingSignedTrailer
+}
+
+func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
+	decContLength := ctx.Get("X-Amz-Decoded-Content-Length")
+	if decContLength == "" {
+		return nil, s3err.GetAPIError(s3err.ErrMissingDecodedContentLength)
+	}
+	contentSha256 := payloadType(ctx.Get("X-Amz-Content-Sha256"))
+	if !contentSha256.isValid() {
+		//TODO: Add proper APIError
+		return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
+	}
+
+	checksumType := checksumType(ctx.Get("X-Amz-Trailer"))
+	if checksumType != "" && !checksumType.isValid() {
+		//TODO: Add proper APIError
+		return nil, fmt.Errorf("invalid X-Amz-Trailer: %v", checksumType)
+	}
+
+	switch contentSha256 {
+	case payloadTypeStreamingUnsignedTrailer:
+		return NewUnsignedChunkReader(r, checksumType)
+		//TODO: Add other chunk readers
+	}
+
+	return NewSignedChunkReader(r, authdata, region, secret, date)
 }

s3api/utils/csum-reader.go

@@ -16,13 +16,19 @@ package utils
 
 import (
 	"crypto/md5"
+	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"hash"
+	"hash/crc32"
+	"hash/crc64"
 	"io"
+	"math/bits"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -31,11 +37,21 @@ type HashType string
 
 const (
 	// HashTypeMd5 generates MD5 checksum for the data stream
-	HashTypeMd5 = "md5"
-	// HashTypeSha256 generates SHA256 checksum for the data stream
-	HashTypeSha256 = "sha256"
+	HashTypeMd5 HashType = "md5"
+	// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
+	HashTypeSha256 HashType = "sha256"
+	// HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream
+	HashTypeSha256Hex HashType = "sha256-hex"
+	// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
+	HashTypeSha1 HashType = "sha1"
+	// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
+	HashTypeCRC32 HashType = "crc32"
+	// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
+	HashTypeCRC32C HashType = "crc32c"
+	// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
+	HashTypeCRC64NVME HashType = "crc64nvme"
 	// HashTypeNone is a no-op checksum for the data stream
-	HashTypeNone = "none"
+	HashTypeNone HashType = "none"
 )
 
 // HashReader is an io.Reader that calculates the checksum
@@ -62,8 +78,18 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
 	switch ht {
 	case HashTypeMd5:
 		hash = md5.New()
+	case HashTypeSha256Hex:
+		hash = sha256.New()
 	case HashTypeSha256:
 		hash = sha256.New()
+	case HashTypeSha1:
+		hash = sha1.New()
+	case HashTypeCRC32:
+		hash = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeCRC64NVME:
+		hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
 	case HashTypeNone:
 		hash = noop{}
 	default:
@@ -88,15 +114,40 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	if errors.Is(readerr, io.EOF) && hr.sum != "" {
 		switch hr.hashType {
 		case HashTypeMd5:
-			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
 			}
-		case HashTypeSha256:
-			sum := hex.EncodeToString(hr.hash.Sum(nil))
+		case HashTypeSha256Hex:
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
 			}
+		case HashTypeCRC32:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32)
+			}
+		case HashTypeCRC32C:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c)
+			}
+		case HashTypeSha1:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1)
+			}
+		case HashTypeSha256:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
+			}
+		case HashTypeCRC64NVME:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
+			}
 		default:
 			return n, errInvalidHashType
 		}
@@ -104,20 +155,38 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	return n, readerr
 }
 
+func (hr *HashReader) SetReader(r io.Reader) {
+	hr.r = r
+}
+
 // Sum returns the checksum hash of the data read so far
 func (hr *HashReader) Sum() string {
 	switch hr.hashType {
 	case HashTypeMd5:
-		return Md5SumString(hr.hash.Sum(nil))
-	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256Hex:
 		return hex.EncodeToString(hr.hash.Sum(nil))
+	case HashTypeCRC32:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC32C:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha1:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC64NVME:
+		return Base64SumString(hr.hash.Sum(nil))
 	default:
 		return ""
 	}
 }
 
+func (hr *HashReader) Type() HashType {
+	return hr.hashType
+}
+
 // Md5SumString converts the hash bytes to the string checksum value
-func Md5SumString(b []byte) string {
+func Base64SumString(b []byte) string {
 	return base64.StdEncoding.EncodeToString(b)
 }
 
@@ -128,3 +197,59 @@ func (n noop) Sum(b []byte) []byte         { return []byte{} }
 func (n noop) Reset()                      {}
 func (n noop) Size() int                   { return 0 }
 func (n noop) BlockSize() int              { return 1 }
+
+// NewCompositeChecksumReader initializes a composite checksum
+// processor, which decodes and validates the provided
+// checksums and returns the final checksum based on
+// the previous processings.
+//
+// The supported checksum types are:
+// - CRC32
+// - CRC32C
+// - SHA1
+// - SHA256
+func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
+	var hasher hash.Hash
+	switch ht {
+	case HashTypeSha256:
+		hasher = sha256.New()
+	case HashTypeSha1:
+		hasher = sha1.New()
+	case HashTypeCRC32:
+		hasher = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeNone:
+		hasher = noop{}
+	default:
+		return nil, errInvalidHashType
+	}
+
+	return &CompositeChecksumReader{
+		hasher: hasher,
+	}, nil
+}
+
+type CompositeChecksumReader struct {
+	hasher hash.Hash
+}
+
+// Decodes and writes the checksum in the hasher
+func (ccr *CompositeChecksumReader) Process(checksum string) error {
+	data, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		return fmt.Errorf("base64 decode: %w", err)
+	}
+
+	_, err = ccr.hasher.Write(data)
+	if err != nil {
+		return fmt.Errorf("hash write: %w", err)
+	}
+
+	return nil
+}
+
+// Returns the base64 encoded composite checksum
+func (ccr *CompositeChecksumReader) Sum() string {
+	return Base64SumString(ccr.hasher.Sum(nil))
+}

s3api/utils/presign-auth-reader.go

@@ -41,10 +41,10 @@ const (
 // data requests where the data size is not known until
 // the data is completely read.
 type PresignedAuthReader struct {
-	r      io.Reader
 	ctx    *fiber.Ctx
 	auth   AuthData
 	secret string
+	r      io.Reader
 	debug  bool
 }
 

s3api/utils/presign-auth-reader_test.go

@@ -23,13 +23,13 @@ import (
 
 func Test_validateExpiration(t *testing.T) {
 	type args struct {
-		date time.Time
 		str  string
+		date time.Time
 	}
 	tests := []struct {
+		name string
 		args args
 		err  error
-		name string
 	}{
 		{
 			name: "empty-expiration",

s3api/utils/signed-chunk-reader.go

@@ -0,0 +1,276 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"math"
+	"strconv"
+	"time"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+// chunked uploads described in:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+
+const (
+	chunkHdrStr       = ";chunk-signature="
+	chunkHdrDelim     = "\r\n"
+	zeroLenSig        = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	awsV4             = "AWS4"
+	awsS3Service      = "s3"
+	awsV4Request      = "aws4_request"
+	streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
+)
+
+// ChunkReader reads from chunked upload request body, and returns
+// object data stream
+type ChunkReader struct {
+	r                io.Reader
+	signingKey       []byte
+	prevSig          string
+	parsedSig        string
+	currentChunkSize int64
+	chunkDataLeft    int64
+	trailerExpected  int
+	stash            []byte
+	chunkHash        hash.Hash
+	strToSignPrefix  string
+	skipcheck        bool
+}
+
+// NewChunkReader reads from request body io.Reader and parses out the
+// chunk metadata in stream. The headers are validated for proper signatures.
+// Reading from the chunk reader will read only the object data stream
+// without the chunk headers/trailers.
+func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
+	return &ChunkReader{
+		r:          r,
+		signingKey: getSigningKey(secret, region, date),
+		// the authdata.Signature is validated in the auth-reader,
+		// so we can use that here without any other checks
+		prevSig:         authdata.Signature,
+		chunkHash:       sha256.New(),
+		strToSignPrefix: getStringToSignPrefix(date, region),
+	}, nil
+}
+
+// Read satisfies the io.Reader for this type
+func (cr *ChunkReader) Read(p []byte) (int, error) {
+	n, err := cr.r.Read(p)
+	if err != nil && err != io.EOF {
+		return 0, err
+	}
+
+	if cr.chunkDataLeft < int64(n) {
+		chunkSize := cr.chunkDataLeft
+		if chunkSize > 0 {
+			cr.chunkHash.Write(p[:chunkSize])
+		}
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		n += int(chunkSize)
+		return n, err
+	}
+
+	cr.chunkDataLeft -= int64(n)
+	cr.chunkHash.Write(p[:n])
+	return n, err
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// This part is the same for all chunks,
+// only the previous signature and hash of current chunk changes
+func getStringToSignPrefix(date time.Time, region string) string {
+	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
+		date.Format("20060102"),
+		region,
+		awsS3Service,
+		awsV4Request)
+
+	return fmt.Sprintf("%s\n%s\n%s",
+		streamPayloadAlgo,
+		date.Format("20060102T150405Z"),
+		credentialScope)
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// signature For each chunk, you calculate the signature using the following
+// string to sign. For the first chunk, you use the seed-signature as the
+// previous signature.
+func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
+	return fmt.Sprintf("%s\n%s\n%s\n%s",
+		prefix,
+		prevSig,
+		zeroLenSig,
+		hex.EncodeToString(chunkHash))
+}
+
+// The provided p should have all of the previous chunk data and trailer
+// consumed already. The positioning here is expected that p[0] starts the
+// new chunk size with the ";chunk-signature=" following. The only exception
+// is if we started consuming the trailer, but hit the end of the read buffer.
+// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
+// finish consuming the final trailer bytes.
+// This parses the chunk metadata in situ without allocating an extra buffer.
+// It will just read and validate the chunk metadata and then move the
+// following chunk data to overwrite the metadata in the provided buffer.
+func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
+	n := len(p)
+
+	if !cr.skipcheck && cr.parsedSig != "" {
+		chunkhash := cr.chunkHash.Sum(nil)
+		cr.chunkHash.Reset()
+
+		sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
+		cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
+
+		if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
+	}
+
+	if cr.trailerExpected != 0 {
+		if len(p) < len(chunkHdrDelim) {
+			// This is the special case where we need to consume the
+			// trailer, but instead hit the end of the buffer. The
+			// subsequent call will finish consuming the trailer.
+			cr.chunkDataLeft = 0
+			cr.trailerExpected -= len(p)
+			cr.skipcheck = true
+			return 0, nil
+		}
+		// move data up to remove trailer
+		copy(p, p[cr.trailerExpected:])
+		n -= cr.trailerExpected
+	}
+
+	cr.skipcheck = false
+
+	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
+	cr.currentChunkSize = chunkSize
+	cr.parsedSig = sig
+	if err == errskipHeader {
+		cr.chunkDataLeft = 0
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	if chunkSize == 0 {
+		return 0, io.EOF
+	}
+
+	cr.trailerExpected = len(chunkHdrDelim)
+
+	// move data up to remove chunk header
+	copy(p, p[bufOffset:n])
+	n -= bufOffset
+
+	// if remaining buffer larger than chunk data,
+	// parse next header in buffer
+	if int64(n) > chunkSize {
+		cr.chunkDataLeft = 0
+		cr.chunkHash.Write(p[:chunkSize])
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		if (chunkSize + int64(n)) > math.MaxInt {
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
+		return n + int(chunkSize), err
+	}
+
+	cr.chunkDataLeft = chunkSize - int64(n)
+	cr.chunkHash.Write(p[:n])
+
+	return n, nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+// Task 3: Calculate Signature
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
+func getSigningKey(secret, region string, date time.Time) []byte {
+	dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
+	dateRegionKey := hmac256(dateKey, []byte(region))
+	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
+	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
+	return signingKey
+}
+
+func hmac256(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+var (
+	errInvalidChunkFormat = errors.New("invalid chunk header format")
+	errskipHeader         = errors.New("skip to next header")
+)
+
+const (
+	maxHeaderSize = 1024
+)
+
+// This returns the chunk payload size, signature, data start offset, and
+// error if any. See the AWS documentation for the chunk header format. The
+// header[0] byte is expected to be the first byte of the chunk size here.
+func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
+	stashLen := len(cr.stash)
+	if stashLen > maxHeaderSize {
+		return 0, "", 0, errInvalidChunkFormat
+	}
+	if cr.stash != nil {
+		tmp := make([]byte, maxHeaderSize)
+		copy(tmp, cr.stash)
+		copy(tmp[len(cr.stash):], header)
+		header = tmp
+		cr.stash = nil
+	}
+
+	semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
+	if semicolonIndex == -1 {
+		cr.stash = make([]byte, len(header))
+		copy(cr.stash, header)
+		cr.trailerExpected = 0
+		return 0, "", 0, errskipHeader
+	}
+
+	sigIndex := semicolonIndex + len(chunkHdrStr)
+	sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
+	if sigEndIndex == -1 {
+		cr.stash = make([]byte, len(header))
+		copy(cr.stash, header)
+		cr.trailerExpected = 0
+		return 0, "", 0, errskipHeader
+	}
+
+	chunkSizeBytes := header[:semicolonIndex]
+	chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
+	if err != nil {
+		return 0, "", 0, errInvalidChunkFormat
+	}
+
+	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
+	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
+
+	return chunkSize, signature, dataStartOffset - stashLen, nil
+}

s3api/utils/unsigned-chunk-reader.go

@@ -0,0 +1,235 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"hash"
+	"hash/crc32"
+	"hash/crc64"
+	"io"
+	"math/bits"
+	"strconv"
+	"strings"
+)
+
+var (
+	trailerDelim         = []byte{'\n', '\r', '\n'}
+	errMalformedEncoding = errors.New("malformed chunk encoding")
+)
+
+type UnsignedChunkReader struct {
+	reader           *bufio.Reader
+	checksumType     checksumType
+	expectedChecksum string
+	hasher           hash.Hash
+	stash            []byte
+	chunkCounter     int
+	offset           int
+}
+
+func NewUnsignedChunkReader(r io.Reader, ct checksumType) (*UnsignedChunkReader, error) {
+	hasher, err := getHasher(ct)
+	if err != nil {
+		return nil, err
+	}
+	return &UnsignedChunkReader{
+		reader:       bufio.NewReader(r),
+		checksumType: ct,
+		stash:        make([]byte, 0),
+		hasher:       hasher,
+		chunkCounter: 1,
+	}, nil
+}
+
+func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
+	// First read any stashed data
+	if len(ucr.stash) != 0 {
+		n := copy(p, ucr.stash)
+		ucr.offset += n
+
+		if n < len(ucr.stash) {
+			ucr.stash = ucr.stash[n:]
+			ucr.offset = 0
+			return n, nil
+		}
+	}
+
+	for {
+		// Read the chunk size
+		chunkSize, err := ucr.extractChunkSize()
+		if err != nil {
+			return 0, err
+		}
+
+		if chunkSize == 0 {
+			// Stop reading parsing payloads as 0 sized chunk is reached
+			break
+		}
+		rdr := io.TeeReader(ucr.reader, ucr.hasher)
+		payload := make([]byte, chunkSize)
+		// Read and cache the payload
+		_, err = io.ReadFull(rdr, payload)
+		if err != nil {
+			return 0, err
+		}
+
+		// Skip the trailing "\r\n"
+		if err := ucr.readAndSkip('\r', '\n'); err != nil {
+			return 0, err
+		}
+
+		// Copy the payload into the io.Reader buffer
+		n := copy(p[ucr.offset:], payload)
+		ucr.offset += n
+		ucr.chunkCounter++
+
+		if int64(n) < chunkSize {
+			// stash the remaining data
+			ucr.stash = payload[n:]
+			dataRead := ucr.offset
+			ucr.offset = 0
+			return dataRead, nil
+		}
+	}
+
+	// Read and validate trailers
+	if err := ucr.readTrailer(); err != nil {
+		return 0, err
+	}
+
+	return ucr.offset, io.EOF
+}
+
+// Reads and validates the bytes provided from the underlying io.Reader
+func (ucr *UnsignedChunkReader) readAndSkip(data ...byte) error {
+	for _, d := range data {
+		b, err := ucr.reader.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+
+		if b != d {
+			return errMalformedEncoding
+		}
+	}
+
+	return nil
+}
+
+// Extracts the chunk size from the payload
+func (ucr *UnsignedChunkReader) extractChunkSize() (int64, error) {
+	line, err := ucr.reader.ReadString('\n')
+	if err != nil {
+		return 0, errMalformedEncoding
+	}
+	line = strings.TrimSpace(line)
+
+	chunkSize, err := strconv.ParseInt(line, 16, 64)
+	if err != nil {
+		return 0, errMalformedEncoding
+	}
+
+	return chunkSize, nil
+}
+
+// Reads and validates the trailer at the end
+func (ucr *UnsignedChunkReader) readTrailer() error {
+	var trailerBuffer bytes.Buffer
+
+	for {
+		v, err := ucr.reader.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+		if v != '\r' {
+			trailerBuffer.WriteByte(v)
+			continue
+		}
+		var tmp [3]byte
+		_, err = io.ReadFull(ucr.reader, tmp[:])
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+		if !bytes.Equal(tmp[:], trailerDelim) {
+			return errMalformedEncoding
+		}
+		break
+	}
+
+	// Parse the trailer
+	trailerHeader := trailerBuffer.String()
+	trailerHeader = strings.TrimSpace(trailerHeader)
+	trailerHeaderParts := strings.Split(trailerHeader, ":")
+	if len(trailerHeaderParts) != 2 {
+		return errMalformedEncoding
+	}
+
+	if trailerHeaderParts[0] != string(ucr.checksumType) {
+		//TODO: handle the error
+		return errMalformedEncoding
+	}
+
+	ucr.expectedChecksum = trailerHeaderParts[1]
+
+	// Validate checksum
+	return ucr.validateChecksum()
+}
+
+// Validates the trailing checksum sent at the end
+func (ucr *UnsignedChunkReader) validateChecksum() error {
+	csum := ucr.hasher.Sum(nil)
+	checksum := base64.StdEncoding.EncodeToString(csum)
+
+	if checksum != ucr.expectedChecksum {
+		return fmt.Errorf("actual checksum: %v, expected checksum: %v", checksum, ucr.expectedChecksum)
+	}
+
+	return nil
+}
+
+// Retruns the hash calculator based on the hash type provided
+func getHasher(ct checksumType) (hash.Hash, error) {
+	switch ct {
+	case checksumTypeCrc32:
+		return crc32.NewIEEE(), nil
+	case checksumTypeCrc32c:
+		return crc32.New(crc32.MakeTable(crc32.Castagnoli)), nil
+	case checksumTypeCrc64nvme:
+		table := crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))
+		return crc64.New(table), nil
+	case checksumTypeSha1:
+		return sha1.New(), nil
+	case checksumTypeSha256:
+		return sha256.New(), nil
+	default:
+		return nil, errors.New("unsupported checksum type")
+	}
+}

s3api/utils/utils.go

@@ -16,6 +16,7 @@ package utils
 
 import (
 	"bytes"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -178,9 +179,15 @@ func ParseUint(str string) (int32, error) {
 	if str == "" {
 		return 1000, nil
 	}
-	num, err := strconv.ParseUint(str, 10, 16)
+	num, err := strconv.ParseInt(str, 10, 32)
 	if err != nil {
-		return 1000, fmt.Errorf("invalid uint: %w", err)
+		return 1000, fmt.Errorf("invalid int: %w", err)
+	}
+	if num < 0 {
+		return 1000, fmt.Errorf("negative uint: %v", num)
+	}
+	if num > 1000 {
+		num = 1000
 	}
 	return int32(num), nil
 }
@@ -196,6 +203,13 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
 	}
 }
 
+// Streams the response body by chunks
+func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser, bodysize int) {
+	// SetBodyStream will call Close() on the reader when the stream is done
+	// since rdr is a ReadCloser
+	ctx.Context().SetBodyStream(rdr, bodysize)
+}
+
 func IsValidBucketName(bucket string) bool {
 	if len(bucket) < 3 || len(bucket) > 63 {
 		return false
@@ -272,7 +286,9 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp
 	if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
 		output.StorageClass = ""
 	}
-	fmt.Printf("%+v\n", output)
+	if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok {
+		output.Checksum = nil
+	}
 
 	return output
 }
@@ -432,3 +448,186 @@ func shouldEscape(c byte) bool {
 
 	return true
 }
+
+func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) {
+	sdkAlgorithm := types.ChecksumAlgorithm(strings.ToUpper(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")))
+
+	err := IsChecksumAlgorithmValid(sdkAlgorithm)
+	if err != nil {
+		return "", nil, err
+	}
+
+	checksums := map[types.ChecksumAlgorithm]string{}
+
+	var hdrErr error
+	// Parse and validate checksum headers
+	ctx.Request().Header.VisitAll(func(key, value []byte) {
+		// Skip `X-Amz-Checksum-Type` as it's a special header
+		if hdrErr != nil || !strings.HasPrefix(string(key), "X-Amz-Checksum-") || string(key) == "X-Amz-Checksum-Type" {
+			return
+		}
+
+		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(key), "X-Amz-Checksum-")))
+		err := IsChecksumAlgorithmValid(algo)
+		if err != nil {
+			hdrErr = s3err.GetAPIError(s3err.ErrInvalidChecksumHeader)
+			return
+		}
+
+		checksums[algo] = string(value)
+	})
+
+	if hdrErr != nil {
+		return sdkAlgorithm, nil, hdrErr
+	}
+
+	headerCtr := 0
+	for al, val := range checksums {
+		if val != "" && !IsValidChecksum(val, al) {
+			return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al))))
+		}
+		// If any other checksum value is provided,
+		// rather than x-amz-sdk-checksum-algorithm
+		if sdkAlgorithm != "" && sdkAlgorithm != al && val != "" {
+			return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
+		}
+		if val != "" {
+			sdkAlgorithm = al
+			headerCtr++
+		}
+
+		if headerCtr > 1 {
+			return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
+		}
+	}
+
+	return sdkAlgorithm, checksums, nil
+}
+
+var checksumLengths = map[types.ChecksumAlgorithm]int{
+	types.ChecksumAlgorithmCrc32:     4,
+	types.ChecksumAlgorithmCrc32c:    4,
+	types.ChecksumAlgorithmCrc64nvme: 8,
+	types.ChecksumAlgorithmSha1:      20,
+	types.ChecksumAlgorithmSha256:    32,
+}
+
+func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
+	decoded, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		return false
+	}
+
+	expectedLength, exists := checksumLengths[algorithm]
+	if !exists {
+		return false
+	}
+
+	return len(decoded) == expectedLength
+}
+
+func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
+	alg = types.ChecksumAlgorithm(strings.ToUpper(string(alg)))
+	if alg != "" &&
+		alg != types.ChecksumAlgorithmCrc32 &&
+		alg != types.ChecksumAlgorithmCrc32c &&
+		alg != types.ChecksumAlgorithmSha1 &&
+		alg != types.ChecksumAlgorithmSha256 &&
+		alg != types.ChecksumAlgorithmCrc64nvme {
+		return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
+	}
+
+	return nil
+}
+
+// Validates the provided checksum type
+func IsChecksumTypeValid(t types.ChecksumType) error {
+	if t != "" &&
+		t != types.ChecksumTypeComposite &&
+		t != types.ChecksumTypeFullObject {
+		return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")
+	}
+	return nil
+}
+
+type checksumTypeSchema map[types.ChecksumType]struct{}
+type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema
+
+// A table defining the checksum algorithm/type support
+var checksumMap checksumSchema = checksumSchema{
+	types.ChecksumAlgorithmCrc32: checksumTypeSchema{
+		types.ChecksumTypeComposite:  struct{}{},
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	types.ChecksumAlgorithmCrc32c: checksumTypeSchema{
+		types.ChecksumTypeComposite:  struct{}{},
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	types.ChecksumAlgorithmSha1: checksumTypeSchema{
+		types.ChecksumTypeComposite: struct{}{},
+		"":                          struct{}{},
+	},
+	types.ChecksumAlgorithmSha256: checksumTypeSchema{
+		types.ChecksumTypeComposite: struct{}{},
+		"":                          struct{}{},
+	},
+	types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	// Both could be empty
+	"": checksumTypeSchema{
+		"": struct{}{},
+	},
+}
+
+// Checks if checksum type and algorithm are supported together
+func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType) error {
+	typeSchema := checksumMap[algo]
+	_, ok := typeSchema[t]
+	if !ok {
+		return s3err.GetChecksumSchemaMismatchErr(algo, t)
+	}
+
+	return nil
+}
+
+// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers
+func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) {
+	algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
+	if err := IsChecksumAlgorithmValid(algo); err != nil {
+		return "", "", err
+	}
+
+	chType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
+	if err := IsChecksumTypeValid(chType); err != nil {
+		return "", "", err
+	}
+
+	// Verify if checksum algorithm is provided, if
+	// checksum type is specified
+	if chType != "" && algo == "" {
+		return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)
+	}
+
+	// Verify if the checksum type is supported for
+	// the provided checksum algorithm
+	if err := checkChecksumTypeAndAlgo(algo, chType); err != nil {
+		return algo, chType, err
+	}
+
+	// x-amz-checksum-type defaults to COMPOSITE
+	// if x-amz-checksum-algorithm is set except
+	// for the CRC64NVME algorithm: it defaults to FULL_OBJECT
+	if algo != "" && chType == "" {
+		if algo == types.ChecksumAlgorithmCrc64nvme {
+			chType = types.ChecksumTypeFullObject
+		} else {
+			chType = types.ChecksumTypeComposite
+		}
+	}
+
+	return algo, chType, nil
+}

s3api/utils/utils_test.go

@@ -49,11 +49,11 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 	request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
 
 	tests := []struct {
+		name    string
 		args    args
 		want    *http.Request
-		name    string
-		hdrs    []string
 		wantErr bool
+		hdrs    []string
 	}{
 		{
 			name: "Success-response",
@@ -101,9 +101,9 @@ func TestGetUserMetaData(t *testing.T) {
 	req := ctx.Request()
 
 	tests := []struct {
+		name         string
 		args         args
 		wantMetadata map[string]string
-		name         string
 	}{
 		{
 			name: "Success-empty-response",
@@ -268,6 +268,14 @@ func TestParseUint(t *testing.T) {
 			want:    23,
 			wantErr: false,
 		},
+		{
+			name: "Parse-uint-greater-than-1000",
+			args: args{
+				str: "25000000",
+			},
+			want:    1000,
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -527,3 +535,325 @@ func Test_escapePath(t *testing.T) {
 		})
 	}
 }
+
+func TestIsChecksumAlgorithmValid(t *testing.T) {
+	type args struct {
+		alg types.ChecksumAlgorithm
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "empty",
+			args: args{
+				alg: "",
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc32",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc32,
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc32c",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc32c,
+			},
+			wantErr: false,
+		},
+		{
+			name: "sha1",
+			args: args{
+				alg: types.ChecksumAlgorithmSha1,
+			},
+			wantErr: false,
+		},
+		{
+			name: "sha256",
+			args: args{
+				alg: types.ChecksumAlgorithmSha256,
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc64nvme",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc64nvme,
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid",
+			args: args{
+				alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := IsChecksumAlgorithmValid(tt.args.alg); (err != nil) != tt.wantErr {
+				t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestIsValidChecksum(t *testing.T) {
+	type args struct {
+		checksum  string
+		algorithm types.ChecksumAlgorithm
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "invalid-base64",
+			args: args{
+				checksum:  "invalid_base64_string",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: false,
+		},
+		{
+			name: "invalid-crc32",
+			args: args{
+				checksum:  "YXNkZmFzZGZhc2Rm",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: false,
+		},
+		{
+			name: "valid-crc32",
+			args: args{
+				checksum:  "ww2FVQ==",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-crc32c",
+			args: args{
+				checksum:  "Zmdoa2doZmtnZmhr",
+				algorithm: types.ChecksumAlgorithmCrc32c,
+			},
+			want: false,
+		},
+		{
+			name: "valid-crc32c",
+			args: args{
+				checksum:  "DOsb4w==",
+				algorithm: types.ChecksumAlgorithmCrc32c,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-sha1",
+			args: args{
+				checksum:  "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm",
+				algorithm: types.ChecksumAlgorithmSha1,
+			},
+			want: false,
+		},
+		{
+			name: "valid-sha1",
+			args: args{
+				checksum:  "L4q6V59Zcwn12wyLIytoE2c1ugk=",
+				algorithm: types.ChecksumAlgorithmSha1,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-sha256",
+			args: args{
+				checksum:  "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==",
+				algorithm: types.ChecksumAlgorithmSha256,
+			},
+			want: false,
+		},
+		{
+			name: "valid-sha256",
+			args: args{
+				checksum:  "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
+				algorithm: types.ChecksumAlgorithmSha256,
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm); got != tt.want {
+				t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsChecksumTypeValid(t *testing.T) {
+	type args struct {
+		t types.ChecksumType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "valid_FULL_OBJECT",
+			args: args{
+				t: types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "valid_COMPOSITE",
+			args: args{
+				t: types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid",
+			args: args{
+				t: types.ChecksumType("invalid_checksum_type"),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := IsChecksumTypeValid(tt.args.t); (err != nil) != tt.wantErr {
+				t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_checkChecksumTypeAndAlgo(t *testing.T) {
+	type args struct {
+		algo types.ChecksumAlgorithm
+		t    types.ChecksumType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "full_object-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-crc32c",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32c,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-sha1",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-sha256",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-crc64nvme",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc64nvme,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc32c",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32c,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-sha1",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-sha256",
+			args: args{
+				algo: types.ChecksumAlgorithmSha256,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc64nvme",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc64nvme,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: true,
+		},
+		{
+			name: "composite-empty",
+			args: args{
+				t: types.ChecksumTypeComposite,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-empty",
+			args: args{
+				t: types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t); (err != nil) != tt.wantErr {
+				t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -17,7 +17,11 @@ package s3err
 import (
 	"bytes"
 	"encoding/xml"
+	"fmt"
 	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 // APIError structure
@@ -72,10 +76,14 @@ const (
 	ErrInvalidPartNumberMarker
 	ErrInvalidObjectAttributes
 	ErrInvalidPart
+	ErrEmptyParts
 	ErrInvalidPartNumber
+	ErrInvalidPartOrder
+	ErrInvalidCompleteMpPartNumber
 	ErrInternalError
 	ErrInvalidCopyDest
 	ErrInvalidCopySource
+	ErrInvalidCopySourceRange
 	ErrInvalidTag
 	ErrAuthHeaderEmpty
 	ErrSignatureVersionNotSupported
@@ -105,6 +113,7 @@ const (
 	ErrSignatureTerminationStr
 	ErrSignatureIncorrService
 	ErrContentSHA256Mismatch
+	ErrMissingDecodedContentLength
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
 	ErrMissingDateHeader
@@ -139,11 +148,17 @@ const (
 	ErrInvalidVersionId
 	ErrNoSuchVersion
 	ErrSuspendedVersioningNotAllowed
+	ErrMultipleChecksumHeaders
+	ErrInvalidChecksumAlgorithm
+	ErrInvalidChecksumPart
+	ErrChecksumTypeWithAlgo
+	ErrInvalidChecksumHeader
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
+	ErrDirectoryNotEmpty
 	ErrQuotaExceeded
 	ErrVersioningNotConfigured
 
@@ -153,6 +168,7 @@ const (
 	ErrAdminUserExists
 	ErrAdminInvalidUserRole
 	ErrAdminMissingUserAcess
+	ErrAdminMethodNotSupported
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -251,11 +267,26 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrEmptyParts: {
+		Code:           "InvalidRequest",
+		Description:    "You must specify at least one part",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidPartNumber: {
 		Code:           "InvalidArgument",
 		Description:    "Part number must be an integer between 1 and 10000, inclusive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidPartOrder: {
+		Code:           "InvalidPartOrder",
+		Description:    "The list of parts was not in ascending order. Parts must be ordered by part number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCompleteMpPartNumber: {
+		Code:           "InvalidArgument",
+		Description:    "PartNumber must be >= 1",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -266,6 +297,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidCopySourceRange: {
+		Code:           "InvalidArgument",
+		Description:    "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidTag: {
 		Code:           "InvalidArgument",
 		Description:    "The Tag value you have provided is invalid",
@@ -416,6 +452,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingDecodedContentLength: {
+		Code:           "MissingContentLength",
+		Description:    "You must provide the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
 	ErrMissingDateHeader: {
 		Code:           "AccessDenied",
 		Description:    "AWS authentication requires a valid Date or x-amz-date header.",
@@ -576,6 +617,31 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMultipleChecksumHeaders: {
+		Code:           "InvalidRequest",
+		Description:    "Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumAlgorithm: {
+		Code:           "InvalidRequest",
+		Description:    "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumPart: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid Base64 or multiple checksums present in request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrChecksumTypeWithAlgo: {
+		Code:           "InvalidRequest",
+		Description:    "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumHeader: {
+		Code:           "InvalidRequest",
+		Description:    "The algorithm type you specified in x-amz-checksum- header is invalid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {
@@ -593,6 +659,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Directory object contains data payload.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrDirectoryNotEmpty: {
+		Code:           "ErrDirectoryNotEmpty",
+		Description:    "Directory object not empty.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrQuotaExceeded: {
 		Code:           "QuotaExceeded",
 		Description:    "Your request was denied due to quota exceeded.",
@@ -630,6 +701,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "User access key ID is missing.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrAdminMethodNotSupported: {
+		Code:           "XAdminMethodNotSupported",
+		Description:    "The method is not supported in single root user mode.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.
@@ -660,3 +736,72 @@ func encodeResponse(response interface{}) []byte {
 	e.Encode(response)
 	return bytesBuffer.Bytes()
 }
+
+// Returns invalid checksum error with the provided header in the error description
+func GetInvalidChecksumHeaderErr(header string) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Value for %v header is invalid.", header),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch APIError
+func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Checksum Type mismatch occurred, expected checksum Type: %v, actual checksum Type: %v", expected, actual),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns incorrect checksum APIError
+func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError {
+	return APIError{
+		Code:           "BadDigest",
+		Description:    fmt.Sprintf("The %v you specified did not match the calculated checksum.", strings.ToLower(string(algo))),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch error with checksum algorithm
+func GetChecksumSchemaMismatchErr(algo types.ChecksumAlgorithm, t types.ChecksumType) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The %v checksum type cannot be used with the %v checksum algorithm.", algo, strings.ToLower(string(t))),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch error for multipart uploads
+func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The upload was created using the %v checksum mode. The complete request must use the same checksum mode.", t),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func GetIncorrectMpObjectSizeErr(expected, actual int64) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The provided 'x-amz-mp-object-size' header value %v does not match what was computed: %v", expected, actual),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func GetInvalidMpObjectSizeErr(val int64) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Value for x-amz-mp-object-size header is less than zero: '%v'", val),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func CreateExceedingRangeErr(objSize int64) APIError {
+	return APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}

s3event/event.go

@@ -30,11 +30,11 @@ type S3EventSender interface {
 }
 
 type EventMeta struct {
-	ObjectETag  *string
-	VersionId   *string
 	BucketOwner string
 	EventName   EventType
 	ObjectSize  int64
+	ObjectETag  *string
+	VersionId   *string
 }
 
 type EventSchema struct {
@@ -42,8 +42,6 @@ type EventSchema struct {
 }
 
 type EventRecord struct {
-	ResponseElements  EventResponseElements `json:"responseElements"`
-	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
 	AwsRegion         string                `json:"awsRegion"`
@@ -51,7 +49,9 @@ type EventRecord struct {
 	EventName         EventType             `json:"eventName"`
 	UserIdentity      EventUserIdentity     `json:"userIdentity"`
 	RequestParameters EventRequestParams    `json:"requestParameters"`
+	ResponseElements  EventResponseElements `json:"responseElements"`
 	S3                EventS3Data           `json:"s3"`
+	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 }
 
 type EventUserIdentity struct {
@@ -99,11 +99,11 @@ type EventS3BucketData struct {
 }
 
 type EventObjectData struct {
+	Key       string  `json:"key"`
+	Size      int64   `json:"size"`
 	ETag      *string `json:"eTag"`
 	VersionId *string `json:"versionId"`
-	Key       string  `json:"key"`
 	Sequencer string  `json:"sequencer"`
-	Size      int64   `json:"size"`
 }
 
 type EventConfig struct {

s3event/kafka.go

@@ -31,9 +31,9 @@ import (
 var sequencer = 0
 
 type Kafka struct {
+	key    string
 	writer *kafka.Writer
 	filter EventFilter
-	key    string
 	mu     sync.Mutex
 }
 

s3event/nats.go

@@ -27,10 +27,10 @@ import (
 )
 
 type NatsEventSender struct {
-	client *nats.Conn
-	filter EventFilter
 	topic  string
+	client *nats.Conn
 	mu     sync.Mutex
+	filter EventFilter
 }
 
 func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {

s3event/webhook.go

@@ -30,9 +30,9 @@ import (
 )
 
 type Webhook struct {
+	url    string
 	client *http.Client
 	filter EventFilter
-	url    string
 	mu     sync.Mutex
 }
 

s3log/audit-logger.go

@@ -33,8 +33,8 @@ type AuditLogger interface {
 
 type LogMeta struct {
 	BucketOwner string
-	Action      string
 	ObjectSize  int64
+	Action      string
 	HttpStatus  int
 }
 
@@ -45,16 +45,21 @@ type LogConfig struct {
 }
 
 type LogFields struct {
-	Time               time.Time
 	BucketOwner        string
 	Bucket             string
+	Time               time.Time
 	RemoteIP           string
 	Requester          string
 	RequestID          string
 	Operation          string
 	Key                string
 	RequestURI         string
+	HttpStatus         int
 	ErrorCode          string
+	BytesSent          int
+	ObjectSize         int64
+	TotalTime          int64
+	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	VersionID          string
@@ -66,11 +71,6 @@ type LogFields struct {
 	TLSVersion         string
 	AccessPointARN     string
 	AclRequired        string
-	HttpStatus         int
-	BytesSent          int
-	ObjectSize         int64
-	TotalTime          int64
-	TurnAroundTime     int64
 }
 
 type AdminLogFields struct {
@@ -80,17 +80,17 @@ type AdminLogFields struct {
 	RequestID          string
 	Operation          string
 	RequestURI         string
+	HttpStatus         int
 	ErrorCode          string
+	BytesSent          int
+	TotalTime          int64
+	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	SignatureVersion   string
 	CipherSuite        string
 	AuthenticationType string
 	TLSVersion         string
-	HttpStatus         int
-	BytesSent          int
-	TotalTime          int64
-	TurnAroundTime     int64
 }
 
 type Loggers struct {

s3log/file.go

@@ -34,10 +34,10 @@ const (
 
 // FileLogger is a local file audit log
 type FileLogger struct {
-	f       *os.File
 	logfile string
-	mu      sync.Mutex
+	f       *os.File
 	gotErr  bool
+	mu      sync.Mutex
 }
 
 var _ AuditLogger = &FileLogger{}

s3log/webhook.go

@@ -33,8 +33,8 @@ import (
 
 // WebhookLogger is a webhook URL audit log
 type WebhookLogger struct {
-	url string
 	mu  sync.Mutex
+	url string
 }
 
 var _ AuditLogger = &WebhookLogger{}

s3response/s3response.go

@@ -29,23 +29,34 @@ const (
 )
 
 type PutObjectOutput struct {
-	ETag      string
-	VersionID string
+	ETag              string
+	VersionID         string
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+	ChecksumType      types.ChecksumType
 }
 
 // Part describes part metadata.
 type Part struct {
-	LastModified time.Time
-	ETag         string
-	PartNumber   int
-	Size         int64
+	PartNumber        int
+	LastModified      time.Time
+	ETag              string
+	Size              int64
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
 }
 
 func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Part
 	aux := &struct {
-		*Alias
 		LastModified string `xml:"LastModified"`
+		*Alias
 	}{
 		Alias: (*Alias)(&p),
 	}
@@ -59,23 +70,25 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
+	Bucket            string
+	Key               string
+	UploadID          string `xml:"UploadId"`
+	ChecksumAlgorithm types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
+
 	Initiator Initiator
 	Owner     Owner
 
-	Bucket   string
-	Key      string
-	UploadID string `xml:"UploadId"`
-
 	// The class of storage used to store the object.
 	StorageClass types.StorageClass
 
-	// List of parts.
-	Parts []Part `xml:"Part"`
-
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
 }
 
 type ObjectAttributes string
@@ -97,23 +110,24 @@ func (o ObjectAttributes) IsValid() bool {
 }
 
 type GetObjectAttributesResponse struct {
-	ETag        *string
-	ObjectSize  *int64
-	ObjectParts *ObjectParts
+	ETag         *string
+	ObjectSize   *int64
+	StorageClass types.StorageClass `xml:",omitempty"`
+	ObjectParts  *ObjectParts
+	Checksum     *types.Checksum
 
 	// Not included in the response body
 	VersionId    *string
 	LastModified *time.Time
 	DeleteMarker *bool
-	StorageClass types.StorageClass `xml:",omitempty"`
 }
 
 type ObjectParts struct {
-	Parts                []types.ObjectPart `xml:"Part"`
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
+	Parts                []types.ObjectPart `xml:"Part"`
 }
 
 // ListMultipartUploadsResponse - s3 api list multipart uploads response.
@@ -128,17 +142,18 @@ type ListMultipartUploadsResult struct {
 	Delimiter          string
 	Prefix             string
 	EncodingType       string `xml:"EncodingType,omitempty"`
+	MaxUploads         int
+	IsTruncated        bool
 
 	// List of pending uploads.
 	Uploads []Upload `xml:"Upload"`
 
 	// Delimed common prefixes.
 	CommonPrefixes []CommonPrefix
-	MaxUploads     int
-	IsTruncated    bool
 }
 
 type ListObjectsResult struct {
+	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name           *string
 	Prefix         *string
 	Marker         *string
@@ -146,13 +161,13 @@ type ListObjectsResult struct {
 	MaxKeys        *int32
 	Delimiter      *string
 	IsTruncated    *bool
-	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-	EncodingType   types.EncodingType
 	Contents       []Object
 	CommonPrefixes []types.CommonPrefix
+	EncodingType   types.EncodingType
 }
 
 type ListObjectsV2Result struct {
+	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name                  *string
 	Prefix                *string
 	StartAfter            *string
@@ -162,20 +177,21 @@ type ListObjectsV2Result struct {
 	MaxKeys               *int32
 	Delimiter             *string
 	IsTruncated           *bool
-	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-	EncodingType          types.EncodingType
 	Contents              []Object
 	CommonPrefixes        []types.CommonPrefix
+	EncodingType          types.EncodingType
 }
 
 type Object struct {
-	ETag          *string
-	Key           *string
-	LastModified  *time.Time
-	Owner         *types.Owner
-	RestoreStatus *types.RestoreStatus
-	Size          *int64
-	StorageClass  types.ObjectStorageClass
+	ChecksumAlgorithm []types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
+	ETag              *string
+	Key               *string
+	LastModified      *time.Time
+	Owner             *types.Owner
+	RestoreStatus     *types.RestoreStatus
+	Size              *int64
+	StorageClass      types.ObjectStorageClass
 }
 
 func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -197,19 +213,21 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 // Upload describes in progress multipart upload
 type Upload struct {
-	Initiated    time.Time
-	Initiator    Initiator
-	Owner        Owner
-	Key          string
-	UploadID     string `xml:"UploadId"`
-	StorageClass types.StorageClass
+	Key               string
+	UploadID          string `xml:"UploadId"`
+	Initiator         Initiator
+	Owner             Owner
+	StorageClass      types.StorageClass
+	Initiated         time.Time
+	ChecksumAlgorithm types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
 }
 
 func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Upload
 	aux := &struct {
-		*Alias
 		Initiated string `xml:"Initiated"`
+		*Alias
 	}{
 		Alias: (*Alias)(&u),
 	}
@@ -261,11 +279,11 @@ type DeleteResult struct {
 }
 type SelectObjectContentPayload struct {
 	Expression          *string
+	ExpressionType      types.ExpressionType
 	RequestProgress     *types.RequestProgress
 	InputSerialization  *types.InputSerialization
 	OutputSerialization *types.OutputSerialization
 	ScanRange           *types.ScanRange
-	ExpressionType      types.ExpressionType
 }
 
 type SelectObjectContentResult struct {
@@ -283,30 +301,30 @@ type Bucket struct {
 
 type ListBucketsInput struct {
 	Owner             string
+	IsAdmin           bool
 	ContinuationToken string
 	Prefix            string
 	MaxBuckets        int32
-	IsAdmin           bool
 }
 
 type ListAllMyBucketsResult struct {
 	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
 	Owner             CanonicalUser
+	Buckets           ListAllMyBucketsList
 	ContinuationToken string `xml:"ContinuationToken,omitempty"`
 	Prefix            string `xml:"Prefix,omitempty"`
-	Buckets           ListAllMyBucketsList
 }
 
 type ListAllMyBucketsEntry struct {
-	CreationDate time.Time
 	Name         string
+	CreationDate time.Time
 }
 
 func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias ListAllMyBucketsEntry
 	aux := &struct {
-		*Alias
 		CreationDate string `xml:"CreationDate"`
+		*Alias
 	}{
 		Alias: (*Alias)(&r),
 	}
@@ -332,11 +350,25 @@ type CopyObjectResult struct {
 	CopySourceVersionId string `xml:"-"`
 }
 
+type CopyPartResult struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
+	LastModified      time.Time
+	ETag              *string
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+
+	// not included in the body
+	CopySourceVersionId string `xml:"-"`
+}
+
 func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias CopyObjectResult
 	aux := &struct {
-		*Alias
 		LastModified string `xml:"LastModified"`
+		*Alias
 	}{
 		Alias: (*Alias)(&r),
 	}
@@ -404,15 +436,15 @@ type ListVersionsResult struct {
 }
 
 type GetBucketVersioningOutput struct {
+	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
 	MFADelete *types.MFADeleteStatus
 	Status    *types.BucketVersioningStatus
-	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
 }
 
 type PutObjectRetentionInput struct {
-	RetainUntilDate AmzDate
 	XMLName         xml.Name `xml:"Retention"`
 	Mode            types.ObjectLockRetentionMode
+	RetainUntilDate AmzDate
 }
 
 type AmzDate struct {
@@ -467,3 +499,14 @@ func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
 type ListBucketsResult struct {
 	Buckets []Bucket
 }
+
+type Checksum struct {
+	Algorithm types.ChecksumAlgorithm
+	Type      types.ChecksumType
+
+	CRC32     *string
+	CRC32C    *string
+	SHA1      *string
+	SHA256    *string
+	CRC64NVME *string
+}

s3select/message-handler.go

@@ -204,6 +204,7 @@ func genErrorMessage(errorCode, errorMessage string) []byte {
 type GetProgress func() (bytesScanned int64, bytesProcessed int64)
 
 type MessageHandler struct {
+	sync.Mutex
 	ctx           context.Context
 	cancel        context.CancelFunc
 	writer        *bufio.Writer
@@ -212,7 +213,6 @@ type MessageHandler struct {
 	stopCh        chan bool
 	resetCh       chan bool
 	bytesReturned int64
-	sync.Mutex
 }
 
 // NewMessageHandler creates a new MessageHandler instance and starts the event streaming

tests/.env.docker.default

@@ -25,4 +25,5 @@ USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 RECREATE_BUCKETS=true
-REMOVE_TEST_FILE_FOLDER=true
+REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning

tests/Dockerfile_test_bats

@@ -3,8 +3,9 @@ FROM ubuntu:latest
 ARG DEBIAN_FRONTEND=noninteractive
 ARG SECRETS_FILE=tests/.secrets
 ARG CONFIG_FILE=tests/.env.docker
-ARG GO_LIBRARY=go1.23.1.linux-arm64.tar.gz
-ARG AWS_CLI=awscli-exe-linux-aarch64.zip
+ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz
+# see https://github.com/versity/versitygw/issues/1034
+ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip
 ARG MC_FOLDER=linux-arm64
 
 ENV TZ=Etc/UTC
@@ -85,6 +86,7 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
 
 ENV WORKSPACE=.
 ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
+#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
 
 ENTRYPOINT ["tests/run.sh"]
 CMD ["s3api,s3,s3cmd,mc,rest"]

tests/README.md

@@ -1,5 +1,19 @@
 # Command-Line Tests
 
+## Table of Contents
+
+[Instructions - Running Locally](#instructions---running-locally)<br>
+[* Posix Backend](#posix-backend)<br>
+[* Static Bucket Mode](#static-bucket-mode)<br>
+[* S3 Backend](#s3-backend)<br>
+[* Direct Mode](#direct-mode)<br>
+[Instructions - Running With Docker](#instructions---running-with-docker)<br>
+[Instructions - Running With Docker-Compose](#instructions---running-with-docker-compose)<br>
+[Environment Parameters](#environment-parameters)<br>
+[* Secret](#secret)<br>
+[* Non-Secret](#non-secret)<br>
+[REST Scripts](#rest-scripts)<br>
+
 ## Instructions - Running Locally
 
 ### Posix Backend
@@ -61,10 +75,11 @@ To communicate directly with s3, in order to compare the gateway results to dire
 1.  Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
 2.  By default, the dockerfile uses the **arm** architecture (usually modern Mac).  If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="<param>=<amd library or folder>"`  Also, you can determine which is used by your OS with `uname -a`.
 3.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
+4.  To run the entire suite, run `docker run -it <image name>`.  To run an individual suite, pass in the name of the suite as defined in `tests/run.sh` (e.g. REST tests -> `docker run -it <image name> rest`).  Also, multiple specific suites can be run, if separated by comma.
 
 ## Instructions - Running with docker-compose
 
-A file named `docker-compose-bats.yml` is provided in the root folder.  Four configurations are provided:
+A file named `docker-compose-bats.yml` is provided in the root folder.  A few configurations are provided, and you can also create your own provided you have a secrets and config file:
 * insecure (without certificates), with creation/removal of buckets
 * secure, posix backend, with static buckets
 * secure, posix backend, with creation/removal of buckets
@@ -85,8 +100,18 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 ## Environment Parameters
 
+### Secret
+
 **AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**:  identical to the same parameters in **s3**.
 
+**AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the main user (owner)
+
+**ACL_AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the user to test ACL changes and access by non-owners
+
+**ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**:  for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account.
+
+### Non-Secret
+
 **VERSITY_EXE**:  location of the versity executable relative to test folder.
 
 **RUN_VERSITYGW**:  whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
@@ -134,3 +159,21 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 **VERSIONING_DIR**:  where to put gateway file versioning info.
 
 **COMMAND_LOG**:  where to store list of client commands, which if using will be reported during test failures.
+
+**TIME_LOG**:  optional log to show duration of individual tests
+
+**DIRECT_S3_ROOT_ACCOUNT_NAME**:  for direct mode, S3 username
+
+**DELETE_BUCKETS_AFTER_TEST**:  whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked
+
+## REST Scripts
+
+REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder.  To call a script, the following parameters are needed:
+* **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**, etc.
+* **AWS_ENDPOINT_URL** (default:  `https://localhost:7070`)
+* **OUTPUT_FILE**:  file where the command's response data is written
+* Any other parameters specified at the top of the script file, such as payloads and variables.  Sometimes, defaults are included.
+
+Upon success, the script will return a response code, and write the data to the **OUTPUT_FILE** location.
+
+Example:  `AWS_ACCESS_KEY_ID={id} AWS_SECRET_ACCESS_KEY={key} AWS_ENDPOINT_URL=https://s3.amazonaws.com OUTPUT_FILE=./output_file.xml ./tests/rest_scripts/list_buckets.sh`

tests/commands/copy_object.sh

@@ -16,7 +16,7 @@
 
 copy_object() {
   if [ $# -ne 4 ]; then
-    echo "copy object command requires command type, source, bucket, key"
+    log 2 "copy object command requires command type, source, bucket, key"
     return 1
   fi
   local exit_code=0
@@ -24,7 +24,7 @@ copy_object() {
   record_command "copy-object" "client:$1"
   if [[ $1 == 's3' ]]; then
     error=$(send_command aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+  elif [[ $1 == 's3api' ]]; then
     error=$(send_command aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4"
@@ -32,12 +32,12 @@ copy_object() {
   elif [[ $1 == 'mc' ]]; then
     error=$(send_command mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
   else
-    echo "'copy-object' not implemented for '$1'"
+    log 2 "'copy-object' not implemented for '$1'"
     return 1
   fi
   log 5 "copy object exit code: $exit_code"
   if [ $exit_code -ne 0 ]; then
-    echo "error copying object to bucket: $error"
+    log 2 "error copying object to bucket: $error"
     return 1
   fi
   return 0

tests/commands/create_bucket.sh

@@ -31,7 +31,7 @@ create_bucket() {
   log 6 "create bucket"
   if [[ $1 == 's3' ]]; then
     error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
-  elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
+  elif [[ $1 == 's3api' ]]; then
     error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"

tests/commands/create_multipart_upload.sh

@@ -54,6 +54,7 @@ create_multipart_upload_with_user() {
     return 1
   fi
   upload_id="${upload_id//\"/}"
+  echo "$upload_id"
   return 0
 }
 

tests/commands/create_presigned_url.sh

@@ -7,7 +7,7 @@ create_presigned_url() {
   fi
 
   local presign_result=0
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 's3api' ]]; then
     presigned_url=$(send_command aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     presigned_url=$(send_command s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?

tests/commands/delete_bucket.sh

@@ -32,7 +32,7 @@ delete_bucket() {
   exit_code=0
   if [[ $1 == 's3' ]]; then
     error=$(send_command aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
-  elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+  elif [[ $1 == 's3api' ]]; then
     error=$(send_command aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?

tests/commands/delete_bucket_policy.sh

@@ -21,7 +21,7 @@ delete_bucket_policy() {
     return 1
   fi
   local delete_result=0
-  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+  if [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
     error=$(send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?

tests/commands/delete_bucket_tagging.sh

@@ -21,7 +21,7 @@ delete_bucket_tagging() {
     return 1
   fi
   local result
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 's3api' ]]; then
     tags=$(send_command aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
     tags=$(send_command mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?

tests/commands/delete_object.sh

@@ -25,7 +25,7 @@ delete_object() {
   local exit_code=0
   if [[ $1 == 's3' ]]; then
     delete_object_error=$(send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+  elif [[ $1 == 's3api' ]]; then
     delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?

tests/commands/delete_object_tagging.sh

@@ -17,22 +17,22 @@
 delete_object_tagging() {
   record_command "delete-object-tagging" "client:$1"
   if [[ $# -ne 3 ]]; then
-    echo "delete object tagging command missing command type, bucket, key"
+    log 2 "delete object tagging command missing command type, bucket, key"
     return 1
   fi
   delete_result=0
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 's3api' ]]; then
     error=$(send_command aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
     error=$(send_command mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
   elif [ "$1" == 'rest' ]; then
     delete_object_tagging_rest "$2" "$3" || delete_result=$?
   else
-    echo "delete-object-tagging command not implemented for '$1'"
+    log 2 "delete-object-tagging command not implemented for '$1'"
     return 1
   fi
   if [[ $delete_result -ne 0 ]]; then
-    echo "error deleting object tagging: $error"
+    log 2 "error deleting object tagging: $error"
     return 1
   fi
   return 0
@@ -43,38 +43,12 @@ delete_object_tagging_rest() {
     log 2 "'delete_object_tagging' requires bucket, key"
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="DELETE
-/$1/$2
-tagging=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/delete_object_tagging.sh); then
+    log 2 "error sending delete object tagging REST command: $result"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" \
-    -d "$tagging" -o "$TEST_FILE_FOLDER"/delete_tagging_error.txt 2>&1)
-  log 5 "reply status code: $reply"
-  if [[ "$reply" != "204" ]]; then
-    log 2 "reply error: $reply"
-    log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/delete_tagging_error.txt)"
+  if [ "$result" != "204" ]; then
+    log 2 "delete-object-tagging returned code $result (response: $(cat "$TEST_FILE_FOLDER/response.txt"))"
     return 1
   fi
   return 0

tests/commands/get_bucket_acl.sh

@@ -21,7 +21,7 @@ get_bucket_acl() {
     return 1
   fi
   local exit_code=0
-  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+  if [[ $1 == 's3api' ]]; then
     acl=$(send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
     acl=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"

tests/commands/get_bucket_location.sh

@@ -17,17 +17,18 @@
 get_bucket_location() {
   record_command "get-bucket-location" "client:$1"
   if [[ $# -ne 2 ]]; then
-    echo "get bucket location command requires command type, bucket name"
+    log 2 "get bucket location command requires command type, bucket name"
     return 1
   fi
-  if [[ $1 == 'aws' ]]; then
+  get_result=0
+  if [[ $1 == 's3api' ]]; then
     get_bucket_location_aws "$2" || get_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_location_s3cmd "$2" || get_result=$?
   elif [[ $1 == 'mc' ]]; then
     get_bucket_location_mc "$2" || get_result=$?
   else
-    echo "command type '$1' not implemented for get_bucket_location"
+    log 2 "command type '$1' not implemented for get_bucket_location"
     return 1
   fi
   if [[ $get_result -ne 0 ]]; then
@@ -39,7 +40,7 @@ get_bucket_location() {
 get_bucket_location_aws() {
   record_command "get-bucket-location" "client:s3api"
   if [[ $# -ne 1 ]]; then
-    echo "get bucket location (aws) requires bucket name"
+    log 2 "get bucket location (aws) requires bucket name"
     return 1
   fi
   location_json=$(send_command aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
@@ -59,7 +60,7 @@ get_bucket_location_s3cmd() {
   fi
   info=$(send_command s3cmd --no-check-certificate info "s3://$1") || results=$?
   if [[ $results -ne 0 ]]; then
-    echo "error getting s3cmd info: $info"
+    log 2 "error getting bucket location: $location"
     return 1
   fi
   bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}')
@@ -69,12 +70,12 @@ get_bucket_location_s3cmd() {
 get_bucket_location_mc() {
   record_command "get-bucket-location" "client:mc"
   if [[ $# -ne 1 ]]; then
-    echo "get bucket location (mc) requires bucket name"
+    log 2 "get bucket location (mc) requires bucket name"
     return 1
   fi
   info=$(send_command mc --insecure stat "$MC_ALIAS/$1") || results=$?
   if [[ $results -ne 0 ]]; then
-    echo "error getting s3cmd info: $info"
+    log 2 "error getting s3cmd info: $info"
     return 1
   fi
   # shellcheck disable=SC2034

tests/commands/get_bucket_ownership_controls.sh

@@ -26,6 +26,7 @@ get_bucket_ownership_controls() {
     return 1
   fi
 
+  raw_bucket_ownership_controls=""
   if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
     log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
     return 1

tests/commands/get_bucket_policy.sh

@@ -21,7 +21,7 @@ get_bucket_policy() {
     return 1
   fi
   local get_bucket_policy_result=0
-  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+  if [[ $1 == 's3api' ]]; then
     get_bucket_policy_aws "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
@@ -97,41 +97,57 @@ get_bucket_policy_s3cmd() {
   policy_brackets=false
   # NOTE:  versitygw sends policies back in multiple lines here, direct in single line
   while IFS= read -r line; do
-    if [[ $policy_brackets == false ]]; then
-      policy_line=$(echo "$line" | grep 'Policy: ')
-      if [[ $policy_line != "" ]]; then
-        if [[ $policy_line != *'{'* ]]; then
-          break
-        fi
-        if [[ $policy_line == *'}'* ]]; then
-          log 5 "policy on single line"
-          bucket_policy=${policy_line//Policy:/}
-          break
-        else
-          policy_brackets=true
-          bucket_policy+="{"
-        fi
-      fi
-    else
-      bucket_policy+=$line
-      if [[ $line == "" ]]; then
-        break
-      fi
+    if check_and_load_policy_info; then
+      break
     fi
   done <<< "$info"
   log 5 "bucket policy: $bucket_policy"
   return 0
 }
 
+# return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading
+check_and_load_policy_info() {
+  if [[ $policy_brackets == false ]]; then
+    if search_for_first_policy_line_or_full_policy; then
+      return 0
+    fi
+  else
+    bucket_policy+=$line
+    if [[ $line == "}" ]]; then
+      return 0
+    fi
+  fi
+  return 1
+}
+
+# return 0 for empty or single-line policy, 1 for other cases
+search_for_first_policy_line_or_full_policy() {
+  policy_line=$(echo "$line" | grep 'Policy: ')
+  if [[ $policy_line != "" ]]; then
+    if [[ $policy_line != *'{'* ]]; then
+      return 0
+    fi
+    if [[ $policy_line == *'}'* ]]; then
+      log 5 "policy on single line"
+      bucket_policy=${policy_line//Policy:/}
+      return 0
+    else
+      policy_brackets=true
+      bucket_policy+="{"
+    fi
+  fi
+  return 1
+}
+
 get_bucket_policy_mc() {
   record_command "get-bucket-policy" "client:mc"
   if [[ $# -ne 1 ]]; then
-    echo "aws 'get bucket policy' command requires bucket"
+    log 2 "aws 'get bucket policy' command requires bucket"
     return 1
   fi
   bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
   if [[ $get_result -ne 0 ]]; then
-    echo "error getting policy: $bucket_policy"
+    log 2 "error getting policy: $bucket_policy"
     return 1
   fi
   return 0

tests/commands/get_bucket_tagging.sh

@@ -21,7 +21,7 @@ get_bucket_tagging() {
   assert [ $# -eq 2 ]
   record_command "get-bucket-tagging" "client:$1"
   local result
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 's3api' ]]; then
     tags=$(send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
     tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
@@ -35,7 +35,7 @@ get_bucket_tagging() {
       export tags=
       return 0
     fi
-    echo "error getting bucket tags: $tags"
+    log 2 "error getting bucket tags: $tags"
     return 1
   fi
   export tags

tests/commands/get_object.sh

@@ -24,7 +24,7 @@ get_object() {
   local exit_code=0
   if [[ $1 == 's3' ]]; then
     get_object_error=$(send_command aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
-  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+  elif [[ $1 == 's3api' ]]; then
     get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?

tests/commands/get_object_lock_configuration.sh

@@ -36,35 +36,12 @@ get_object_lock_configuration_rest() {
     log 2 "'get_object_lock_configuration_rest' requires bucket name"
     return 1
   fi
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-object-lock=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object-lock-config.txt" ./tests/rest_scripts/get_object_lock_config.sh); then
+    log 2 "error getting lock configuration: $result"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
+  if [[ "$result" != "200" ]]; then
+    log 2 "expected '200', returned '$result': $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
     return 1
   fi
   return 0

tests/commands/get_object_tagging.sh

@@ -21,7 +21,7 @@ get_object_tagging() {
     return 1
   fi
   local result
-  if [[ "$1" == 'aws' ]] || [[ $1 == 's3api' ]]; then
+  if [[ $1 == 's3api' ]]; then
     tags=$(send_command aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
   elif [[ "$1" == 'mc' ]]; then
     tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?

tests/commands/head_bucket.sh

@@ -29,14 +29,14 @@ head_bucket() {
     return 1
   fi
   local exit_code=0
-  if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+  if [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
     bucket_info=$(send_command aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
     bucket_info=$(send_command s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
     bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
-    fail "invalid command type $1"
+    log 2 "invalid command type $1"
   fi
   if [ $exit_code -ne 0 ]; then
     if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
@@ -45,5 +45,7 @@ head_bucket() {
     log 2 "error getting bucket info: $bucket_info"
     return 2
   fi
+  bucket_info="$(echo -n "$bucket_info" | grep -v "InsecureRequestWarning")"
+  echo "$bucket_info"
   return 0
 }

tests/commands/head_object.sh

@@ -21,7 +21,7 @@ head_object() {
     return 2
   fi
   local exit_code=0
-  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+  if [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
     metadata=$(send_command aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
     metadata=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?"

tests/commands/list_buckets.sh

@@ -18,14 +18,14 @@ list_buckets() {
   log 6 "list_buckets"
   record_command "list-buckets" "client:$1"
   if [ $# -ne 1 ]; then
-    echo "list buckets command missing command type"
+    log 2 "list buckets command missing command type"
     return 1
   fi
 
   local exit_code=0
   if [[ $1 == 's3' ]]; then
     buckets=$(send_command aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
-  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+  elif [[ $1 == 's3api' ]]; then
     list_buckets_s3api "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     buckets=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
@@ -34,11 +34,11 @@ list_buckets() {
   elif [[ $1 == 'rest' ]]; then
     list_buckets_rest || exit_code=$?
   else
-    echo "list buckets command not implemented for '$1'"
+    log 2 "list buckets command not implemented for '$1'"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    echo "error listing buckets: $buckets"
+    log 2 "error listing buckets: $buckets"
     return 1
   fi
 
@@ -57,7 +57,7 @@ list_buckets() {
 list_buckets_with_user() {
   record_command "list-buckets" "client:$1"
   if [ $# -ne 3 ]; then
-    echo "'list buckets as user' command missing command type, username, password"
+    log 2 "'list buckets as user' command missing command type, username, password"
     return 1
   fi
 
@@ -71,11 +71,11 @@ list_buckets_with_user() {
   elif [[ $1 == 'mc' ]]; then
     buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
   else
-    echo "list buckets command not implemented for '$1'"
+    log 2 "list buckets command not implemented for '$1'"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    echo "error listing buckets: $buckets"
+    log 2 "error listing buckets: $buckets"
     return 1
   fi
 
@@ -97,7 +97,7 @@ list_buckets_s3api() {
     return 1
   fi
   if ! output=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api list-buckets 2>&1); then
-    echo "error listing buckets: $output"
+    log 2 "error listing buckets: $output"
     return 1
   fi
   log 5 "bucket data: $output"

tests/commands/list_object_versions.sh

@@ -34,30 +34,10 @@ list_object_versions_rest() {
     log 2 "'list_object_versions_rest' requires bucket name"
     return 1
   fi
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-versions=
-host:${AWS_ENDPOINT_URL#*//}
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "list object versions REST"
+  if ! result=$(BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object_versions.txt" ./tests/rest_scripts/list_object_versions.sh); then
+    log 2 "error listing object versions: $result"
     return 1
   fi
-
-  get_signature
-  # shellcheck disable=SC2034,SC2154
-  reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \
-         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-         -H "x-amz-content-sha256: $payload_hash" \
-         -H "x-amz-date: $current_date_time" \
-         -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1)
+  return 0
 }

tests/commands/list_objects.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-source ./tests/util_list_objects.sh
+source ./tests/util/util_list_objects.sh
 source ./tests/commands/command.sh
 
 # Copyright 2024 Versity Software
@@ -29,7 +29,7 @@ list_objects() {
 
   local output
   local result=0
-  if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then
+  if [[ $1 == 's3' ]]; then
     output=$(send_command aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$?
   elif [[ $1 == 's3api' ]]; then
     list_objects_s3api "$2" || result=$?
@@ -91,7 +91,7 @@ list_objects_s3api() {
 # export objects on success, return 1 for failure
 list_objects_s3api_v1() {
   if [ $# -lt 1 ] || [ $# -gt 2 ]; then
-    echo "list objects command requires bucket, (optional) delimiter"
+    log 2 "list objects command requires bucket, (optional) delimiter"
     return 1
   fi
   if [ "$2" == "" ]; then
@@ -100,7 +100,7 @@ list_objects_s3api_v1() {
     objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
   fi
   if [[ $result -ne 0 ]]; then
-    echo "error listing objects: $objects"
+    log 2 "error listing objects: $objects"
     return 1
   fi
   export objects
@@ -138,35 +138,19 @@ list_objects_rest() {
     log 2 "'list_objects_rest' requires bucket name"
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  log 5 "canonical request: $canonical_request"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "bucket name: $1"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error listing objects: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))"
+    return 1
+  fi
+  # shellcheck disable=SC2034
+  reply=$(cat "$TEST_FILE_FOLDER/objects.txt")
+  if ! parse_objects_list_rest; then
+    log 2 "error parsing list objects"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" 2>&1)
-  log 5 "reply: $reply"
-  parse_objects_list_rest
 }

tests/commands/list_objects_v2.sh

@@ -19,13 +19,13 @@
 # export objects on success, return 1 for failure
 list_objects_v2() {
   if [ $# -ne 1 ]; then
-    echo "list objects command missing bucket and/or path"
+    log 2 "list objects command missing bucket and/or path"
     return 1
   fi
   record_command "list-objects-v2 client:s3api"
   objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$?
   if [[ $result -ne 0 ]]; then
-    echo "error listing objects: $objects"
+    log 2 "error listing objects: $objects"
     return 1
   fi
 }

tests/commands/put_bucket_acl.sh

@@ -14,7 +14,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-source ./tests/util_file.sh
+source ./tests/util/util_file.sh
 source ./tests/commands/command.sh
 
 put_bucket_acl_s3api() {
@@ -24,7 +24,7 @@ put_bucket_acl_s3api() {
     log 2 "put bucket acl command requires bucket name, acl file"
     return 1
   fi
-  log 5 "bucket name: $1, acls: $2"
+  log 5 "bucket name: $1, acls: $(cat "$2")"
   if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
     log 2 "error putting bucket acl: $error"
     return 1
@@ -58,22 +58,15 @@ reset_bucket_acl() {
     return 1
   fi
   # shellcheck disable=SC2154
-  cat <<EOF > "$TEST_FILE_FOLDER/$acl_file"
-{
-  "Grants": [
-    {
-      "Grantee": {
-        "ID": "$AWS_ACCESS_KEY_ID",
-        "Type": "CanonicalUser"
-      },
-      "Permission": "FULL_CONTROL"
-    }
-  ],
-  "Owner": {
-    "ID": "$AWS_ACCESS_KEY_ID"
-  }
-}
-EOF
+  if [ "$DIRECT" != "true" ]; then
+    if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
+      log 2 "error resetting versitygw ACL"
+      return 1
+    fi
+  elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
+    log 2 "error resetting direct ACL"
+    return 1
+  fi
   if ! put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$acl_file"; then
     log 2 "error putting bucket acl (s3api)"
     return 1

tests/commands/put_bucket_policy.sh

@@ -21,7 +21,7 @@ put_bucket_policy() {
     return 1
   fi
   local put_policy_result=0
-  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+  if [[ $1 == 's3api' ]]; then
     policy=$(send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     policy=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$?