Merge pull request #663 from versity/test_cmdline_acl_correction

Test cmdline acl correction

.github/workflows/shellcheck.yml

@@ -0,0 +1,16 @@
+name: shellcheck
+on: pull_request
+jobs:
+
+  build:
+    name: Run shellcheck
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Run checks
+      run: |
+        shellcheck --version
+        shellcheck -e SC1091 tests/*.sh tests/*/*.sh

.github/workflows/system.yml

@@ -4,16 +4,91 @@ jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          #- set: 1
+          #  LOCAL_FOLDER: /tmp/gw1
+          #  BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
+          #  BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+          #  IAM_TYPE: folder
+          #  USERS_FOLDER: /tmp/iam1
+          #  AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          #  RUN_SET: "s3cmd"
+          #  RECREATE_BUCKETS: "true"
+          #  PORT: 7070
+          #  BACKEND: "posix"
+          - set: 2
+            LOCAL_FOLDER: /tmp/gw2
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam2
+            AWS_ENDPOINT_URL: https://127.0.0.1:7071
+            RUN_SET: "s3"
+            RECREATE_BUCKETS: "true"
+            PORT: 7071
+            BACKEND: "posix"
+          - set: 3
+            LOCAL_FOLDER: /tmp/gw3
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam3
+            AWS_ENDPOINT_URL: https://127.0.0.1:7072
+            RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
+            PORT: 7072
+            BACKEND: "posix"
+          - set: 4
+            LOCAL_FOLDER: /tmp/gw4
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam4
+            AWS_ENDPOINT_URL: https://127.0.0.1:7073
+            RUN_SET: "mc"
+            RECREATE_BUCKETS: "true"
+            PORT: 7073
+            BACKEND: "posix"
+          - set: 5
+            LOCAL_FOLDER: /tmp/gw5
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
+            IAM_TYPE: s3
+            USERS_BUCKET: versity-gwtest-iam
+            AWS_ENDPOINT_URL: https://127.0.0.1:7074
+            RUN_SET: "aws-user"
+            RECREATE_BUCKETS: "true"
+            PORT: 7074
+            BACKEND: "posix"
+          - set: 6
+            LOCAL_FOLDER: /tmp/gw6
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam6
+            AWS_ENDPOINT_URL: https://127.0.0.1:7075
+            RUN_SET: "aws"
+            RECREATE_BUCKETS: "false"
+            PORT: 7075
+            BACKEND: "posix"
+          - set: 7
+            LOCAL_FOLDER: /tmp/gw7
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam7
+            AWS_ENDPOINT_URL: https://127.0.0.1:7076
+            RUN_SET: "aws"
+            RECREATE_BUCKETS: "true"
+            PORT: 7076
+            BACKEND: "s3"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
 
-      - name: Install ShellCheck and md5
-        run: sudo apt-get install shellcheck
-
-      - name: Run ShellCheck
-        run: shellcheck -S warning ./tests/*.sh
-
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -39,34 +114,46 @@ jobs:
           chmod 755 /usr/local/bin/mc
 
       - name: Build and run, posix backend
+        env:
+          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
+          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
+          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
+          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
+          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
+          IAM_TYPE: ${{ matrix.IAM_TYPE }}
+          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
+          RUN_SET: ${{ matrix.RUN_SET }}
+          PORT: ${{ matrix.PORT }}
+          AWS_PROFILE: versity
+          VERSITY_EXE: ${{ github.workspace }}/versitygw
+          RUN_VERSITYGW: true
+          BACKEND: ${{ matrix.BACKEND }}
+          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
+          CERT: ${{ github.workspace }}/cert.pem
+          KEY: ${{ github.workspace }}/versitygw.pem
+          S3CMD_CONFIG: tests/s3cfg.local.default
+          MC_ALIAS: versity
+          LOG_LEVEL: 4
+          GOCOVERDIR: ${{ github.workspace }}/cover
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
+          export AWS_ACCESS_KEY_ID_TWO=user
+          export AWS_SECRET_ACCESS_KEY_TWO=pass
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity
-          mkdir /tmp/gw
+          mkdir $LOCAL_FOLDER
           export WORKSPACE=$GITHUB_WORKSPACE
-          openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
-          openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir cover iam
-          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
-
-      #- name: Build and run, s3 backend
-      #  run: |
-      #    make testbin
-      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export AWS_REGION=us-east-1
-      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
-      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
-      #    aws configure set aws_region $AWS_REGION --profile versity_s3
-      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export WORKSPACE=$GITHUB_WORKSPACE
-      #    VERSITYGW_TEST_ENV=./tests/.env.s3 GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          mkdir $GOCOVERDIR $USERS_FOLDER
+          if [[ $RECREATE_BUCKETS == "false" ]]; then
+            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
+          fi
+          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
       - name: Coverage report
         run: |

README.md

@@ -6,7 +6,7 @@
   <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
 </picture>
 
- [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
 
 ### Binary release builds
 Download [latest release](https://github.com/versity/versitygw/releases)
@@ -22,8 +22,7 @@ Download [latest release](https://github.com/versity/versitygw/releases)
 * Simplified interface for adding new storage system support
 
 ### News
-* New performance (scale up) analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
-* New performance (scale out) Part 2 analysis article [https://github.com/versity/versitygw/wiki/Performance-Part-2](https://github.com/versity/versitygw/wiki/Performance-Part-2)
+Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
 
 ### Mailing List
 Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).

auth/acl.go

@@ -28,7 +28,6 @@ import (
 )
 
 type ACL struct {
-	ACL      types.BucketCannedACL
 	Owner    string
 	Grantees []Grantee
 }
@@ -94,12 +93,34 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
 	}
 
+	defaultGrantees := []Grantee{
+		{
+			Permission: types.PermissionFullControl,
+			Access:     acl.Owner,
+		},
+	}
+
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
-		acl.ACL = input.ACL
-		acl.Grantees = []Grantee{}
+		switch input.ACL {
+		case types.BucketCannedACLPublicRead:
+			defaultGrantees = append(defaultGrantees, Grantee{
+				Permission: types.PermissionRead,
+				Access:     "all-users",
+			})
+		case types.BucketCannedACLPublicReadWrite:
+			defaultGrantees = append(defaultGrantees, []Grantee{
+				{
+					Permission: types.PermissionRead,
+					Access:     "all-users",
+				},
+				{
+					Permission: types.PermissionWrite,
+					Access:     "all-users",
+				},
+			}...)
+		}
 	} else {
-		grantees := []Grantee{}
 		accs := []string{}
 
 		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -108,31 +129,31 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
 				}
 			}
 			if input.GrantRead != nil && *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "READ"})
 				}
 			}
 			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "READ_ACP"})
 				}
 			}
 			if input.GrantWrite != nil && *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "WRITE"})
 				}
 			}
 			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "WRITE_ACP"})
 				}
 			}
 
@@ -143,7 +164,7 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
 					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 				}
-				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
+				defaultGrantees = append(defaultGrantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
 				if _, ok := cache[*grt.Grantee.ID]; !ok {
 					cache[*grt.Grantee.ID] = true
 					accs = append(accs, *grt.Grantee.ID)
@@ -159,11 +180,10 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		if len(accList) > 0 {
 			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
 		}
-
-		acl.Grantees = grantees
-		acl.ACL = ""
 	}
 
+	acl.Grantees = defaultGrantees
+
 	result, err := json.Marshal(acl)
 	if err != nil {
 		return nil, err
@@ -207,34 +227,21 @@ func splitUnique(s, divider string) []string {
 }
 
 func verifyACL(acl ACL, access string, permission types.Permission) error {
-	if acl.ACL != "" {
-		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
+	grantee := Grantee{Access: access, Permission: permission}
+	granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
+	granteeAllUsers := Grantee{Access: "all-users", Permission: permission}
 
+	isFound := false
+
+	for _, grt := range acl.Grantees {
+		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
+			isFound = true
+			break
+		}
+	}
+
+	if isFound {
 		return nil
-	} else {
-		if len(acl.Grantees) == 0 {
-			return nil
-		}
-		grantee := Grantee{Access: access, Permission: permission}
-		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
-
-		isFound := false
-
-		for _, grt := range acl.Grantees {
-			if grt == grantee || grt == granteeFullCtrl {
-				isFound = true
-				break
-			}
-		}
-
-		if isFound {
-			return nil
-		}
 	}
 
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -295,23 +302,16 @@ func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) e
 	if opts.Acc.Role == RoleAdmin {
 		return nil
 	}
-	if opts.Acc.Access == opts.Acl.Owner {
-		return nil
-	}
 
 	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
-	if policyErr != nil && !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-		return policyErr
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
 	}
 
-	// If bucket policy is not set and the ACL is default, only the owner has access
-	if errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	if err := VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
-		return err
-	}
 	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
 		return err
 	}

auth/bucket_policy.go

@@ -16,12 +16,22 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
+	"errors"
 	"net/http"
 
 	"github.com/versity/versitygw/s3err"
 )
 
+var (
+	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidResource = errors.New("Policy has invalid resource")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidPrincipal = errors.New("Invalid principal in policy")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidAction = errors.New("Policy has invalid action")
+)
+
 type BucketPolicy struct {
 	Statement []BucketPolicyItem `json:"Statement"`
 }
@@ -75,11 +85,14 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 
 	for action := range bpi.Actions {
 		isObjectAction := action.IsObjectAction()
-		if isObjectAction && !containsObjectAction {
-			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		if isObjectAction == nil {
+			break
 		}
-		if !isObjectAction && !containsBucketAction {
-			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		if *isObjectAction && !containsObjectAction {
+			return errResourceMismatch
+		}
+		if !*isObjectAction && !containsBucketAction {
+			return errResourceMismatch
 		}
 	}
 
@@ -108,6 +121,11 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 		return getMalformedPolicyError(err)
 	}
 
+	if len(policy.Statement) == 0 {
+		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
+	}
+
 	if err := policy.Validate(bucket, iam); err != nil {
 		return getMalformedPolicyError(err)
 	}
@@ -116,11 +134,6 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 }
 
 func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
-	// If bucket policy is not set
-	if policy == nil {
-		return nil
-	}
-
 	var bucketPolicy BucketPolicy
 	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
 		return err

auth/bucket_policy_actions.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -56,6 +55,8 @@ const (
 	GetObjectRetentionAction               Action = "s3:GetObjectRetention"
 	PutObjectRetentionAction               Action = "s3:PutObjectRetention"
 	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
+	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
+	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
 	AllActions                             Action = "s3:*"
 )
 
@@ -92,6 +93,8 @@ var supportedActionList = map[Action]struct{}{
 	GetObjectRetentionAction:               {},
 	PutObjectRetentionAction:               {},
 	BypassGovernanceRetentionAction:        {},
+	PutBucketOwnershipControlsAction:       {},
+	GetBucketOwnershipControlsAction:       {},
 	AllActions:                             {},
 }
 
@@ -119,7 +122,7 @@ var supportedObjectActionList = map[Action]struct{}{
 // Validates Action: it should either wildcard match with supported actions list or be in it
 func (a Action) IsValid() error {
 	if !strings.HasPrefix(string(a), "s3:") {
-		return fmt.Errorf("invalid action: %v", a)
+		return errInvalidAction
 	}
 
 	if a == AllActions {
@@ -134,31 +137,39 @@ func (a Action) IsValid() error {
 			}
 		}
 
-		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+		return errInvalidAction
 	}
 
 	_, found := supportedActionList[a]
 	if !found {
-		return fmt.Errorf("unsupported action: %v", a)
+		return errInvalidAction
 	}
 	return nil
 }
 
+func getBoolPtr(bl bool) *bool {
+	return &bl
+}
+
 // Checks if the action is object action
-func (a Action) IsObjectAction() bool {
+// nil points to 's3:*'
+func (a Action) IsObjectAction() *bool {
+	if a == AllActions {
+		return nil
+	}
 	if a[len(a)-1] == '*' {
 		pattern := strings.TrimSuffix(string(a), "*")
 		for act := range supportedObjectActionList {
 			if strings.HasPrefix(string(act), pattern) {
-				return true
+				return getBoolPtr(true)
 			}
 		}
 
-		return false
+		return getBoolPtr(false)
 	}
 
 	_, found := supportedObjectActionList[a]
-	return found
+	return &found
 }
 
 func (a Action) WildCardMatch(act Action) bool {
@@ -177,7 +188,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("actions can't be empty")
+			return errInvalidAction
 		}
 		*a = make(Actions)
 		for _, s := range ss {
@@ -190,7 +201,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("actions can't be empty")
+				return errInvalidAction
 			}
 			*a = make(Actions)
 			err = a.Add(s)

auth/bucket_policy_effect.go

@@ -30,5 +30,6 @@ func (bpat BucketPolicyAccessType) Validate() error {
 		return nil
 	}
 
-	return fmt.Errorf("invalid effect: %v", bpat)
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	return fmt.Errorf("Invalid effect: %v", bpat)
 }

auth/bucket_policy_principals.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 )
 
 type Principals map[string]struct{}
@@ -37,7 +36,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		for _, s := range ss {
@@ -46,7 +45,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &s); err == nil {
 		if s == "" {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(s)
@@ -54,7 +53,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &k); err == nil {
 		if k.AWS == "" {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(k.AWS)
@@ -66,7 +65,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		}
 		if err = json.Unmarshal(data, &sk); err == nil {
 			if len(sk.AWS) == 0 {
-				return fmt.Errorf("principals can't be empty")
+				return errInvalidPrincipal
 			}
 			*p = make(Principals)
 			for _, s := range sk.AWS {
@@ -98,7 +97,7 @@ func (p Principals) Validate(iam IAMService) error {
 		if len(p) == 1 {
 			return nil
 		}
-		return fmt.Errorf("principals should either contain * or user access keys")
+		return errInvalidPrincipal
 	}
 
 	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -106,7 +105,7 @@ func (p Principals) Validate(iam IAMService) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user accounts don't exist: %v", accs)
+		return errInvalidPrincipal
 	}
 
 	return nil

auth/bucket_policy_resources.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -30,7 +29,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("resources can't be empty")
+			return errInvalidResource
 		}
 		*r = make(Resources)
 		for _, s := range ss {
@@ -43,7 +42,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("resources can't be empty")
+				return errInvalidResource
 			}
 			*r = make(Resources)
 			err = r.Add(s)
@@ -60,12 +59,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 func (r Resources) Add(rc string) error {
 	ok, pattern := isValidResource(rc)
 	if !ok {
-		return fmt.Errorf("invalid resource: %v", rc)
-	}
-
-	_, found := r[pattern]
-	if found {
-		return fmt.Errorf("duplicate resource: %v", rc)
+		return errInvalidResource
 	}
 
 	r[pattern] = struct{}{}
@@ -99,7 +93,7 @@ func (r Resources) ContainsBucketPattern() bool {
 func (r Resources) Validate(bucket string) error {
 	for resource := range r {
 		if !strings.HasPrefix(resource, bucket) {
-			return fmt.Errorf("incorrect bucket name in %v", resource)
+			return errInvalidResource
 		}
 	}
 

auth/iam.go

@@ -37,39 +37,75 @@ type Account struct {
 	GroupID int    `json:"groupID"`
 }
 
+// Mutable props, which could be changed when updating an IAM account
+type MutableProps struct {
+	Secret  *string `json:"secret"`
+	UserID  *int    `json:"userID"`
+	GroupID *int    `json:"groupID"`
+}
+
+func updateAcc(acc *Account, props MutableProps) {
+	if props.Secret != nil {
+		acc.Secret = *props.Secret
+	}
+	if props.GroupID != nil {
+		acc.GroupID = *props.GroupID
+	}
+	if props.UserID != nil {
+		acc.UserID = *props.UserID
+	}
+}
+
 // IAMService is the interface for all IAM service implementations
 //
 //go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
 type IAMService interface {
 	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
+	UpdateUserAccount(access string, props MutableProps) error
 	DeleteUserAccount(access string) error
 	ListUserAccounts() ([]Account, error)
 	Shutdown() error
 }
 
-var ErrNoSuchUser = errors.New("user not found")
+var (
+	// ErrUserExists is returned when the user already exists
+	ErrUserExists = errors.New("user already exists")
+	// ErrNoSuchUser is returned when the user does not exist
+	ErrNoSuchUser = errors.New("user not found")
+)
 
 type Opts struct {
-	Dir                string
-	LDAPServerURL      string
-	LDAPBindDN         string
-	LDAPPassword       string
-	LDAPQueryBase      string
-	LDAPObjClasses     string
-	LDAPAccessAtr      string
-	LDAPSecretAtr      string
-	LDAPRoleAtr        string
-	S3Access           string
-	S3Secret           string
-	S3Region           string
-	S3Bucket           string
-	S3Endpoint         string
-	S3DisableSSlVerfiy bool
-	S3Debug            bool
-	CacheDisable       bool
-	CacheTTL           int
-	CachePrune         int
+	Dir                    string
+	LDAPServerURL          string
+	LDAPBindDN             string
+	LDAPPassword           string
+	LDAPQueryBase          string
+	LDAPObjClasses         string
+	LDAPAccessAtr          string
+	LDAPSecretAtr          string
+	LDAPRoleAtr            string
+	LDAPUserIdAtr          string
+	LDAPGroupIdAtr         string
+	VaultEndpointURL       string
+	VaultSecretStoragePath string
+	VaultMountPath         string
+	VaultRootToken         string
+	VaultRoleId            string
+	VaultRoleSecret        string
+	VaultServerCert        string
+	VaultClientCert        string
+	VaultClientCertKey     string
+	S3Access               string
+	S3Secret               string
+	S3Region               string
+	S3Bucket               string
+	S3Endpoint             string
+	S3DisableSSlVerfiy     bool
+	S3Debug                bool
+	CacheDisable           bool
+	CacheTTL               int
+	CachePrune             int
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -82,14 +118,19 @@ func New(o *Opts) (IAMService, error) {
 		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
 		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
-			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
-			o.LDAPObjClasses)
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
+			o.LDAPGroupIdAtr, o.LDAPObjClasses)
 		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
 	case o.S3Endpoint != "":
 		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
 			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
 		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
 			o.S3Endpoint, o.S3Bucket)
+	case o.VaultEndpointURL != "":
+		svc, err = NewVaultIAMService(o.VaultEndpointURL, o.VaultSecretStoragePath,
+			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
+			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
+		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")

auth/iam_cache.go

@@ -66,6 +66,21 @@ func (i *icache) get(k string) (Account, bool) {
 	return v.value, true
 }
 
+func (i *icache) update(k string, props MutableProps) {
+	i.Lock()
+	defer i.Unlock()
+
+	item, found := i.items[k]
+	if found {
+		updateAcc(&item.value, props)
+
+		// refresh the expiration date
+		item.exp = time.Now().Add(i.expire)
+
+		i.items[k] = item
+	}
+}
+
 func (i *icache) Delete(k string) {
 	i.Lock()
 	delete(i.items, k)
@@ -166,6 +181,16 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
 	return nil
 }
 
+func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
+	err := c.service.UpdateUserAccount(access, props)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.update(access, props)
+	return nil
+}
+
 // ListUserAccounts is a passthrough to the underlying service and
 // does not make use of the cache
 func (c *IAMCache) ListUserAccounts() ([]Account, error) {

auth/iam_internal.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"sync"
 	"time"
 )
 
@@ -32,6 +33,13 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
 	dir string
 }
 
@@ -62,6 +70,9 @@ func NewInternal(dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -70,7 +81,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 
 		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
-			return nil, fmt.Errorf("account already exists")
+			return nil, ErrUserExists
 		}
 		conf.AccessAccounts[account.Access] = account
 
@@ -86,6 +97,9 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -99,9 +113,41 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+// UpdateUserAccount updates the specified user account fields. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		acc, found := conf.AccessAccounts[access]
+		if !found {
+			return nil, ErrNoSuchUser
+		}
+
+		updateAcc(&acc, props)
+		conf.AccessAccounts[access] = acc
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -121,6 +167,9 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 
 // ListUserAccounts lists all the user accounts stored.
 func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return []Account{}, fmt.Errorf("get iam data: %w", err)
@@ -188,6 +237,10 @@ func parseIAM(b []byte) (iAMConfig, error) {
 		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
 	}
 
+	if conf.AccessAccounts == nil {
+		conf.AccessAccounts = make(map[string]Account)
+	}
+
 	return conf, nil
 }
 

auth/iam_ldap.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/go-ldap/ldap/v3"
@@ -28,12 +29,15 @@ type LdapIAMService struct {
 	accessAtr  string
 	secretAtr  string
 	roleAtr    string
+	groupIdAtr string
+	userIdAtr  string
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
-	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
+		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
 	conn, err := ldap.DialURL(url)
@@ -52,15 +56,19 @@ func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objCl
 		accessAtr:  accAtr,
 		secretAtr:  secAtr,
 		roleAtr:    roleAtr,
+		userIdAtr:  userIdAtr,
+		groupIdAtr: groupIdAtr,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
-	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
 	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
+	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
+	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -79,7 +87,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
 		nil,
 	)
 
@@ -88,14 +96,48 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		return Account{}, err
 	}
 
+	if len(result.Entries) == 0 {
+		return Account{}, ErrNoSuchUser
+	}
+
 	entry := result.Entries[0]
+	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
+	}
+	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
+	}
 	return Account{
-		Access: entry.GetAttributeValue(ld.accessAtr),
-		Secret: entry.GetAttributeValue(ld.secretAtr),
-		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
+		Access:  entry.GetAttributeValue(ld.accessAtr),
+		Secret:  entry.GetAttributeValue(ld.secretAtr),
+		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
+		GroupID: groupId,
+		UserID:  userId,
 	}, nil
 }
 
+func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+	if props.Secret != nil {
+		req.Replace(ld.secretAtr, []string{*props.Secret})
+	}
+	if props.GroupID != nil {
+		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
+	}
+	if props.UserID != nil {
+		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
+	}
+
+	err := ld.conn.Modify(req)
+	//TODO: Handle non existing user case
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (ld *LdapIAMService) DeleteUserAccount(access string) error {
 	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
 
@@ -120,7 +162,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(&%v)", searchFilter),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
 		nil,
 	)
 
@@ -131,10 +173,20 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 
 	result := []Account{}
 	for _, el := range resp.Entries {
+		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
+		}
+		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
+		}
 		result = append(result, Account{
-			Access: el.GetAttributeValue(ld.accessAtr),
-			Secret: el.GetAttributeValue(ld.secretAtr),
-			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
+			Access:  el.GetAttributeValue(ld.accessAtr),
+			Secret:  el.GetAttributeValue(ld.secretAtr),
+			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
+			GroupID: groupId,
+			UserID:  userId,
 		})
 	}
 

auth/iam_s3_object.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"sort"
+	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -41,6 +42,14 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+
 	access        string
 	secret        string
 	region        string
@@ -85,11 +94,21 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 		return nil, fmt.Errorf("init s3 IAM: %v", err)
 	}
 
+	if endpoint != "" {
+		i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &endpoint
+		})
+		return i, nil
+	}
+
 	i.client = s3.NewFromConfig(cfg)
 	return i, nil
 }
 
 func (s *IAMServiceS3) CreateAccount(account Account) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -97,7 +116,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 
 	_, ok := conf.AccessAccounts[account.Access]
 	if ok {
-		return fmt.Errorf("account already exists")
+		return ErrUserExists
 	}
 	conf.AccessAccounts[account.Access] = account
 
@@ -105,6 +124,9 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 }
 
 func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return Account{}, err
@@ -118,7 +140,30 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	acc, ok := conf.AccessAccounts[access]
+	if !ok {
+		return ErrNoSuchUser
+	}
+
+	updateAcc(&acc, props)
+	conf.AccessAccounts[access] = acc
+
+	return s.storeAccts(conf)
+}
+
 func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -134,6 +179,9 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
 }
 
 func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return nil, err
@@ -159,16 +207,6 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
 	return accs, nil
 }
 
-// ResolveEndpoint is used for on prem or non-aws endpoints
-func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
-	return aws.Endpoint{
-		PartitionID:       "aws",
-		URL:               s.endpoint,
-		SigningRegion:     s.region,
-		HostnameImmutable: true,
-	}, nil
-}
-
 func (s *IAMServiceS3) Shutdown() error {
 	return nil
 }
@@ -187,11 +225,6 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
 		config.WithHTTPClient(client),
 	}
 
-	if s.endpoint != "" {
-		opts = append(opts,
-			config.WithEndpointResolverWithOptions(s))
-	}
-
 	if s.debug {
 		opts = append(opts,
 			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
@@ -212,12 +245,12 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
 		// init empty accounts stuct and return that
 		var nsk *types.NoSuchKey
 		if errors.As(err, &nsk) {
-			return iAMConfig{}, nil
+			return iAMConfig{AccessAccounts: map[string]Account{}}, nil
 		}
 		var apiErr smithy.APIError
 		if errors.As(err, &apiErr) {
 			if apiErr.ErrorCode() == "NotFound" {
-				return iAMConfig{}, nil
+				return iAMConfig{AccessAccounts: map[string]Account{}}, nil
 			}
 		}
 

auth/iam_single.go

@@ -35,6 +35,11 @@ func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
 	return Account{}, ErrNoSuchUser
 }
 
+// UpdateUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
+	return ErrNotSupported
+}
+
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
 	return ErrNotSupported

auth/iam_vault.go

@@ -0,0 +1,248 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	vault "github.com/hashicorp/vault-client-go"
+	"github.com/hashicorp/vault-client-go/schema"
+)
+
+type VaultIAMService struct {
+	client            *vault.Client
+	reqOpts           []vault.RequestOption
+	secretStoragePath string
+}
+
+var _ IAMService = &VaultIAMService{}
+
+func NewVaultIAMService(endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
+	opts := []vault.ClientOption{
+		vault.WithAddress(endpoint),
+		// set request timeout to 10 secs
+		vault.WithRequestTimeout(10 * time.Second),
+	}
+	if serverCert != "" {
+		tls := vault.TLSConfiguration{}
+
+		tls.ServerCertificate.FromBytes = []byte(serverCert)
+		if clientCert != "" {
+			if clientCertKey == "" {
+				return nil, fmt.Errorf("client certificate and client certificate should both be specified")
+			}
+
+			tls.ClientCertificate.FromBytes = []byte(clientCert)
+			tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
+		}
+
+		opts = append(opts, vault.WithTLS(tls))
+	}
+
+	client, err := vault.New(opts...)
+	if err != nil {
+		return nil, fmt.Errorf("init vault client: %w", err)
+	}
+
+	reqOpts := []vault.RequestOption{}
+	// if mount path is not specified, it defaults to "approle"
+	if mountPath != "" {
+		reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
+	}
+
+	// Authentication
+	switch {
+	case rootToken != "":
+		err := client.SetToken(rootToken)
+		if err != nil {
+			return nil, fmt.Errorf("root token authentication failure: %w", err)
+		}
+	case roleID != "":
+		if roleSecret == "" {
+			return nil, fmt.Errorf("role id and role secret must both be specified")
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
+			RoleId:   roleID,
+			SecretId: roleSecret,
+		}, reqOpts...)
+		cancel()
+		if err != nil {
+			return nil, fmt.Errorf("approle authentication failure: %w", err)
+		}
+
+		if err := client.SetToken(resp.Auth.ClientToken); err != nil {
+			return nil, fmt.Errorf("approle authentication set token failure: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
+	}
+
+	return &VaultIAMService{
+		client:            client,
+		reqOpts:           reqOpts,
+		secretStoragePath: secretStoragePath,
+	}, nil
+}
+
+func (vt *VaultIAMService) CreateAccount(account Account) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
+		Data: map[string]any{
+			account.Access: account,
+		},
+		Options: map[string]interface{}{
+			"cas": 0,
+		},
+	}, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		if strings.Contains(err.Error(), "check-and-set") {
+			return ErrUserExists
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acc, err := parseVaultUserAccount(resp.Data.Data, access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	return acc, nil
+}
+
+func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	//TODO: We need something like a transaction here ?
+	acc, err := vt.GetUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	updateAcc(&acc, props)
+
+	err = vt.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	err = vt.CreateAccount(acc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (vt *VaultIAMService) DeleteUserAccount(access string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		if vault.IsErrorStatus(err, 404) {
+			return []Account{}, nil
+		}
+		return nil, err
+	}
+
+	accs := []Account{}
+
+	for _, acss := range resp.Data.Keys {
+		acc, err := vt.GetUserAccount(acss)
+		if err != nil {
+			return nil, err
+		}
+		accs = append(accs, acc)
+	}
+
+	return accs, nil
+}
+
+// the client doesn't have explicit shutdown, as it uses http.Client
+func (vt *VaultIAMService) Shutdown() error {
+	return nil
+}
+
+var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
+
+func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
+	usrAcc, ok := data[access].(map[string]interface{})
+	if !ok {
+		return acc, errInvalidUser
+	}
+
+	acss, ok := usrAcc["access"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	secret, ok := usrAcc["secret"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	role, ok := usrAcc["role"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userIdJson, ok := usrAcc["userID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userId, err := userIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+	groupIdJson, ok := usrAcc["groupID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	groupId, err := groupIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+
+	return Account{
+		Access:  acss,
+		Secret:  secret,
+		Role:    Role(role),
+		UserID:  int(userId),
+		GroupID: int(groupId),
+	}, nil
+}

aws/signer/v4/functional_test.go

@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 	}
 }
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 	}
 }

backend/azure/azure.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"math"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -35,6 +36,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/auth"
@@ -51,6 +53,7 @@ type key string
 const (
 	keyAclCapital   key = "Acl"
 	keyAclLower     key = "acl"
+	keyOwnership    key = "Ownership"
 	keyTags         key = "Tags"
 	keyPolicy       key = "Policy"
 	keyBucketLock   key = "Bucket-Lock"
@@ -125,6 +128,7 @@ func (az *Azure) String() string {
 func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
 	meta := map[string]*string{
 		string(keyAclCapital): backend.GetStringPtr(string(acl)),
+		string(keyOwnership):  backend.GetStringPtr(string(input.ObjectOwnership)),
 	}
 
 	acct, ok := ctx.Value("account").(auth.Account)
@@ -175,7 +179,12 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 }
 
 func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
-	pager := az.client.NewListContainersPager(nil)
+	pager := az.client.NewListContainersPager(
+		&service.ListContainersOptions{
+			Include: service.ListContainersInclude{
+				Metadata: true,
+			},
+		})
 
 	var buckets []s3response.ListAllMyBucketsEntry
 	var result s3response.ListAllMyBucketsResult
@@ -186,11 +195,26 @@ func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s
 			return result, azureErrToS3Err(err)
 		}
 		for _, v := range resp.ContainerItems {
-			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-				Name: *v.Name,
-				// TODO: using modification date here instead of creation, is that ok?
-				CreationDate: *v.Properties.LastModified,
-			})
+			if isAdmin {
+				buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+					Name: *v.Name,
+					// TODO: using modification date here instead of creation, is that ok?
+					CreationDate: *v.Properties.LastModified,
+				})
+			} else {
+				acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
+				if err != nil {
+					return result, err
+				}
+
+				if acl.Owner == owner {
+					buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+						Name: *v.Name,
+						// TODO: using modification date here instead of creation, is that ok?
+						CreationDate: *v.Properties.LastModified,
+					})
+				}
+			}
 		}
 	}
 
@@ -215,10 +239,81 @@ func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3
 }
 
 func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
-	_, err := az.client.DeleteContainer(ctx, *input.Bucket, nil)
+	pager := az.client.NewListBlobsFlatPager(*input.Bucket, nil)
+
+	pg, err := pager.NextPage(ctx)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	if len(pg.Segment.BlobItems) > 0 {
+		return s3err.GetAPIError(s3err.ErrBucketNotEmpty)
+	}
+	_, err = az.client.DeleteContainer(ctx, *input.Bucket, nil)
 	return azureErrToS3Err(err)
 }
 
+func (az *Azure) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+	resp.Metadata[string(keyOwnership)] = backend.GetStringPtr(string(ownership))
+
+	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: resp.Metadata})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return ownship, err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return ownship, azureErrToS3Err(err)
+	}
+
+	ownership, ok := resp.Metadata[string(keyOwnership)]
+	if !ok {
+		return ownship, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)
+	}
+
+	return types.ObjectOwnership(*ownership), nil
+}
+
+func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	delete(resp.Metadata, string(keyOwnership))
+
+	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: resp.Metadata})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
 func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
 	tags, err := parseTags(po.Tagging)
 	if err != nil {
@@ -316,10 +411,10 @@ func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error {
 	return az.PutBucketTagging(ctx, bucket, nil)
 }
 
-func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	var opts *azblob.DownloadStreamOptions
 	if *input.Range != "" {
-		offset, count, err := parseRange(*input.Range)
+		offset, count, err := backend.ParseRange(0, *input.Range)
 		if err != nil {
 			return nil, err
 		}
@@ -334,12 +429,6 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer
 	if err != nil {
 		return nil, azureErrToS3Err(err)
 	}
-	defer blobDownloadResponse.Body.Close()
-
-	_, err = io.Copy(writer, blobDownloadResponse.Body)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
 
 	var tagcount int32
 	if blobDownloadResponse.TagCount != nil {
@@ -356,10 +445,42 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer
 		Metadata:        parseAzMetadata(blobDownloadResponse.Metadata),
 		TagCount:        &tagcount,
 		ContentRange:    blobDownloadResponse.ContentRange,
+		Body:            blobDownloadResponse.Body,
 	}, nil
 }
 
 func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	if input.PartNumber != nil {
+		client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
+		if err != nil {
+			return nil, err
+		}
+
+		res, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+
+		partsCount := int32(len(res.UncommittedBlocks))
+
+		for _, block := range res.UncommittedBlocks {
+			partNumber, err := decodeBlockId(*block.Name)
+			if err != nil {
+				return nil, err
+			}
+
+			if partNumber == int(*input.PartNumber) {
+				return &s3.HeadObjectOutput{
+					ContentLength: block.Size,
+					ETag:          block.Name,
+					PartsCount:    &partsCount,
+				}, nil
+			}
+		}
+
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
 	client, err := az.getBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
 		return nil, err
@@ -370,7 +491,7 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 		return nil, azureErrToS3Err(err)
 	}
 
-	return &s3.HeadObjectOutput{
+	result := &s3.HeadObjectOutput{
 		AcceptRanges:       resp.AcceptRanges,
 		ContentLength:      resp.ContentLength,
 		ContentType:        resp.ContentType,
@@ -381,7 +502,27 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 		LastModified:       resp.LastModified,
 		Metadata:           parseAzMetadata(resp.Metadata),
 		Expires:            resp.ExpiresOn,
-	}, nil
+	}
+
+	status, ok := resp.Metadata[string(keyObjLegalHold)]
+	if ok {
+		if *status == "1" {
+			result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
+		} else {
+			result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
+		}
+	}
+
+	retention, ok := resp.Metadata[string(keyObjRetention)]
+	if ok {
+		var config types.ObjectLockRetention
+		if err := json.Unmarshal([]byte(*retention), &config); err == nil {
+			result.ObjectLockMode = types.ObjectLockMode(config.Mode)
+			result.ObjectLockRetainUntilDate = config.RetainUntilDate
+		}
+	}
+
+	return result, nil
 }
 
 func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
@@ -433,7 +574,7 @@ func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAtt
 			IsTruncated:          resp.IsTruncated,
 			MaxParts:             resp.MaxParts,
 			PartNumberMarker:     resp.PartNumberMarker,
-			NextPartNumberMarker: resp.PartNumberMarker,
+			NextPartNumberMarker: resp.NextPartNumberMarker,
 			Parts:                parts,
 		},
 	}, nil
@@ -494,8 +635,14 @@ Pager:
 }
 
 func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	marker := ""
+	if *input.ContinuationToken > *input.StartAfter {
+		marker = *input.ContinuationToken
+	} else {
+		marker = *input.StartAfter
+	}
 	pager := az.client.NewListBlobsFlatPager(*input.Bucket, &azblob.ListBlobsFlatOptions{
-		Marker:     input.ContinuationToken,
+		Marker:     &marker,
 		MaxResults: input.MaxKeys,
 		Prefix:     input.Prefix,
 	})
@@ -544,6 +691,7 @@ Pager:
 		NextContinuationToken: nextMarker,
 		Prefix:                input.Prefix,
 		IsTruncated:           &isTruncated,
+		Delimiter:             input.Delimiter,
 	}, nil
 }
 
@@ -760,20 +908,20 @@ func (az *Azure) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3res
 		if err != nil {
 			return s3response.ListPartsResult{}, err
 		}
-		if partNumberMarker != 0 && partNumberMarker < partNumber {
+		if partNumberMarker != 0 && partNumberMarker >= partNumber {
 			continue
 		}
-		if len(parts) >= int(maxParts) {
-			nextPartNumberMarker = partNumber
-			isTruncated = true
-			break
-		}
 		parts = append(parts, s3response.Part{
 			Size:         *el.Size,
 			ETag:         *el.Name,
 			PartNumber:   partNumber,
 			LastModified: time.Now().Format(backend.RFC3339TimeFormat),
 		})
+		if len(parts) >= int(maxParts) {
+			nextPartNumberMarker = partNumber
+			isTruncated = true
+			break
+		}
 	}
 	return s3response.ListPartsResult{
 		Bucket:               *input.Bucket,
@@ -861,9 +1009,37 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 		return nil, err
 	}
 	blockIds := []string{}
-	for _, el := range input.MultipartUpload.Parts {
-		blockIds = append(blockIds, *el.ETag)
+
+	blockList, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
 	}
+
+	if len(blockList.UncommittedBlocks) != len(input.MultipartUpload.Parts) {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+	}
+
+	slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int {
+		ptNumber, _ := decodeBlockId(*a.Name)
+		nextPtNumber, _ := decodeBlockId(*b.Name)
+		return ptNumber - nextPtNumber
+	})
+
+	for i, block := range blockList.UncommittedBlocks {
+		ptNumber, err := decodeBlockId(*block.Name)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		if *input.MultipartUpload.Parts[i].ETag != *block.Name {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		blockIds = append(blockIds, *block.Name)
+	}
+
 	resp, err := client.CommitBlockList(ctx, blockIds, nil)
 	if err != nil {
 		return nil, parseMpError(err)
@@ -954,7 +1130,7 @@ func (az *Azure) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, er
 
 	policyPtr, ok := props.Metadata[string(keyPolicy)]
 	if !ok {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)
 	}
 
 	policy, err := base64.StdEncoding.DecodeString(*policyPtr)
@@ -1064,7 +1240,28 @@ func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, version
 			string(keyObjRetention): backend.GetStringPtr(string(retention)),
 		}
 	} else {
-		meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
+		objLockCfg, ok := meta[string(keyObjRetention)]
+		if !ok {
+			meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
+		} else {
+			var lockCfg types.ObjectLockRetention
+			if err := json.Unmarshal([]byte(*objLockCfg), &lockCfg); err != nil {
+				return fmt.Errorf("unmarshal object lock config: %w", err)
+			}
+
+			switch lockCfg.Mode {
+			// Compliance mode can't be overridden
+			case types.ObjectLockRetentionModeCompliance:
+				return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+			// To override governance mode user should have "s3:BypassGovernanceRetention" permission
+			case types.ObjectLockRetentionModeGovernance:
+				if !bypass {
+					return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+				}
+			}
+
+			meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
+		}
 	}
 
 	_, err = blobClient.SetMetadata(ctx, meta, nil)
@@ -1356,39 +1553,6 @@ func decodeBlockId(blockID string) (int, error) {
 	return int(binary.LittleEndian.Uint32(slice)), nil
 }
 
-func parseRange(rg string) (offset, count int64, err error) {
-	rangeKv := strings.Split(rg, "=")
-
-	if len(rangeKv) < 2 {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 1 || len(bRange) > 2 {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	offset, err = strconv.ParseInt(bRange[0], 10, 64)
-	if err != nil {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	if len(bRange) == 1 || bRange[1] == "" {
-		return offset, count, nil
-	}
-
-	count, err = strconv.ParseInt(bRange[1], 10, 64)
-	if err != nil {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	if count < offset {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	return offset, count - offset + 1, nil
-}
-
 func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 	aclPtr, ok := meta[string(key)]
 	if !ok {

backend/backend.go

@@ -18,9 +18,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 	"github.com/versity/versitygw/s3select"
@@ -43,6 +43,9 @@ type Backend interface {
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
+	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
+	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
+	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
 
 	// multipart operations
 	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
@@ -56,7 +59,7 @@ type Backend interface {
 	// standard object operations
 	PutObject(context.Context, *s3.PutObjectInput) (string, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
 	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
@@ -138,6 +141,15 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
 func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
 func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -167,7 +179,7 @@ func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {

backend/common.go

@@ -18,7 +18,9 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"io/fs"
+	"io"
+	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -61,9 +63,9 @@ var (
 
 // ParseRange parses input range header and returns startoffset, length, and
 // error. If no endoffset specified, then length is set to -1.
-func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
+func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, fi.Size(), nil
+		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
@@ -99,6 +101,14 @@ func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
+func CreateExceedingRangeErr(objSize int64) s3err.APIError {
+	return s3err.APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
@@ -120,3 +130,16 @@ func md5String(data []byte) string {
 	sum := md5.Sum(data)
 	return hex.EncodeToString(sum[:])
 }
+
+type FileSectionReadCloser struct {
+	R io.Reader
+	F *os.File
+}
+
+func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
+	return f.R.Read(p)
+}
+
+func (f *FileSectionReadCloser) Close() error {
+	return f.F.Close()
+}

backend/mkdir.go

@@ -30,7 +30,7 @@ var (
 // Any newly created directory is set to provided uid/gid ownership.
 // If path is already a directory, MkdirAll does nothing
 // and returns nil.
-// Any directoy created will be set to provided uid/gid ownership
+// Any directory created will be set to provided uid/gid ownership
 // if doChown is true.
 func MkdirAll(path string, uid, gid int, doChown bool) error {
 	// Fast path: if we can tell whether path is a directory or file, stop with success or error.

backend/posix/posix.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"math"
 	"os"
 	"path/filepath"
 	"sort"
@@ -74,6 +75,7 @@ const (
 	contentEncHdr       = "content-encoding"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
 	aclkey              = "acl"
+	ownershipkey        = "ownership"
 	etagkey             = "etag"
 	policykey           = "policy"
 	bucketLockKey       = "bucket-lock"
@@ -244,6 +246,9 @@ func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, a
 	if err := p.meta.StoreAttribute(bucket, "", aclkey, acl); err != nil {
 		return fmt.Errorf("set acl: %w", err)
 	}
+	if err := p.meta.StoreAttribute(bucket, "", ownershipkey, []byte(input.ObjectOwnership)); err != nil {
+		return fmt.Errorf("set ownership: %w", err)
+	}
 
 	if input.ObjectLockEnabledForBucket != nil && *input.ObjectLockEnabledForBucket {
 		now := time.Now()
@@ -303,6 +308,61 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
 	return nil
 }
 
+func (p *Posix) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if err := p.meta.StoreAttribute(bucket, "", ownershipkey, []byte(ownership)); err != nil {
+		return fmt.Errorf("set ownership: %w", err)
+	}
+
+	return nil
+}
+func (p *Posix) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return ownship, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return ownship, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	ownership, err := p.meta.RetrieveAttribute(bucket, "", ownershipkey)
+	if errors.Is(err, meta.ErrNoSuchKey) {
+		return ownship, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)
+	}
+	if err != nil {
+		return ownship, fmt.Errorf("get bucket ownership status: %w", err)
+	}
+
+	return types.ObjectOwnership(ownership), nil
+}
+func (p *Posix) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if err := p.meta.DeleteAttribute(bucket, "", ownershipkey); err != nil {
+		if errors.Is(err, meta.ErrNoSuchKey) {
+			return nil
+		}
+
+		return fmt.Errorf("delete ownership: %w", err)
+	}
+
+	return nil
+}
+
 func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	if mpu.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
@@ -508,6 +568,10 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	partsize := int64(0)
 	var totalsize int64
 	for i, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
 		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
 		fullPartPath := filepath.Join(bucket, partObjPath)
 		fi, err := os.Lstat(fullPartPath)
@@ -529,7 +593,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 		if err != nil {
 			etag = ""
 		}
-		if etag != *parts[i].ETag {
+		if parts[i].ETag == nil || etag != *parts[i].ETag {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 	}
@@ -545,6 +609,10 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	defer f.cleanup()
 
 	for _, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
 		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
 		fullPartPath := filepath.Join(bucket, partObjPath)
 		pf, err := os.Open(fullPartPath)
@@ -647,7 +715,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	}
 
 	// cleanup tmp dirs
-	os.RemoveAll(upiddir)
+	os.RemoveAll(filepath.Join(bucket, objdir, uploadID))
 	// use Remove for objdir in case there are still other uploads
 	// for same object name outstanding, this will fail if there are
 	os.Remove(filepath.Join(bucket, objdir))
@@ -992,7 +1060,11 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
 
 	var parts []s3response.Part
 	for _, e := range ents {
-		pn, _ := strconv.Atoi(e.Name())
+		pn, err := strconv.Atoi(e.Name())
+		if err != nil {
+			// file is not a valid part file
+			continue
+		}
 		if pn <= partNumberMarker {
 			continue
 		}
@@ -1186,7 +1258,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 		return s3response.CopyObjectResult{}, fmt.Errorf("stat object: %w", err)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi, *upi.CopySourceRange)
+	startOffset, length, err := backend.ParseRange(fi.Size(), *upi.CopySourceRange)
 	if err != nil {
 		return s3response.CopyObjectResult{}, err
 	}
@@ -1196,7 +1268,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 	}
 
 	if startOffset+length > fi.Size()+1 {
-		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRange)
+		return s3response.CopyObjectResult{}, backend.CreateExceedingRangeErr(fi.Size())
 	}
 
 	f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
@@ -1521,7 +1593,7 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 	}, nil
 }
 
-func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1552,7 +1624,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 	}
 
 	acceptRange := *input.Range
-	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
 	if err != nil {
 		return nil, err
 	}
@@ -1565,11 +1637,11 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 	}
 
 	if length == -1 {
-		length = objSize - startOffset + 1
+		length = objSize - startOffset
 	}
 
-	if startOffset+length > objSize+1 {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRange)
+	if startOffset+length > objSize {
+		length = objSize - startOffset
 	}
 
 	var contentRange string
@@ -1612,21 +1684,6 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		}, nil
 	}
 
-	f, err := os.Open(objPath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("open object: %w", err)
-	}
-	defer f.Close()
-
-	rdr := io.NewSectionReader(f, startOffset, length)
-	_, err = io.Copy(writer, rdr)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
-
 	userMetaData := make(map[string]string)
 
 	contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
@@ -1647,6 +1704,16 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		tagCount = &tgCount
 	}
 
+	f, err := os.Open(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("open object: %w", err)
+	}
+
+	rdr := io.NewSectionReader(f, startOffset, length)
+
 	return &s3.GetObjectOutput{
 		AcceptRanges:    &acceptRange,
 		ContentLength:   &length,
@@ -1657,6 +1724,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		Metadata:        userMetaData,
 		TagCount:        tagCount,
 		ContentRange:    &contentRange,
+		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
 	}, nil
 }
 
@@ -1816,6 +1884,11 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr
 	parts := []types.ObjectPart{}
 
 	for _, p := range resp.Parts {
+		if !(p.PartNumber > 0 && p.PartNumber <= math.MaxInt32) {
+			return s3response.GetObjectAttributesResult{},
+				s3err.GetAPIError(s3err.ErrInvalidPartNumber)
+		}
+
 		partNumber := int32(p.PartNumber)
 		size := p.Size
 
@@ -2506,7 +2579,7 @@ func (p *Posix) PutObjectRetention(_ context.Context, bucket, object, versionId
 	}
 
 	switch lockCfg.Mode {
-	// Compliance mode can't be overriden
+	// Compliance mode can't be overridden
 	case types.ObjectLockRetentionModeCompliance:
 		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
 	// To override governance mode user should have "s3:BypassGovernanceRetention" permission

backend/s3proxy/client.go

@@ -33,6 +33,12 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
 		return nil, err
 	}
 
+	if s.endpoint != "" {
+		return s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &s.endpoint
+		}), nil
+	}
+
 	return s3.NewFromConfig(cfg), nil
 }
 
@@ -50,11 +56,6 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 		config.WithHTTPClient(client),
 	}
 
-	if s.endpoint != "" {
-		opts = append(opts,
-			config.WithEndpointResolverWithOptions(s))
-	}
-
 	if s.disableChecksum {
 		opts = append(opts,
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -67,13 +68,3 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 
 	return config.LoadDefaultConfig(ctx, opts...)
 }
-
-// ResolveEndpoint is used for on prem or non-aws endpoints
-func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
-	return aws.Endpoint{
-		PartitionID:       "aws",
-		URL:               s.endpoint,
-		SigningRegion:     s.awsRegion,
-		HostnameImmutable: true,
-	}, nil
-}

backend/s3proxy/s3.go

@@ -55,6 +55,8 @@ type S3Proxy struct {
 	debug           bool
 }
 
+var _ backend.Backend = &S3Proxy{}
+
 func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
 	s := &S3Proxy{
 		access:          access,
@@ -128,6 +130,37 @@ func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput)
 	return handleError(err)
 }
 
+func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
+	_, err := s.client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+		Bucket: &bucket,
+		OwnershipControls: &types.OwnershipControls{
+			Rules: []types.OwnershipControlsRule{
+				{
+					ObjectOwnership: ownership,
+				},
+			},
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	resp, err := s.client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return ownship, handleError(err)
+	}
+	return resp.OwnershipControls.Rules[0].ObjectOwnership, nil
+}
+func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	return handleError(err)
+}
+
 func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	out, err := s.client.CreateMultipartUpload(ctx, input)
 	return out, handleError(err)
@@ -281,17 +314,11 @@ func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	return out, handleError(err)
 }
 
-func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	output, err := s.client.GetObject(ctx, input)
 	if err != nil {
 		return nil, handleError(err)
 	}
-	defer output.Body.Close()
-
-	_, err = io.Copy(w, output.Body)
-	if err != nil {
-		return nil, err
-	}
 
 	return output, nil
 }

backend/scoutfs/scoutfs.go

@@ -26,12 +26,14 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
 )
@@ -47,6 +49,9 @@ type ScoutFS struct {
 	rootfd  *os.File
 	rootdir string
 
+	// bucket/object metadata storage facility
+	meta meta.MetadataStorer
+
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
 	// HEAD object: if file offline, set obj storage class to GLACIER
@@ -75,8 +80,13 @@ const (
 	metaTmpDir          = ".sgwtmp"
 	metaTmpMultipartDir = metaTmpDir + "/multipart"
 	tagHdr              = "X-Amz-Tagging"
+	metaHdr             = "X-Amz-Meta"
+	contentTypeHdr      = "content-type"
+	contentEncHdr       = "content-encoding"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
-	etagkey             = "user.etag"
+	etagkey             = "etag"
+	objectRetentionKey  = "object-retention"
+	objectLegalHoldKey  = "object-legal-hold"
 )
 
 var (
@@ -87,11 +97,12 @@ var (
 
 const (
 	// ScoutFS special xattr types
-
 	systemPrefix = "scoutfs.hide."
 	onameAttr    = systemPrefix + "objname"
 	flagskey     = systemPrefix + "sam_flags"
 	stagecopykey = systemPrefix + "sam_stagereq"
+
+	fsBlocksize = 4096
 )
 
 const (
@@ -179,18 +190,20 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		return nil, err
 	}
 
-	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+	objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
 
 	// check all parts ok
 	last := len(parts) - 1
 	partsize := int64(0)
 	var totalsize int64
-	for i, p := range parts {
-		if p.PartNumber == nil {
+	for i, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
-		fi, err := os.Lstat(partPath)
+
+		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
+		fullPartPath := filepath.Join(bucket, partObjPath)
+		fi, err := os.Lstat(fullPartPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
@@ -198,23 +211,25 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		if i == 0 {
 			partsize = fi.Size()
 		}
+
+		// partsize must be a multiple of the filesystem blocksize
+		// except for last part
+		if i < last && partsize%fsBlocksize != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
 		totalsize += fi.Size()
 		// all parts except the last need to be the same size
 		if i < last && partsize != fi.Size() {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		// non-last part sizes need to be multiples of 4k for move blocks
-		// TODO: fallback to no move blocks if not 4k aligned?
-		if i == 0 && i < last && fi.Size()%4096 != 0 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
 
-		b, err := xattr.Get(partPath, "user.etag")
+		b, err := s.meta.RetrieveAttribute(bucket, partObjPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
 		}
-		if etag != *parts[i].ETag {
+		if parts[i].ETag == nil || etag != *parts[i].ETag {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 	}
@@ -230,10 +245,16 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}
 	defer f.cleanup()
 
-	for _, p := range parts {
-		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
+	for _, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
+		fullPartPath := filepath.Join(bucket, partObjPath)
+		pf, err := os.Open(fullPartPath)
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
+			return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
@@ -242,13 +263,13 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
+			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
 		}
 	}
 
 	userMetaData := make(map[string]string)
 	upiddir := filepath.Join(objdir, uploadID)
-	loadUserMetaData(upiddir, userMetaData)
+	cType, _ := s.loadUserMetaData(bucket, upiddir, userMetaData)
 
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
@@ -265,7 +286,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}
 
 	for k, v := range userMetaData {
-		err = xattr.Set(objname, "user."+k, []byte(v))
+		err = s.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
 			// cleanup object if returning error
 			os.Remove(objname)
@@ -273,10 +294,58 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		}
 	}
 
+	// load and set tagging
+	tagging, err := s.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
+	if err == nil {
+		if err := s.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object tagging: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object tagging: %w", err)
+	}
+
+	// set content-type
+	if cType != "" {
+		if err := s.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object content type: %w", err)
+		}
+	}
+
+	// load and set legal hold
+	lHold, err := s.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
+	if err == nil {
+		if err := s.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object legal hold: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object legal hold: %w", err)
+	}
+
+	// load and set retention
+	ret, err := s.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
+	if err == nil {
+		if err := s.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object retention: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object retention: %w", err)
+	}
+
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
-	err = xattr.Set(objname, "user.etag", []byte(s3MD5))
+	err = s.meta.StoreAttribute(bucket, object, etagkey, []byte(s3MD5))
 	if err != nil {
 		// cleanup object if returning error
 		os.Remove(objname)
@@ -310,61 +379,104 @@ func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte
 	return sum, nil
 }
 
-func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
-	ents, err := xattr.List(path)
+// fll out the user metadata map with the metadata for the object
+// and return the content type and encoding
+func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
+	ents, err := s.meta.ListAttributes(bucket, object)
 	if err != nil || len(ents) == 0 {
-		return
+		return "", ""
 	}
 	for _, e := range ents {
 		if !isValidMeta(e) {
 			continue
 		}
-		b, err := xattr.Get(path, e)
-		if err == errNoData {
-			m[strings.TrimPrefix(e, "user.")] = ""
-			continue
-		}
+		b, err := s.meta.RetrieveAttribute(bucket, object, e)
 		if err != nil {
 			continue
 		}
-		m[strings.TrimPrefix(e, "user.")] = string(b)
+		if b == nil {
+			m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = ""
+			continue
+		}
+		m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
 	}
 
-	b, err := xattr.Get(path, "user.content-type")
+	var contentType, contentEncoding string
+	b, _ := s.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
 	contentType = string(b)
-	if err != nil {
-		contentType = ""
-	}
 	if contentType != "" {
-		m["content-type"] = contentType
+		m[contentTypeHdr] = contentType
 	}
 
-	b, err = xattr.Get(path, "user.content-encoding")
+	b, _ = s.meta.RetrieveAttribute(bucket, object, contentEncHdr)
 	contentEncoding = string(b)
-	if err != nil {
-		contentEncoding = ""
-	}
 	if contentEncoding != "" {
-		m["content-encoding"] = contentEncoding
+		m[contentEncHdr] = contentEncoding
 	}
 
-	return
+	return contentType, contentEncoding
 }
 
 func isValidMeta(val string) bool {
-	if strings.HasPrefix(val, "user.X-Amz-Meta") {
+	if strings.HasPrefix(val, metaHdr) {
 		return true
 	}
-	if strings.EqualFold(val, "user.Expires") {
+	if strings.EqualFold(val, "Expires") {
 		return true
 	}
 	return false
 }
 
-func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	if input.Key == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
 	bucket := *input.Bucket
 	object := *input.Key
 
+	if input.PartNumber != nil {
+		uploadId, sum, err := s.retrieveUploadId(bucket, object)
+		if err != nil {
+			return nil, err
+		}
+
+		ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read parts: %w", err)
+		}
+
+		partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
+
+		part, err := os.Stat(filepath.Join(bucket, partPath))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat part: %w", err)
+		}
+
+		b, err := s.meta.RetrieveAttribute(bucket, partPath, etagkey)
+		etag := string(b)
+		if err != nil {
+			etag = ""
+		}
+		partsCount := int32(len(ents))
+		size := part.Size()
+
+		return &s3.HeadObjectOutput{
+			LastModified:  backend.GetTimePtr(part.ModTime()),
+			ETag:          &etag,
+			PartsCount:    &partsCount,
+			ContentLength: &size,
+		}, nil
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -383,9 +495,14 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 	}
 
 	userMetaData := make(map[string]string)
-	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := xattr.Get(objPath, etagkey)
+	if fi.IsDir() {
+		// this is the media type for directories in AWS and Nextcloud
+		contentType = "application/x-directory"
+	}
+
+	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -424,19 +541,55 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 
 	contentLength := fi.Size()
 
+	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, "")
+	if err == nil {
+		if *status {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
+		} else {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
+		}
+	}
+
+	var objectLockMode types.ObjectLockMode
+	var objectLockRetainUntilDate *time.Time
+	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, "")
+	if err == nil {
+		var config types.ObjectLockRetention
+		if err := json.Unmarshal(retention, &config); err == nil {
+			objectLockMode = types.ObjectLockMode(config.Mode)
+			objectLockRetainUntilDate = config.RetainUntilDate
+		}
+	}
+
 	return &s3.HeadObjectOutput{
-		ContentLength:   &contentLength,
-		ContentType:     &contentType,
-		ContentEncoding: &contentEncoding,
-		ETag:            &etag,
-		LastModified:    backend.GetTimePtr(fi.ModTime()),
-		Metadata:        userMetaData,
-		StorageClass:    stclass,
-		Restore:         &requestOngoing,
+		ContentLength:             &contentLength,
+		ContentType:               &contentType,
+		ContentEncoding:           &contentEncoding,
+		ETag:                      &etag,
+		LastModified:              backend.GetTimePtr(fi.ModTime()),
+		Metadata:                  userMetaData,
+		StorageClass:              stclass,
+		Restore:                   &requestOngoing,
+		ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
+		ObjectLockMode:            objectLockMode,
+		ObjectLockRetainUntilDate: objectLockRetainUntilDate,
 	}, nil
 }
 
-func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
+	sum := sha256.Sum256([]byte(object))
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	entries, err := os.ReadDir(objdir)
+	if err != nil || len(entries) == 0 {
+		return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
+	return entries[0].Name(), sum, nil
+}
+
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	acceptRange := *input.Range
@@ -458,7 +611,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
 	if err != nil {
 		return nil, err
 	}
@@ -505,19 +658,14 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 	if err != nil {
 		return nil, fmt.Errorf("open object: %w", err)
 	}
-	defer f.Close()
 
 	rdr := io.NewSectionReader(f, startOffset, length)
-	_, err = io.Copy(writer, rdr)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
 
 	userMetaData := make(map[string]string)
 
-	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := xattr.Get(objPath, etagkey)
+	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -541,6 +689,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
 		ContentRange:    &contentRange,
+		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
 	}, nil
 }
 
@@ -671,14 +820,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			etagBytes, err := xattr.Get(objPath, etagkey)
-			if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
-				return types.Object{}, backend.ErrSkipObj
-			}
+			b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
 			if err != nil {
 				return types.Object{}, fmt.Errorf("get etag: %w", err)
 			}
-			etag := string(etagBytes)
+			etag := string(b)
 
 			fi, err := d.Info()
 			if errors.Is(err, fs.ErrNotExist) {
@@ -698,14 +844,14 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		}
 
 		// file object, get object info and fill out object data
-		etagBytes, err := xattr.Get(objPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
 			return types.Object{}, backend.ErrSkipObj
 		}
-		if err != nil && !isNoAttr(err) {
+		if err != nil {
 			return types.Object{}, fmt.Errorf("get etag: %w", err)
 		}
-		etag := string(etagBytes)
+		etag := string(b)
 
 		fi, err := d.Info()
 		if errors.Is(err, fs.ErrNotExist) {
@@ -819,15 +965,9 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 }
 
 func isNoAttr(err error) bool {
-	if err == nil {
-		return false
-	}
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == errNoData {
-		return true
-	}
 	return false
 }

backend/scoutfs/scoutfs_compat.go

@@ -23,7 +23,6 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
-	"syscall"
 
 	"golang.org/x/sys/unix"
 
@@ -35,7 +34,9 @@ import (
 )
 
 func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
-	p, err := posix.New(rootdir, meta.XattrMeta{}, posix.PosixOpts{
+	metastore := meta.XattrMeta{}
+
+	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
 		ChownUID: opts.ChownUID,
 		ChownGID: opts.ChownGID,
 	})
@@ -49,11 +50,13 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:    p,
-		rootfd:   f,
-		rootdir:  rootdir,
-		chownuid: opts.ChownUID,
-		chowngid: opts.ChownGID,
+		Posix:       p,
+		rootfd:      f,
+		rootdir:     rootdir,
+		meta:        metastore,
+		chownuid:    opts.ChownUID,
+		chowngid:    opts.ChownGID,
+		glaciermode: opts.GlacierMode,
 	}, nil
 }
 
@@ -100,11 +103,6 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 		gid:        gid,
 	}
 
-	// falloc is best effort, its fine if this fails
-	if size > 0 {
-		tmp.falloc()
-	}
-
 	if doChown {
 		err := f.Chown(uid, gid)
 		if err != nil {
@@ -115,14 +113,6 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 	return tmp, nil
 }
 
-func (tmp *tmpfile) falloc() error {
-	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
-	if err != nil {
-		return fmt.Errorf("fallocate: %v", err)
-	}
-	return nil
-}
-
 func (tmp *tmpfile) link() error {
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict

backend/scoutfs/with_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build !freebsd && !openbsd && !netbsd
-// +build !freebsd,!openbsd,!netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENODATA
-)

backend/scoutfs/without_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build freebsd || openbsd || netbsd
-// +build freebsd openbsd netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENOATTR
-)

cmd/versitygw/admin.go

@@ -82,6 +82,34 @@ func adminCommand() *cli.Command {
 					},
 				},
 			},
+			{
+				Name:   "update-user",
+				Usage:  "Updates a user account",
+				Action: updateUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "user access key id to be updated",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:    "secret",
+						Usage:   "secret access key for the new user",
+						Aliases: []string{"s"},
+					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+				},
+			},
 			{
 				Name:   "delete-user",
 				Usage:  "Delete a user",
@@ -276,6 +304,63 @@ func deleteUser(ctx *cli.Context) error {
 	return nil
 }
 
+func updateUser(ctx *cli.Context) error {
+	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
+	props := auth.MutableProps{}
+	if ctx.IsSet("secret") {
+		props.Secret = &secret
+	}
+	if ctx.IsSet("user-id") {
+		props.UserID = &userId
+	}
+	if ctx.IsSet("group-id") {
+		props.GroupID = &groupId
+	}
+
+	propsJSON, err := json.Marshal(props)
+	if err != nil {
+		return fmt.Errorf("failed to parse user attributes: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(propsJSON)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
 func listUsers(ctx *cli.Context) error {
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
 	if err != nil {

cmd/versitygw/main.go

@@ -35,37 +35,43 @@ import (
 )
 
 var (
-	port, admPort                          string
-	rootUserAccess                         string
-	rootUserSecret                         string
-	region                                 string
-	admCertFile, admKeyFile                string
-	certFile, keyFile                      string
-	kafkaURL, kafkaTopic, kafkaKey         string
-	natsURL, natsTopic                     string
-	eventWebhookURL                        string
-	eventConfigFilePath                    string
-	logWebhookURL                          string
-	accessLog                              string
-	healthPath                             string
-	debug                                  bool
-	pprof                                  string
-	quiet                                  bool
-	readonly                               bool
-	iamDir                                 string
-	ldapURL, ldapBindDN, ldapPassword      string
-	ldapQueryBase, ldapObjClasses          string
-	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
-	s3IamAccess, s3IamSecret               string
-	s3IamRegion, s3IamBucket               string
-	s3IamEndpoint                          string
-	s3IamSslNoVerify, s3IamDebug           bool
-	iamCacheDisable                        bool
-	iamCacheTTL                            int
-	iamCachePrune                          int
-	metricsService                         string
-	statsdServers                          string
-	dogstatsServers                        string
+	port, admPort                            string
+	rootUserAccess                           string
+	rootUserSecret                           string
+	region                                   string
+	admCertFile, admKeyFile                  string
+	certFile, keyFile                        string
+	kafkaURL, kafkaTopic, kafkaKey           string
+	natsURL, natsTopic                       string
+	eventWebhookURL                          string
+	eventConfigFilePath                      string
+	logWebhookURL                            string
+	accessLog                                string
+	healthPath                               string
+	debug                                    bool
+	pprof                                    string
+	quiet                                    bool
+	readonly                                 bool
+	iamDir                                   string
+	ldapURL, ldapBindDN, ldapPassword        string
+	ldapQueryBase, ldapObjClasses            string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr   string
+	ldapUserIdAtr, ldapGroupIdAtr            string
+	vaultEndpointURL, vaultSecretStoragePath string
+	vaultMountPath, vaultRootToken           string
+	vaultRoleId, vaultRoleSecret             string
+	vaultServerCert, vaultClientCert         string
+	vaultClientCertKey                       string
+	s3IamAccess, s3IamSecret                 string
+	s3IamRegion, s3IamBucket                 string
+	s3IamEndpoint                            string
+	s3IamSslNoVerify, s3IamDebug             bool
+	iamCacheDisable                          bool
+	iamCacheTTL                              int
+	iamCachePrune                            int
+	metricsService                           string
+	statsdServers                            string
+	dogstatsServers                          string
 )
 
 var (
@@ -326,6 +332,72 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-user-id-atr",
+			Usage:       "ldap server user id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_USER_ID_ATR"},
+			Destination: &ldapUserIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-group-id-atr",
+			Usage:       "ldap server user group id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
+			Destination: &ldapGroupIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-endpoint-url",
+			Usage:       "vault server url",
+			EnvVars:     []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
+			Destination: &vaultEndpointURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-secret-storage-path",
+			Usage:       "vault server secret storage path",
+			EnvVars:     []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
+			Destination: &vaultSecretStoragePath,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-mount-path",
+			Usage:       "vault server mount path",
+			EnvVars:     []string{"VGW_IAM_VAULT_MOUNT_PATH"},
+			Destination: &vaultMountPath,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-root-token",
+			Usage:       "vault server root token",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
+			Destination: &vaultRootToken,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-role-id",
+			Usage:       "vault server user role id",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_ID"},
+			Destination: &vaultRoleId,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-role-secret",
+			Usage:       "vault server user role secret",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_SECRET"},
+			Destination: &vaultRoleSecret,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-server_cert",
+			Usage:       "vault server TLS certificate",
+			EnvVars:     []string{"VGW_IAM_VAULT_SERVER_CERT"},
+			Destination: &vaultServerCert,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-client_cert",
+			Usage:       "vault client TLS certificate",
+			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT"},
+			Destination: &vaultClientCert,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-client_cert_key",
+			Usage:       "vault client TLS certificate key",
+			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
+			Destination: &vaultClientCertKey,
+		},
 		&cli.StringFlag{
 			Name:        "s3-iam-access",
 			Usage:       "s3 IAM access key",
@@ -501,25 +573,36 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
-		Dir:                iamDir,
-		LDAPServerURL:      ldapURL,
-		LDAPBindDN:         ldapBindDN,
-		LDAPPassword:       ldapPassword,
-		LDAPQueryBase:      ldapQueryBase,
-		LDAPObjClasses:     ldapObjClasses,
-		LDAPAccessAtr:      ldapAccessAtr,
-		LDAPSecretAtr:      ldapSecAtr,
-		LDAPRoleAtr:        ldapRoleAtr,
-		S3Access:           s3IamAccess,
-		S3Secret:           s3IamSecret,
-		S3Region:           s3IamRegion,
-		S3Bucket:           s3IamBucket,
-		S3Endpoint:         s3IamEndpoint,
-		S3DisableSSlVerfiy: s3IamSslNoVerify,
-		S3Debug:            s3IamDebug,
-		CacheDisable:       iamCacheDisable,
-		CacheTTL:           iamCacheTTL,
-		CachePrune:         iamCachePrune,
+		Dir:                    iamDir,
+		LDAPServerURL:          ldapURL,
+		LDAPBindDN:             ldapBindDN,
+		LDAPPassword:           ldapPassword,
+		LDAPQueryBase:          ldapQueryBase,
+		LDAPObjClasses:         ldapObjClasses,
+		LDAPAccessAtr:          ldapAccessAtr,
+		LDAPSecretAtr:          ldapSecAtr,
+		LDAPRoleAtr:            ldapRoleAtr,
+		LDAPUserIdAtr:          ldapUserIdAtr,
+		LDAPGroupIdAtr:         ldapGroupIdAtr,
+		VaultEndpointURL:       vaultEndpointURL,
+		VaultSecretStoragePath: vaultSecretStoragePath,
+		VaultMountPath:         vaultMountPath,
+		VaultRootToken:         vaultRootToken,
+		VaultRoleId:            vaultRoleId,
+		VaultRoleSecret:        vaultRoleSecret,
+		VaultServerCert:        vaultServerCert,
+		VaultClientCert:        vaultClientCert,
+		VaultClientCertKey:     vaultClientCertKey,
+		S3Access:               s3IamAccess,
+		S3Secret:               s3IamSecret,
+		S3Region:               s3IamRegion,
+		S3Bucket:               s3IamBucket,
+		S3Endpoint:             s3IamEndpoint,
+		S3DisableSSlVerfiy:     s3IamSslNoVerify,
+		S3Debug:                s3IamDebug,
+		CacheDisable:           iamCacheDisable,
+		CacheTTL:               iamCacheTTL,
+		CachePrune:             iamCachePrune,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)

cmd/versitygw/utils.go

@@ -71,7 +71,7 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
 		s3event.EventObjectRestoreCompleted:     true,
 	}
 
-	configBytes, err := json.Marshal(config)
+	configBytes, err := json.MarshalIndent(config, "", "  ")
 	if err != nil {
 		return fmt.Errorf("parse event config: %w", err)
 	}

extra/example.conf

@@ -153,6 +153,14 @@ ROOT_SECRET_ACCESS_KEY=
 # specified, all configured bucket events will be sent to the webhook.
 #VGW_EVENT_WEBHOOK_URL=
 
+# Bucket events can be filtered for any of the above event types. The
+# VGW_EVENT_FILTER option specifies a config file that contains the event
+# filter rules. The event filter rules are used to determine which events are
+# sent to the configured event services. Run:
+# versitygw utils gen-event-filter-config --path .
+# to generate a default rules file "event_config.json" in the current directory.
+#VGW_EVENT_FILTER=
+
 #######################
 # Debug / Diagnostics #
 #######################
@@ -182,20 +190,25 @@ ROOT_SECRET_ACCESS_KEY=
 # as a dedicated IAM service.
 #VGW_IAM_DIR=
 
-# The ldap options will enable the LDAP IAM service with accounts stored in an
-# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
-# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
-# secret credentials and role respectively. The other options are used to
-# connect to the LDAP service.
-#VGW_IAM_LDAP_URL=
-#VGW_IAM_LDAP_BASE_DN=
-#VGW_IAM_LDAP_BIND_DN=
-#VGW_IAM_LDAP_BIND_PASS=
-#VGW_IAM_LDAP_QUERY_BASE=
-#VGW_IAM_LDAP_OBJECT_CLASSES=
-#VGW_IAM_LDAP_ACCESS_ATR=
-#VGW_IAM_LDAP_SECRET_ATR=
-#VGW_IAM_LDAP_ROLE_ATR=
+# The Vault options will enable the Vault IAM service with accounts stored in
+# the HashiCorp Vault service. The Vault URL is the address and port of the
+# Vault server with the format <IP/host>:<port>. A root taken can be used for
+# testing, but it is recommended to use the role based authentication in
+# production. The Vault server certificate, client certificate, and client
+# certificate key are optional, and will default to not verifying the server
+# certificate and not using client certificates. The Vault server certificate
+# is used to verify the Vault server, and the client certificate and key are
+# used to authenticate the gateway to the Vault server. See wiki documentation
+# for an example of using Vault in dev mode with the gateway.
+#VGW_IAM_VAULT_ENDPOINT_URL=
+#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
+#VGW_IAM_VAULT_MOUNT_PATH=
+#VGW_IAM_VAULT_ROOT_TOKEN=
+#VGW_IAM_VAULT_ROLE_ID=
+#VGW_IAM_VAULT_ROLE_SECRET=
+#VGW_IAM_VAULT_SERVER_CERT=
+#VGW_IAM_VAULT_CLIENT_CERT=
+#VGW_IAM_VAULT_CLIENT_CERT_KEY=
 
 # The VGW_S3 IAM service is similar to the internal IAM service, but instead
 # stores the account information JSON encoded in an S3 object. This should use
@@ -210,6 +223,21 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_S3_IAM_BUCKET=
 #VGW_S3_IAM_NO_VERIFY=
 
+# The LDAP options will enable the LDAP IAM service with accounts stored in an
+# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
+# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
+# secret credentials and role respectively. The other options are used to
+# connect to the LDAP service.
+#VGW_IAM_LDAP_URL=
+#VGW_IAM_LDAP_BASE_DN=
+#VGW_IAM_LDAP_BIND_DN=
+#VGW_IAM_LDAP_BIND_PASS=
+#VGW_IAM_LDAP_QUERY_BASE=
+#VGW_IAM_LDAP_OBJECT_CLASSES=
+#VGW_IAM_LDAP_ACCESS_ATR=
+#VGW_IAM_LDAP_SECRET_ATR=
+#VGW_IAM_LDAP_ROLE_ATR=
+
 ###############
 # IAM caching #
 ###############
@@ -228,6 +256,29 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_CACHE_TTL=120
 #VGW_IAM_CACHE_PRUNE=3600
 
+###########
+# Metrics #
+###########
+
+# The metrics service name is a tag that is added to all metrics to help
+# identify the source of the metrics. This is especially useful when multiple
+# gateways are running. The default is the hostname of the system.
+#VGW_METRICS_SERVICE_NAME=$HOSTNAME
+
+# The metrics service will send metrics to the configured statsd servers. The
+# servers are specified as a comma separated list of host:port pairs. The
+# default is to not send metrics to any statsd servers. The gateway uses
+# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
+#VGW_METRICS_STATSD_SERVERS=
+
+# The metrics service will send metrics to the configured dogstatsd servers.
+# The servers are specified as a comma separated list of host:port pairs. The
+# default is to not send metrics to any dogstatsd servers. Generally
+# DataDog recommends installing a local agent to collect metrics and forward
+# them to the DataDog service. In this case the option value would be the
+# local agent address: 127.0.0.1:8125.
+#VGW_METRICS_DOGSTATS_SERVERS=
+
 ######################################
 # VersityGW Backend Specific Options #
 ######################################

go.mod

@@ -3,65 +3,73 @@ module github.com/versity/versitygw
 go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
-	github.com/aws/aws-sdk-go-v2 v1.27.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3
-	github.com/aws/smithy-go v1.20.2
+	github.com/DataDog/datadog-go/v5 v5.5.0
+	github.com/aws/aws-sdk-go-v2 v1.30.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0
+	github.com/aws/smithy-go v1.20.3
 	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/gofiber/fiber/v2 v2.52.4
+	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.35.0
+	github.com/hashicorp/vault-client-go v0.4.3
+	github.com/nats-io/nats.go v1.36.0
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.3
 	github.com/urfave/cli/v2 v2.27.2
-	github.com/valyala/fasthttp v1.54.0
+	github.com/valyala/fasthttp v1.55.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sys v0.20.0
+	golang.org/x/sys v0.22.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
-	github.com/DataDog/datadog-go/v5 v5.5.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	golang.org/x/crypto v0.23.0 // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.16
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.16
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.21
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.24
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.24
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.5
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect

go.sum

@@ -1,9 +1,9 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1 h1:Xy/qV1DyOhhqsU/z0PyFMJfYCxnzna+vBEUtFW0ksQo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1/go.mod h1:oib6iWdC+sILvNUoJbbBn3xv7TXow7mEp/WRcsYvmow=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
@@ -21,56 +21,58 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo=
-github.com/aws/aws-sdk-go-v2 v1.27.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
-github.com/aws/aws-sdk-go-v2/config v1.27.16 h1:knpCuH7laFVGYTNd99Ns5t+8PuRjDn4HnnZK48csipM=
-github.com/aws/aws-sdk-go-v2/config v1.27.16/go.mod h1:vutqgRhDUktwSge3hrC3nkuirzkJ4E/mLj5GvI0BQas=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.16 h1:7d2QxY83uYl0l58ceyiSpxg9bSbStqBC6BeEeHEchwo=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.16/go.mod h1:Ae6li/6Yc6eMzysRL2BXlPYvnrLLBg3D11/AmOjw50k=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 h1:dQLK4TjtnlRGb0czOht2CevZ5l6RSyRWAnKeGd7VAFE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3/go.mod h1:TL79f2P6+8Q7dTsILpiVST+AL9lkF6PPGI167Ny0Cjw=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.21 h1:1v8Ii0MRVGYB/sdhkbxrtolCA7Tp+lGh+5OJTs5vmZ8=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.21/go.mod h1:cxdd1rc8yxCjKz28hi30XN1jDXr2DxZvD44vLxTz/bg=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 h1:lf/8VTF2cM+N4SLzaYJERKEWAXq8MOMpZfU6wEPWsPk=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7/go.mod h1:4SjkU7QiqK2M9oozyMzfZ/23LmUY+h3oFqhdeP5OMiI=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 h1:4OYVp0705xu8yjdyoWix0r9wPIRXnIzzOoUpQVHIJ/g=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7/go.mod h1:vd7ESTEvI76T2Na050gODNmNU7+OyKrIKroYTu4ABiI=
+github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o=
+github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
+github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo=
+github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.5 h1:qkipTyOc+ElVS+TgGJCf/6gqu0CL5+ii19W/eMQfY94=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.5/go.mod h1:UjB35RXl+ESpnVtyaKqdw11NhMxm90lF9o2zqJNbi14=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 h1:/FUtT3xsoHO3cfh+I/kCbcMCN98QZRsiFet/V8QkWSs=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7/go.mod h1:MaCAgWpGooQoCWZnMur97rGn5dp350w2+CeiV5406wE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 h1:UXqEWQI0n+q0QixzU0yUUQBZXRd5037qdInTIHFTl98=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9/go.mod h1:xP6Gq6fzGZT8w/ZN+XvGMZ2RU1LeEs7b2yUP5DN8NY4=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 h1:Wx0rlZoEJR7JwlSZcHnEa7CNjrSIyVxMFWGAaXy4fJY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9/go.mod h1:aVMHdE0aHO3v+f/iw01fmXV/5DbfQ3Bi9nN7nd9bE9Y=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 h1:uO5XR6QGBcmPyo2gxofYJLFkcVQ4izOoGDNenlZhTEk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7/go.mod h1:feeeAYfAcwTReM6vbwjEyDmiGho+YgBhaFULuXDW8kc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3 h1:57NtjG+WLims0TxIQbjTqebZUKDM03DfM11ANAekW0s=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3/go.mod h1:739CllldowZiPPsDFcJHNF4FXrVxaSGVnZ9Ez9Iz9hc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 h1:aD7AGQhvPuAxlSUfo0CWU7s6FpkbyykMhGYMvlqTjVs=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.9/go.mod h1:c1qtZUWtygI6ZdvKppzCSXsDOq5I4luJPZ0Ud3juFCA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 h1:Pav5q3cA260Zqez42T9UhIlsd9QeypszRPwC9LdSSsQ=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3/go.mod h1:9lmoVDVLz/yUZwLaQ676TK02fhCu4+PgRSmMaKR1ozk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 h1:69tpbPED7jKPyzMcrwSvhWcJ9bPnZsZs18NT40JwM0g=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.10/go.mod h1:0Aqn1MnEuitqfsCNyKsdKLhDUOr4txD/g19EfiUqgws=
-github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
-github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 h1:THZJJ6TU/FOiM7DZFnisYV9d49oxXWUzsVIMTuf3VNU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13/go.mod h1:VISUTg6n+uBaYIWPBaIG0jk7mbBxm7DUqBtU2cUDDWI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 h1:2jyRZ9rVIMisyQRnhSS/SqlckveoxXneIumECVFP91Y=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15/go.mod h1:bDRG3m382v1KJBk1cKz7wIajg87/61EiiymEyfLvAe0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 h1:Eq2THzHt6P41mpjS2sUzz/3dJYFRqdWZ+vQaEMm98EM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13/go.mod h1:FgwTca6puegxgCInYwGjmd4tB9195Dd6LCuA+8MjpWw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0 h1:4rhV0Hn+bf8IAIUphRX1moBcEvKJipCPmswMCl6Q5mw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0/go.mod h1:hdV0NTYd0RwV4FvNKhKUNbPLZoq9CTr/lke+3I7aCAI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
 github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
 github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
-github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
+github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
@@ -80,9 +82,21 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
+github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@@ -100,8 +114,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -111,8 +125,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.35.0 h1:XFNqNM7v5B+MQMKqVGAyHwYhyKb48jrenXNxIU20ULk=
-github.com/nats-io/nats.go v1.35.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU=
+github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -132,6 +148,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
+github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -139,6 +157,7 @@ github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI
 github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -151,8 +170,8 @@ github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
 github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0=
-github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM=
+github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
+github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -174,8 +193,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -191,8 +210,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -216,8 +235,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -233,8 +252,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

metrics/actions.go

@@ -24,51 +24,54 @@ var (
 )
 
 var (
-	ActionUndetected                 = "ActionUnDetected"
-	ActionAbortMultipartUpload       = "s3_AbortMultipartUpload"
-	ActionCompleteMultipartUpload    = "s3_CompleteMultipartUpload"
-	ActionCopyObject                 = "s3_CopyObject"
-	ActionCreateBucket               = "s3_CreateBucket"
-	ActionCreateMultipartUpload      = "s3_CreateMultipartUpload"
-	ActionDeleteBucket               = "s3_DeleteBucket"
-	ActionDeleteBucketPolicy         = "s3_DeleteBucketPolicy"
-	ActionDeleteBucketTagging        = "s3_DeleteBucketTagging"
-	ActionDeleteObject               = "s3_DeleteObject"
-	ActionDeleteObjectTagging        = "s3_DeleteObjectTagging"
-	ActionDeleteObjects              = "s3_DeleteObjects"
-	ActionGetBucketAcl               = "s3_GetBucketAcl"
-	ActionGetBucketPolicy            = "s3_GetBucketPolicy"
-	ActionGetBucketTagging           = "s3_GetBucketTagging"
-	ActionGetBucketVersioning        = "s3_GetBucketVersioning"
-	ActionGetObject                  = "s3_GetObject"
-	ActionGetObjectAcl               = "s3_GetObjectAcl"
-	ActionGetObjectAttributes        = "s3_GetObjectAttributes"
-	ActionGetObjectLegalHold         = "s3_GetObjectLegalHold"
-	ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
-	ActionGetObjectRetention         = "s3_GetObjectRetention"
-	ActionGetObjectTagging           = "s3_GetObjectTagging"
-	ActionHeadBucket                 = "s3_HeadBucket"
-	ActionHeadObject                 = "s3_HeadObject"
-	ActionListAllMyBuckets           = "s3_ListAllMyBuckets"
-	ActionListMultipartUploads       = "s3_ListMultipartUploads"
-	ActionListObjectVersions         = "s3_ListObjectVersions"
-	ActionListObjects                = "s3_ListObjects"
-	ActionListObjectsV2              = "s3_ListObjectsV2"
-	ActionListParts                  = "s3_ListParts"
-	ActionPutBucketAcl               = "s3_PutBucketAcl"
-	ActionPutBucketPolicy            = "s3_PutBucketPolicy"
-	ActionPutBucketTagging           = "s3_PutBucketTagging"
-	ActionPutBucketVersioning        = "s3_PutBucketVersioning"
-	ActionPutObject                  = "s3_PutObject"
-	ActionPutObjectAcl               = "s3_PutObjectAcl"
-	ActionPutObjectLegalHold         = "s3_PutObjectLegalHold"
-	ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
-	ActionPutObjectRetention         = "s3_PutObjectRetention"
-	ActionPutObjectTagging           = "s3_PutObjectTagging"
-	ActionRestoreObject              = "s3_RestoreObject"
-	ActionSelectObjectContent        = "s3_SelectObjectContent"
-	ActionUploadPart                 = "s3_UploadPart"
-	ActionUploadPartCopy             = "s3_UploadPartCopy"
+	ActionUndetected                    = "ActionUnDetected"
+	ActionAbortMultipartUpload          = "s3_AbortMultipartUpload"
+	ActionCompleteMultipartUpload       = "s3_CompleteMultipartUpload"
+	ActionCopyObject                    = "s3_CopyObject"
+	ActionCreateBucket                  = "s3_CreateBucket"
+	ActionCreateMultipartUpload         = "s3_CreateMultipartUpload"
+	ActionDeleteBucket                  = "s3_DeleteBucket"
+	ActionDeleteBucketPolicy            = "s3_DeleteBucketPolicy"
+	ActionDeleteBucketTagging           = "s3_DeleteBucketTagging"
+	ActionDeleteObject                  = "s3_DeleteObject"
+	ActionDeleteObjectTagging           = "s3_DeleteObjectTagging"
+	ActionDeleteObjects                 = "s3_DeleteObjects"
+	ActionGetBucketAcl                  = "s3_GetBucketAcl"
+	ActionGetBucketPolicy               = "s3_GetBucketPolicy"
+	ActionGetBucketTagging              = "s3_GetBucketTagging"
+	ActionGetBucketVersioning           = "s3_GetBucketVersioning"
+	ActionGetObject                     = "s3_GetObject"
+	ActionGetObjectAcl                  = "s3_GetObjectAcl"
+	ActionGetObjectAttributes           = "s3_GetObjectAttributes"
+	ActionGetObjectLegalHold            = "s3_GetObjectLegalHold"
+	ActionGetObjectLockConfiguration    = "s3_GetObjectLockConfiguration"
+	ActionGetObjectRetention            = "s3_GetObjectRetention"
+	ActionGetObjectTagging              = "s3_GetObjectTagging"
+	ActionHeadBucket                    = "s3_HeadBucket"
+	ActionHeadObject                    = "s3_HeadObject"
+	ActionListAllMyBuckets              = "s3_ListAllMyBuckets"
+	ActionListMultipartUploads          = "s3_ListMultipartUploads"
+	ActionListObjectVersions            = "s3_ListObjectVersions"
+	ActionListObjects                   = "s3_ListObjects"
+	ActionListObjectsV2                 = "s3_ListObjectsV2"
+	ActionListParts                     = "s3_ListParts"
+	ActionPutBucketAcl                  = "s3_PutBucketAcl"
+	ActionPutBucketPolicy               = "s3_PutBucketPolicy"
+	ActionPutBucketTagging              = "s3_PutBucketTagging"
+	ActionPutBucketVersioning           = "s3_PutBucketVersioning"
+	ActionPutObject                     = "s3_PutObject"
+	ActionPutObjectAcl                  = "s3_PutObjectAcl"
+	ActionPutObjectLegalHold            = "s3_PutObjectLegalHold"
+	ActionPutObjectLockConfiguration    = "s3_PutObjectLockConfiguration"
+	ActionPutObjectRetention            = "s3_PutObjectRetention"
+	ActionPutObjectTagging              = "s3_PutObjectTagging"
+	ActionRestoreObject                 = "s3_RestoreObject"
+	ActionSelectObjectContent           = "s3_SelectObjectContent"
+	ActionUploadPart                    = "s3_UploadPart"
+	ActionUploadPartCopy                = "s3_UploadPartCopy"
+	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
+	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
+	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
 )
 
 func init() {

s3api/admin-router.go

@@ -32,6 +32,9 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
 	// DeleteUsers admin api
 	app.Patch("/delete-user", controller.DeleteUser)
 
+	// UpdateUser admin api
+	app.Patch("/update-user", controller.UpdateUser)
+
 	// ListUsers admin api
 	app.Patch("/list-users", controller.ListUsers)
 

s3api/controllers/admin.go

@@ -17,6 +17,7 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
@@ -35,31 +36,69 @@ func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	var usr auth.Account
 	err := json.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return fmt.Errorf("failed to parse request body: %w", err)
+		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("failed to parse request body: %w", err).Error())
 	}
 
 	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
+		return ctx.Status(fiber.StatusBadRequest).SendString("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		return fmt.Errorf("failed to create user: %w", err)
+		status := fiber.StatusInternalServerError
+		msg := fmt.Errorf("failed to create user: %w", err).Error()
+
+		if strings.Contains(msg, "user already exists") {
+			status = fiber.StatusConflict
+		}
+
+		return ctx.Status(status).SendString(msg)
 	}
 
-	return ctx.SendString("The user has been created successfully")
+	return ctx.Status(fiber.StatusCreated).SendString("The user has been created successfully")
+}
+
+func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
+	}
+
+	access := ctx.Query("access")
+	if access == "" {
+		return ctx.Status(fiber.StatusBadRequest).SendString("missing user access parameter")
+	}
+
+	var props auth.MutableProps
+	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
+		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("invalid request body %w", err).Error())
+	}
+
+	err := c.iam.UpdateUserAccount(access, props)
+	if err != nil {
+		status := fiber.StatusInternalServerError
+		msg := fmt.Errorf("failed to update user account: %w", err).Error()
+
+		if strings.Contains(msg, "user not found") {
+			status = fiber.StatusNotFound
+		}
+
+		return ctx.Status(status).SendString(msg)
+	}
+
+	return ctx.SendString("the user has been updated successfully")
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 
 	err := c.iam.DeleteUserAccount(access)
@@ -73,7 +112,7 @@ func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	accs, err := c.iam.ListUserAccounts()
 	if err != nil {
@@ -86,7 +125,7 @@ func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
@@ -96,7 +135,7 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user specified as the new bucket owner does not exist")
+		return ctx.Status(fiber.StatusNotFound).SendString("user specified as the new bucket owner does not exist")
 	}
 
 	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
@@ -104,13 +143,13 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		return err
 	}
 
-	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+	return ctx.SendString("Bucket owner has been updated successfully")
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())

s3api/controllers/admin_test.go

@@ -85,7 +85,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 201,
 		},
 		{
 			name: "Admin-create-user-invalid-user-role",
@@ -94,7 +94,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
@@ -103,7 +103,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -119,6 +119,122 @@ func TestAdminController_CreateUser(t *testing.T) {
 	}
 }
 
+func TestAdminController_UpdateUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/update-user", adminController.UpdateUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/update-user", adminController.UpdateUser)
+
+	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	appNotFound := fiber.New()
+
+	appNotFound.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-update-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-update-user-missing-access",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-request-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
+		{
+			name: "Admin-update-user-not-found",
+			app:  appNotFound,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 404,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
 func TestAdminController_DeleteUser(t *testing.T) {
 	type args struct {
 		req *http.Request
@@ -173,7 +289,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -251,7 +367,7 @@ func TestAdminController_ListUsers(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Admin-list-users-iam-error",
@@ -368,7 +484,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -386,7 +502,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 404,
 		},
 		{
 			name: "Change-bucket-owner-success",
@@ -395,7 +511,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 201,
+			statusCode: 200,
 		},
 	}
 	for _, tt := range tests {
@@ -455,7 +571,7 @@ func TestAdminController_ListBuckets(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "List-buckets-success",

s3api/controllers/backend_moq_test.go

@@ -7,9 +7,9 @@ import (
 	"bufio"
 	"context"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3response"
-	"io"
 	"sync"
 )
 
@@ -44,6 +44,9 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketOwnershipControls method")
+//			},
 //			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketPolicy method")
 //			},
@@ -62,6 +65,9 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+//				panic("mock out the GetBucketOwnershipControls method")
+//			},
 //			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 //				panic("mock out the GetBucketPolicy method")
 //			},
@@ -71,7 +77,7 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
 //				panic("mock out the GetBucketVersioning method")
 //			},
-//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 //				panic("mock out the GetObject method")
 //			},
 //			GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
@@ -122,6 +128,9 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+//				panic("mock out the PutBucketOwnershipControls method")
+//			},
 //			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
 //				panic("mock out the PutBucketPolicy method")
 //			},
@@ -195,6 +204,9 @@ type BackendMock struct {
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
+	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
+	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
 	DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -213,6 +225,9 @@ type BackendMock struct {
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketOwnershipControlsFunc mocks the GetBucketOwnershipControls method.
+	GetBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error)
+
 	// GetBucketPolicyFunc mocks the GetBucketPolicy method.
 	GetBucketPolicyFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
 
@@ -223,7 +238,7 @@ type BackendMock struct {
 	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 
 	// GetObjectFunc mocks the GetObject method.
-	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 
 	// GetObjectAclFunc mocks the GetObjectAcl method.
 	GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
@@ -273,6 +288,9 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketOwnershipControlsFunc mocks the PutBucketOwnershipControls method.
+	PutBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error
+
 	// PutBucketPolicyFunc mocks the PutBucketPolicy method.
 	PutBucketPolicyFunc func(contextMoqParam context.Context, bucket string, policy []byte) error
 
@@ -373,6 +391,13 @@ type BackendMock struct {
 			// DeleteBucketInput is the deleteBucketInput argument value.
 			DeleteBucketInput *s3.DeleteBucketInput
 		}
+		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
+		DeleteBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
 		DeleteBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -417,6 +442,13 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketOwnershipControls holds details about calls to the GetBucketOwnershipControls method.
+		GetBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetBucketPolicy holds details about calls to the GetBucketPolicy method.
 		GetBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -444,8 +476,6 @@ type BackendMock struct {
 			ContextMoqParam context.Context
 			// GetObjectInput is the getObjectInput argument value.
 			GetObjectInput *s3.GetObjectInput
-			// Writer is the writer argument value.
-			Writer io.Writer
 		}
 		// GetObjectAcl holds details about calls to the GetObjectAcl method.
 		GetObjectAcl []struct {
@@ -571,6 +601,15 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketOwnershipControls holds details about calls to the PutBucketOwnershipControls method.
+		PutBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Ownership is the ownership argument value.
+			Ownership types.ObjectOwnership
+		}
 		// PutBucketPolicy holds details about calls to the PutBucketPolicy method.
 		PutBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -693,54 +732,57 @@ type BackendMock struct {
 			UploadPartCopyInput *s3.UploadPartCopyInput
 		}
 	}
-	lockAbortMultipartUpload       sync.RWMutex
-	lockChangeBucketOwner          sync.RWMutex
-	lockCompleteMultipartUpload    sync.RWMutex
-	lockCopyObject                 sync.RWMutex
-	lockCreateBucket               sync.RWMutex
-	lockCreateMultipartUpload      sync.RWMutex
-	lockDeleteBucket               sync.RWMutex
-	lockDeleteBucketPolicy         sync.RWMutex
-	lockDeleteBucketTagging        sync.RWMutex
-	lockDeleteObject               sync.RWMutex
-	lockDeleteObjectTagging        sync.RWMutex
-	lockDeleteObjects              sync.RWMutex
-	lockGetBucketAcl               sync.RWMutex
-	lockGetBucketPolicy            sync.RWMutex
-	lockGetBucketTagging           sync.RWMutex
-	lockGetBucketVersioning        sync.RWMutex
-	lockGetObject                  sync.RWMutex
-	lockGetObjectAcl               sync.RWMutex
-	lockGetObjectAttributes        sync.RWMutex
-	lockGetObjectLegalHold         sync.RWMutex
-	lockGetObjectLockConfiguration sync.RWMutex
-	lockGetObjectRetention         sync.RWMutex
-	lockGetObjectTagging           sync.RWMutex
-	lockHeadBucket                 sync.RWMutex
-	lockHeadObject                 sync.RWMutex
-	lockListBuckets                sync.RWMutex
-	lockListBucketsAndOwners       sync.RWMutex
-	lockListMultipartUploads       sync.RWMutex
-	lockListObjectVersions         sync.RWMutex
-	lockListObjects                sync.RWMutex
-	lockListObjectsV2              sync.RWMutex
-	lockListParts                  sync.RWMutex
-	lockPutBucketAcl               sync.RWMutex
-	lockPutBucketPolicy            sync.RWMutex
-	lockPutBucketTagging           sync.RWMutex
-	lockPutBucketVersioning        sync.RWMutex
-	lockPutObject                  sync.RWMutex
-	lockPutObjectAcl               sync.RWMutex
-	lockPutObjectLegalHold         sync.RWMutex
-	lockPutObjectLockConfiguration sync.RWMutex
-	lockPutObjectRetention         sync.RWMutex
-	lockPutObjectTagging           sync.RWMutex
-	lockRestoreObject              sync.RWMutex
-	lockSelectObjectContent        sync.RWMutex
-	lockShutdown                   sync.RWMutex
-	lockString                     sync.RWMutex
-	lockUploadPart                 sync.RWMutex
-	lockUploadPartCopy             sync.RWMutex
+	lockAbortMultipartUpload          sync.RWMutex
+	lockChangeBucketOwner             sync.RWMutex
+	lockCompleteMultipartUpload       sync.RWMutex
+	lockCopyObject                    sync.RWMutex
+	lockCreateBucket                  sync.RWMutex
+	lockCreateMultipartUpload         sync.RWMutex
+	lockDeleteBucket                  sync.RWMutex
+	lockDeleteBucketOwnershipControls sync.RWMutex
+	lockDeleteBucketPolicy            sync.RWMutex
+	lockDeleteBucketTagging           sync.RWMutex
+	lockDeleteObject                  sync.RWMutex
+	lockDeleteObjectTagging           sync.RWMutex
+	lockDeleteObjects                 sync.RWMutex
+	lockGetBucketAcl                  sync.RWMutex
+	lockGetBucketOwnershipControls    sync.RWMutex
+	lockGetBucketPolicy               sync.RWMutex
+	lockGetBucketTagging              sync.RWMutex
+	lockGetBucketVersioning           sync.RWMutex
+	lockGetObject                     sync.RWMutex
+	lockGetObjectAcl                  sync.RWMutex
+	lockGetObjectAttributes           sync.RWMutex
+	lockGetObjectLegalHold            sync.RWMutex
+	lockGetObjectLockConfiguration    sync.RWMutex
+	lockGetObjectRetention            sync.RWMutex
+	lockGetObjectTagging              sync.RWMutex
+	lockHeadBucket                    sync.RWMutex
+	lockHeadObject                    sync.RWMutex
+	lockListBuckets                   sync.RWMutex
+	lockListBucketsAndOwners          sync.RWMutex
+	lockListMultipartUploads          sync.RWMutex
+	lockListObjectVersions            sync.RWMutex
+	lockListObjects                   sync.RWMutex
+	lockListObjectsV2                 sync.RWMutex
+	lockListParts                     sync.RWMutex
+	lockPutBucketAcl                  sync.RWMutex
+	lockPutBucketOwnershipControls    sync.RWMutex
+	lockPutBucketPolicy               sync.RWMutex
+	lockPutBucketTagging              sync.RWMutex
+	lockPutBucketVersioning           sync.RWMutex
+	lockPutObject                     sync.RWMutex
+	lockPutObjectAcl                  sync.RWMutex
+	lockPutObjectLegalHold            sync.RWMutex
+	lockPutObjectLockConfiguration    sync.RWMutex
+	lockPutObjectRetention            sync.RWMutex
+	lockPutObjectTagging              sync.RWMutex
+	lockRestoreObject                 sync.RWMutex
+	lockSelectObjectContent           sync.RWMutex
+	lockShutdown                      sync.RWMutex
+	lockString                        sync.RWMutex
+	lockUploadPart                    sync.RWMutex
+	lockUploadPartCopy                sync.RWMutex
 }
 
 // AbortMultipartUpload calls AbortMultipartUploadFunc.
@@ -1003,6 +1045,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketOwnershipControls calls DeleteBucketOwnershipControlsFunc.
+func (mock *BackendMock) DeleteBucketOwnershipControls(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.DeleteBucketOwnershipControlsFunc: method is nil but Backend.DeleteBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketOwnershipControls.Lock()
+	mock.calls.DeleteBucketOwnershipControls = append(mock.calls.DeleteBucketOwnershipControls, callInfo)
+	mock.lockDeleteBucketOwnershipControls.Unlock()
+	return mock.DeleteBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketOwnershipControlsCalls gets all the calls that were made to DeleteBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketOwnershipControlsCalls())
+func (mock *BackendMock) DeleteBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketOwnershipControls.RLock()
+	calls = mock.calls.DeleteBucketOwnershipControls
+	mock.lockDeleteBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // DeleteBucketPolicy calls DeleteBucketPolicyFunc.
 func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketPolicyFunc == nil {
@@ -1223,6 +1301,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketOwnershipControls calls GetBucketOwnershipControlsFunc.
+func (mock *BackendMock) GetBucketOwnershipControls(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+	if mock.GetBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.GetBucketOwnershipControlsFunc: method is nil but Backend.GetBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketOwnershipControls.Lock()
+	mock.calls.GetBucketOwnershipControls = append(mock.calls.GetBucketOwnershipControls, callInfo)
+	mock.lockGetBucketOwnershipControls.Unlock()
+	return mock.GetBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// GetBucketOwnershipControlsCalls gets all the calls that were made to GetBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketOwnershipControlsCalls())
+func (mock *BackendMock) GetBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketOwnershipControls.RLock()
+	calls = mock.calls.GetBucketOwnershipControls
+	mock.lockGetBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // GetBucketPolicy calls GetBucketPolicyFunc.
 func (mock *BackendMock) GetBucketPolicy(contextMoqParam context.Context, bucket string) ([]byte, error) {
 	if mock.GetBucketPolicyFunc == nil {
@@ -1332,23 +1446,21 @@ func (mock *BackendMock) GetBucketVersioningCalls() []struct {
 }
 
 // GetObject calls GetObjectFunc.
-func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	if mock.GetObjectFunc == nil {
 		panic("BackendMock.GetObjectFunc: method is nil but Backend.GetObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}{
 		ContextMoqParam: contextMoqParam,
 		GetObjectInput:  getObjectInput,
-		Writer:          writer,
 	}
 	mock.lockGetObject.Lock()
 	mock.calls.GetObject = append(mock.calls.GetObject, callInfo)
 	mock.lockGetObject.Unlock()
-	return mock.GetObjectFunc(contextMoqParam, getObjectInput, writer)
+	return mock.GetObjectFunc(contextMoqParam, getObjectInput)
 }
 
 // GetObjectCalls gets all the calls that were made to GetObject.
@@ -1358,12 +1470,10 @@ func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInp
 func (mock *BackendMock) GetObjectCalls() []struct {
 	ContextMoqParam context.Context
 	GetObjectInput  *s3.GetObjectInput
-	Writer          io.Writer
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}
 	mock.lockGetObject.RLock()
 	calls = mock.calls.GetObject
@@ -1971,6 +2081,46 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketOwnershipControls calls PutBucketOwnershipControlsFunc.
+func (mock *BackendMock) PutBucketOwnershipControls(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+	if mock.PutBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.PutBucketOwnershipControlsFunc: method is nil but Backend.PutBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Ownership:       ownership,
+	}
+	mock.lockPutBucketOwnershipControls.Lock()
+	mock.calls.PutBucketOwnershipControls = append(mock.calls.PutBucketOwnershipControls, callInfo)
+	mock.lockPutBucketOwnershipControls.Unlock()
+	return mock.PutBucketOwnershipControlsFunc(contextMoqParam, bucket, ownership)
+}
+
+// PutBucketOwnershipControlsCalls gets all the calls that were made to PutBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketOwnershipControlsCalls())
+func (mock *BackendMock) PutBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Ownership       types.ObjectOwnership
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}
+	mock.lockPutBucketOwnershipControls.RLock()
+	calls = mock.calls.PutBucketOwnershipControls
+	mock.lockPutBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // PutBucketPolicy calls PutBucketPolicyFunc.
 func (mock *BackendMock) PutBucketPolicy(contextMoqParam context.Context, bucket string, policy []byte) error {
 	if mock.PutBucketPolicyFunc == nil {

s3api/controllers/base.go

@@ -402,7 +402,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		Key:       &key,
 		Range:     &acceptRange,
 		VersionId: &versionId,
-	}, ctx.Response().BodyWriter())
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -412,15 +412,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 				BucketOwner: parsedAcl.Owner,
 			})
 	}
-	if res == nil {
-		return SendResponse(ctx, fmt.Errorf("get object nil response"),
-			&MetaOpts{
-				Logger:      c.logger,
-				MetricsMng:  c.mm,
-				Action:      metrics.ActionGetObject,
-				BucketOwner: parsedAcl.Owner,
-			})
-	}
 
 	utils.SetMetaHeaders(ctx, res.Metadata)
 	var lastmod string
@@ -429,10 +420,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	utils.SetResponseHeaders(ctx, []utils.CustomHeader{
-		{
-			Key:   "Content-Length",
-			Value: fmt.Sprint(getint64(res.ContentLength)),
-		},
 		{
 			Key:   "Content-Type",
 			Value: getstring(res.ContentType),
@@ -477,6 +464,10 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		status = http.StatusPartialContent
 	}
 
+	if res.Body != nil {
+		ctx.Response().SetBodyStream(res.Body, int(getint64(res.ContentLength)))
+	}
+
 	return SendResponse(ctx, nil,
 		&MetaOpts{
 			Logger:        c.logger,
@@ -566,6 +557,43 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Readonly:      c.readonly,
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketOwnershipControlsAction,
+		})
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionGetBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		data, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket)
+		return SendXMLResponse(ctx,
+			s3response.OwnershipControls{
+				Rules: []types.OwnershipControlsRule{
+					{
+						ObjectOwnership: data,
+					},
+				},
+			}, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionGetBucketOwnershipControls,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
@@ -933,6 +961,9 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp")
 	granWrite := ctx.Get("X-Amz-Grant-Write")
 	grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp")
+	objectOwnership := types.ObjectOwnership(
+		ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)),
+	)
 	mfa := ctx.Get("X-Amz-Mfa")
 	contentMD5 := ctx.Get("Content-MD5")
 	acct := ctx.Locals("account").(auth.Account)
@@ -1000,6 +1031,57 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
+		var ownershipControls s3response.OwnershipControls
+		if err := xml.Unmarshal(ctx.Body(), &ownershipControls); err != nil {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		if len(ownershipControls.Rules) != 1 || !utils.IsValidOwnership(ownershipControls.Rules[0].ObjectOwnership) {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Readonly:      c.readonly,
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketOwnershipControlsAction,
+		}); err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err := c.be.PutBucketOwnershipControls(ctx.Context(), bucket, ownershipControls.Rules[0].ObjectOwnership)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionPutBucketOwnershipControls,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
@@ -1141,10 +1223,33 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
+		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 		var input *s3.PutBucketAclInput
 
-		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-		err := auth.VerifyAccess(ctx.Context(), c.be,
+		ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket)
+		if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketAcl,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+		if ownership == types.ObjectOwnershipBucketOwnerEnforced {
+			if c.debug {
+				log.Println("bucket acls are disabled")
+			}
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAclNotSupported),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketAcl,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = auth.VerifyAccess(ctx.Context(), c.be,
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
@@ -1259,7 +1364,6 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			}
 		}
 
-		fmt.Println(*input, parsedAcl)
 		updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam)
 		if err != nil {
 			return SendResponse(ctx, err,
@@ -1289,16 +1393,45 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				Action:     metrics.ActionCreateBucket,
 			})
 	}
+	if ok := utils.IsValidOwnership(objectOwnership); !ok {
+		if c.debug {
+			log.Printf("invalid bucket object ownership: %v", objectOwnership)
+		}
+		return SendResponse(ctx, s3err.APIError{
+			Code:           "InvalidArgument",
+			Description:    fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership),
+			HTTPStatusCode: http.StatusBadRequest,
+		},
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionCreateBucket,
+				BucketOwner: acct.Access,
+			})
+	}
+
+	if acl+grants != "" && objectOwnership == types.ObjectOwnershipBucketOwnerEnforced {
+		if c.debug {
+			log.Printf("bucket acls are disabled for %v object ownership", objectOwnership)
+		}
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership),
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionCreateBucket,
+				BucketOwner: acct.Access,
+			})
+	}
 
 	if acl != "" && grants != "" {
 		if c.debug {
 			log.Printf("invalid request: %q (grants) %q (acl)", grants, acl)
 		}
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest),
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants),
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
-				Action:      metrics.ActionPutBucketAcl,
+				Action:      metrics.ActionCreateBucket,
 				BucketOwner: acct.Access,
 			})
 	}
@@ -1334,7 +1467,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 
 	err = c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{
 		Bucket:                     &bucket,
-		ObjectOwnership:            types.ObjectOwnership(acct.Access),
+		ObjectOwnership:            objectOwnership,
 		ObjectLockEnabledForBucket: &lockEnabled,
 	}, updAcl)
 	return SendResponse(ctx, err,
@@ -1379,6 +1512,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		contentLengthStr = "0"
 	}
 	bucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner")
+	storageClass := ctx.Get("X-Amz-Storage-Class")
 
 	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
 
@@ -1882,6 +2016,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				CopySourceIfUnmodifiedSince: umtime,
 				ExpectedBucketOwner:         &acct.Access,
 				Metadata:                    metadata,
+				StorageClass:                types.StorageClass(storageClass),
 			})
 		if err == nil {
 			return SendXMLResponse(ctx, res.CopyObjectResult, err,
@@ -2040,6 +2175,38 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+		err := auth.VerifyAccess(ctx.Context(), c.be,
+			auth.AccessOptions{
+				Readonly:      c.readonly,
+				Acl:           parsedAcl,
+				AclPermission: types.PermissionWrite,
+				IsRoot:        isRoot,
+				Acc:           acct,
+				Bucket:        bucket,
+				Action:        auth.PutBucketOwnershipControlsAction,
+			})
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionDeleteBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = c.be.DeleteBucketOwnershipControls(ctx.Context(), bucket)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionDeleteBucketOwnershipControls,
+				BucketOwner: parsedAcl.Owner,
+				Status:      http.StatusNoContent,
+			})
+	}
+
 	if ctx.Request().URI().QueryArgs().Has("policy") {
 		err := auth.VerifyAccess(ctx.Context(), c.be,
 			auth.AccessOptions{
@@ -2546,23 +2713,20 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 		key = key + "/"
 	}
 
-	var restoreRequest s3.RestoreObjectInput
 	if ctx.Request().URI().QueryArgs().Has("restore") {
-		err := xml.Unmarshal(ctx.Body(), &restoreRequest)
-		if err != nil {
-			if c.debug {
-				log.Printf("error unmarshalling restore object: %v", err)
+		var restoreRequest types.RestoreRequest
+		if err := xml.Unmarshal(ctx.Body(), &restoreRequest); err != nil {
+			if !errors.Is(io.EOF, err) {
+				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+					&MetaOpts{
+						Logger:      c.logger,
+						MetricsMng:  c.mm,
+						Action:      metrics.ActionRestoreObject,
+						BucketOwner: parsedAcl.Owner,
+					})
 			}
-			return SendResponse(ctx, err,
-				&MetaOpts{
-					Logger:      c.logger,
-					MetricsMng:  c.mm,
-					Action:      metrics.ActionRestoreObject,
-					BucketOwner: parsedAcl.Owner,
-				})
 		}
-
-		err = auth.VerifyAccess(ctx.Context(), c.be,
+		err := auth.VerifyAccess(ctx.Context(), c.be,
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
@@ -2583,10 +2747,11 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		restoreRequest.Bucket = &bucket
-		restoreRequest.Key = &key
-
-		err = c.be.RestoreObject(ctx.Context(), &restoreRequest)
+		err = c.be.RestoreObject(ctx.Context(), &s3.RestoreObjectInput{
+			Bucket:         &bucket,
+			Key:            &key,
+			RestoreRequest: &restoreRequest,
+		})
 		return SendResponse(ctx, err,
 			&MetaOpts{
 				Logger:      c.logger,

s3api/controllers/base_test.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -191,7 +190,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 				return s3response.GetObjectAttributesResult{}, nil
 			},
-			GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+			GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
 					Metadata:        map[string]string{"hello": "world"},
 					ContentType:     getPtr("application/xml"),
@@ -395,6 +394,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 				return objectLockResult, nil
 			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerEnforced, nil
+			},
 		},
 	}
 
@@ -448,6 +450,15 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 404,
 		},
+		{
+			name: "Get-bucket-ownership-control-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-bucket-tagging-success",
 			app:  app,
@@ -562,7 +573,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	app := fiber.New()
 
 	// Mock valid acl
-	acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
+	acl := auth.ACL{Owner: "valid access"}
 	acldata, err := json.Marshal(acl)
 	if err != nil {
 		t.Errorf("Failed to parse the params: %v", err.Error())
@@ -636,6 +647,22 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</ObjectLockConfiguration>
 	`
 
+	ownershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
+	invalidOwnershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>invalid_value</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -659,6 +686,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
 				return nil
 			},
+			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+				return nil
+			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerPreferred, nil
+			},
 		},
 	}
 	// Mock ctx.Locals
@@ -691,6 +724,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
 	errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
+	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -716,6 +752,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Put-bucket-ownership-controls-invalid-ownership",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Put-object-lock-configuration-invalid-body",
 			app:  app,
@@ -816,7 +870,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 200,
 		},
 		{
-			name: "Put-bucket-invalid-bucket-name",
+			name: "Create-bucket-invalid-acl-ownership-combination",
+			app:  app,
+			args: args{
+				req: invAclOwnershipReq,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Create-bucket-invalid-bucket-name",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
@@ -825,7 +888,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 400,
 		},
 		{
-			name: "Put-bucket-success",
+			name: "Create-bucket-success",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
@@ -1160,6 +1223,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 				return nil
 			},
+			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
+			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
 		},
 	}
 
@@ -1198,6 +1267,23 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			wantErr:    false,
 			statusCode: 204,
 		},
+		{
+			name: "Delete-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		}, {
+			name: "Delete-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -1646,20 +1732,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 		{
 			name: "Restore-object-success",
 			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
-		{
-			name: "Restore-object-error",
-			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 200,
 		},
 		{
 			name: "Select-object-content-invalid-body",

s3api/controllers/iam_moq_test.go

@@ -33,6 +33,9 @@ var _ auth.IAMService = &IAMServiceMock{}
 //			ShutdownFunc: func() error {
 //				panic("mock out the Shutdown method")
 //			},
+//			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+//				panic("mock out the UpdateUserAccount method")
+//			},
 //		}
 //
 //		// use mockedIAMService in code that requires auth.IAMService
@@ -55,6 +58,9 @@ type IAMServiceMock struct {
 	// ShutdownFunc mocks the Shutdown method.
 	ShutdownFunc func() error
 
+	// UpdateUserAccountFunc mocks the UpdateUserAccount method.
+	UpdateUserAccountFunc func(access string, props auth.MutableProps) error
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// CreateAccount holds details about calls to the CreateAccount method.
@@ -78,12 +84,20 @@ type IAMServiceMock struct {
 		// Shutdown holds details about calls to the Shutdown method.
 		Shutdown []struct {
 		}
+		// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
+		UpdateUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+			// Props is the props argument value.
+			Props auth.MutableProps
+		}
 	}
 	lockCreateAccount     sync.RWMutex
 	lockDeleteUserAccount sync.RWMutex
 	lockGetUserAccount    sync.RWMutex
 	lockListUserAccounts  sync.RWMutex
 	lockShutdown          sync.RWMutex
+	lockUpdateUserAccount sync.RWMutex
 }
 
 // CreateAccount calls CreateAccountFunc.
@@ -235,3 +249,39 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
 	mock.lockShutdown.RUnlock()
 	return calls
 }
+
+// UpdateUserAccount calls UpdateUserAccountFunc.
+func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
+	if mock.UpdateUserAccountFunc == nil {
+		panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+		Props  auth.MutableProps
+	}{
+		Access: access,
+		Props:  props,
+	}
+	mock.lockUpdateUserAccount.Lock()
+	mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
+	mock.lockUpdateUserAccount.Unlock()
+	return mock.UpdateUserAccountFunc(access, props)
+}
+
+// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.UpdateUserAccountCalls())
+func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
+	Access string
+	Props  auth.MutableProps
+} {
+	var calls []struct {
+		Access string
+		Props  auth.MutableProps
+	}
+	mock.lockUpdateUserAccount.RLock()
+	calls = mock.calls.UpdateUserAccount
+	mock.lockUpdateUserAccount.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -50,7 +50,8 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			!ctx.Request().URI().QueryArgs().Has("tagging") &&
 			!ctx.Request().URI().QueryArgs().Has("versioning") &&
 			!ctx.Request().URI().QueryArgs().Has("policy") &&
-			!ctx.Request().URI().QueryArgs().Has("object-lock") {
+			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
+			!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}

s3api/router.go

@@ -40,6 +40,9 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		// DeleteUsers admin api
 		app.Patch("/delete-user", adminController.DeleteUser)
 
+		// UpdateUser admin api
+		app.Patch("update-user", adminController.UpdateUser)
+
 		// ListUsers admin api
 		app.Patch("/list-users", adminController.ListUsers)
 

s3api/utils/utils.go

@@ -326,3 +326,16 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 		LegalHoldStatus: legalHold,
 	}, nil
 }
+
+func IsValidOwnership(val types.ObjectOwnership) bool {
+	switch val {
+	case types.ObjectOwnershipBucketOwnerEnforced:
+		return true
+	case types.ObjectOwnershipBucketOwnerPreferred:
+		return true
+	case types.ObjectOwnershipObjectWriter:
+		return true
+	default:
+		return false
+	}
+}

s3api/utils/utils_test.go

@@ -335,3 +335,50 @@ func TestFilterObjectAttributes(t *testing.T) {
 		})
 	}
 }
+
+func TestIsValidOwnership(t *testing.T) {
+	type args struct {
+		val types.ObjectOwnership
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "valid-BucketOwnerEnforced",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerEnforced,
+			},
+			want: true,
+		},
+		{
+			name: "valid-BucketOwnerPreferred",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerPreferred,
+			},
+			want: true,
+		},
+		{
+			name: "valid-ObjectWriter",
+			args: args{
+				val: types.ObjectOwnershipObjectWriter,
+			},
+			want: true,
+		},
+		{
+			name: "invalid_value",
+			args: args{
+				val: types.ObjectOwnership("invalid_value"),
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidOwnership(tt.args.val); got != tt.want {
+				t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -123,6 +123,10 @@ const (
 	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
 	ErrRequestTimeTooSkewed
+	ErrInvalidBucketAclWithObjectOwnership
+	ErrBothCannedAndHeaderGrants
+	ErrOwnershipControlsNotFound
+	ErrAclNotSupported
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
@@ -472,6 +476,26 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The difference between the request time and the server's time is too large.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidBucketAclWithObjectOwnership: {
+		Code:           "ErrInvalidBucketAclWithObjectOwnership",
+		Description:    "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBothCannedAndHeaderGrants: {
+		Code:           "InvalidRequest",
+		Description:    "Specifying both Canned ACLs and Header Grants is not allowed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOwnershipControlsNotFound: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAclNotSupported: {
+		Code:           "AccessControlListNotSupported",
+		Description:    "The bucket does not allow ACLs",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {

s3response/s3response.go

@@ -217,3 +217,7 @@ type Grantee struct {
 	ID          string
 	DisplayName string
 }
+
+type OwnershipControls struct {
+	Rules []types.OwnershipControlsRule `xml:"Rule"`
+}

tests/.env.default

@@ -14,4 +14,7 @@ SECRETS_FILE=./tests/.secrets
 MC_ALIAS=versity
 LOG_LEVEL=2
 GOCOVERDIR=$PWD/cover
-USERS_FOLDER=$PWD/iam
+USERS_FOLDER=$PWD/iam
+#TEST_LOG_FILE=test.log
+#VERSITY_LOG_FILE=versity.log
+IAM_TYPE=folder

tests/README.md

@@ -10,6 +10,7 @@
    * **s3cmd**:  Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
    * **mc**:  Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
 3. Install BATS.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
+4. If running on Mac OS, install **jq** with the command `brew install jq`.
 4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
 5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
 ```
@@ -30,6 +31,10 @@
 8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
 9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
+### Static Bucket Mode
+
+To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`.  Two utility functions are included, if needed, to create, and delete buckets for this:  `tests/setup_static.sh` and `tests/remove_static.sh`.
+
 ### S3 Backend
 
 Instructions are mostly the same; however, testing with the S3 backend requires two S3 accounts.  Ideally, these are two real accounts, but one can also be a dummy account that versity uses internally.

tests/commands/abort_multipart_upload.sh

@@ -2,13 +2,24 @@
 
 abort_multipart_upload() {
   if [ $# -ne 3 ]; then
-    echo "command to run abort requires bucket, key, upload ID"
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID"
     return 1
   fi
+  if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $error"
+    return 1
+  fi
+  return 0
+}
 
-  error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3") || local aborted=$?
-  if [[ $aborted -ne 0 ]]; then
-    echo "Error aborting upload: $error"
+abort_multipart_upload_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID, username, password"
+    return 1
+  fi
+  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $abort_multipart_upload_error"
+    export abort_multipart_upload_error
     return 1
   fi
   return 0

tests/commands/copy_object.sh

@@ -12,6 +12,7 @@ copy_object() {
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
     error=$(aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
+    log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4"
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
     error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?

tests/commands/create_bucket.sh

@@ -5,26 +5,28 @@
 # return 0 for success, 1 for failure
 create_bucket() {
   if [ $# -ne 2 ]; then
-    echo "create bucket missing command type, bucket name"
+    log 2 "create bucket missing command type, bucket name"
     return 1
   fi
 
   local exit_code=0
   local error
+  log 6 "create bucket"
   if [[ $1 == 's3' ]]; then
     error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
     error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
+    log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
     error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
-    echo "invalid command type $1"
+    log 2 "invalid command type $1"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    echo "error creating bucket: $error"
+    log 2 "error creating bucket: $error"
     return 1
   fi
   return 0

tests/commands/create_multipart_upload.sh

@@ -9,16 +9,38 @@ create_multipart_upload() {
     return 1
   fi
 
-  local multipart_data
-  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2") || local created=$?
-  if [[ $created -ne 0 ]]; then
-    log 2 "Error creating multipart upload: $upload_id"
+  if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+    log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
 
-  upload_id=$(echo "$multipart_data" | jq '.UploadId')
+  if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
+    log 2 "error parsing upload ID: $upload_id"
+    return 1
+  fi
   upload_id="${upload_id//\"/}"
   export upload_id
+  return 0
+}
+
+create_multipart_upload_with_user() {
+  if [ $# -ne 4 ]; then
+    log 2 "create multipart upload function must have bucket, key, username, password"
+    return 1
+  fi
+
+  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+    log 2 "Error creating multipart upload: $multipart_data"
+    return 1
+  fi
+
+  if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
+    log 2 "error parsing upload ID: $upload_id"
+    return 1
+  fi
+  upload_id="${upload_id//\"/}"
+  export upload_id
+  return 0
 }
 
 create_multipart_upload_params() {
@@ -56,7 +78,7 @@ create_multipart_upload_custom() {
   local multipart_data
   log 5 "additional create multipart params"
   for i in "$@"; do
-    log 5 $i
+    log 5 "$i"
   done
   log 5 "${*:3}"
   log 5 "aws --no-verify-ssl s3api create-multipart-upload --bucket $1 --key $2 ${*:3}"

tests/commands/delete_bucket_policy.sh

@@ -5,14 +5,14 @@ delete_bucket_policy() {
     log 2 "delete bucket policy command requires command type, bucket"
     return 1
   fi
-  if [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2") || delete_result=$?
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2") || delete_result=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure anonymous set none "$MC_ALIAS/$2") || delete_result=$?
+    error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
   else
-    log 2 "command 'get bucket policy' not implemented for '$1'"
+    log 2 "command 'delete bucket policy' not implemented for '$1'"
     return 1
   fi
   if [[ $delete_result -ne 0 ]]; then
@@ -20,4 +20,17 @@ delete_bucket_policy() {
     return 1
   fi
   return 0
+}
+
+delete_bucket_policy_with_user() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete bucket policy with user' command requires bucket, username, password"
+    return 1
+  fi
+  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
+    log 2 "error deleting bucket policy: $delete_bucket_policy_error"
+    export delete_bucket_policy_error
+    return 1
+  fi
+  return 0
 }

tests/commands/delete_object.sh

@@ -6,22 +6,22 @@ delete_object() {
     return 1
   fi
   local exit_code=0
-  local error
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
+    delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
   fi
   log 5 "delete object exit code: $exit_code"
   if [ $exit_code -ne 0 ]; then
-    log 2 "error deleting object: $error"
+    log 2 "error deleting object: $delete_object_error"
+    export delete_object_error
     return 1
   fi
   return 0
@@ -34,18 +34,18 @@ delete_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3' ]]; then
-    error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
+    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
   else
     log 2 "command 'delete object with user' not implemented for '$1'"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    log 2 "error deleting object: $error"
-    export error
+    log 2 "error deleting object: $delete_object_error"
+    export delete_object_error
     return 1
   fi
   return 0

tests/commands/get_bucket_acl.sh

@@ -19,4 +19,17 @@ get_bucket_acl() {
     return 1
   fi
   export acl
+}
+
+get_bucket_acl_with_user() {
+  if [ $# -ne 3 ]; then
+    log 2 "'get bucket ACL with user' command requires bucket name, username, password"
+    return 1
+  fi
+  if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
+    log 2 "error getting bucket ACLs: $bucket_acl"
+    return 1
+  fi
+  export bucket_acl
+  return 0
 }

tests/commands/get_bucket_ownership_controls.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+get_bucket_ownership_controls() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get bucket ownership controls' command requires bucket name"
+    return 1
+  fi
+
+  if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
+    log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
+    return 1
+  fi
+
+  log 5 "Raw bucket Ownership Controls:  $raw_bucket_ownership_controls"
+  bucket_ownership_controls=$(echo "$raw_bucket_ownership_controls" | grep -v "InsecureRequestWarning")
+  export bucket_ownership_controls
+  return 0
+}
+
+get_object_ownership_rule() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get object ownership rule' command requires bucket name"
+    return 1
+  fi
+  if ! get_bucket_ownership_controls "$1"; then
+    log 2 "error getting bucket ownership controls"
+    return 1
+  fi
+  if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then
+    log 2 "error getting object ownership rule: $object_ownership_rule"
+    return 1
+  fi
+  log 5 "object ownership rule: $object_ownership_rule"
+  export object_ownership_rule
+  return 0
+}

tests/commands/get_bucket_policy.sh

@@ -2,22 +2,22 @@
 
 get_bucket_policy() {
   if [[ $# -ne 2 ]]; then
-    echo "get bucket policy command requires command type, bucket"
+    log 2 "get bucket policy command requires command type, bucket"
     return 1
   fi
   local get_bucket_policy_result=0
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
     get_bucket_policy_aws "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
     get_bucket_policy_mc "$2" || get_bucket_policy_result=$?
   else
-    echo "command 'get bucket policy' not implemented for '$1'"
+    log 2 "command 'get bucket policy' not implemented for '$1'"
     return 1
   fi
   if [[ $get_bucket_policy_result -ne 0 ]]; then
-    echo "error getting policy: $bucket_policy"
+    log 2 "error getting policy: $bucket_policy"
     return 1
   fi
   export bucket_policy
@@ -26,22 +26,41 @@ get_bucket_policy() {
 
 get_bucket_policy_aws() {
   if [[ $# -ne 1 ]]; then
-    echo "aws 'get bucket policy' command requires bucket"
+    log 2 "aws 'get bucket policy' command requires bucket"
     return 1
   fi
-  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || get_result=$?
-  if [[ $policy_json == *"InsecureRequestWarning"* ]]; then
-    policy_json=$(awk 'NR>2' <<< "$policy_json")
-  fi
+  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
+  policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
+  log 5 "$policy_json"
   if [[ $get_result -ne 0 ]]; then
     if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
       bucket_policy=
     else
-      echo "error getting policy: $policy_json"
+      log 2 "error getting policy: $policy_json"
       return 1
     fi
   else
-    bucket_policy=$(echo "{$policy_json}" | jq -r '.Policy')
+    bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
+  fi
+  export bucket_policy
+  return 0
+}
+
+get_bucket_policy_with_user() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'get bucket policy with user' command requires bucket, username, password"
+    return 1
+  fi
+  if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
+    policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
+    bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
+  else
+    if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
+      bucket_policy=
+    else
+      log 2 "error getting policy for user $2: $policy_json"
+      return 1
+    fi
   fi
   export bucket_policy
   return 0
@@ -49,27 +68,34 @@ get_bucket_policy_aws() {
 
 get_bucket_policy_s3cmd() {
   if [[ $# -ne 1 ]]; then
-    echo "s3cmd 'get bucket policy' command requires bucket"
+    log 2 "s3cmd 'get bucket policy' command requires bucket"
     return 1
   fi
 
-  info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1") || get_result=$?
-  if [[ $get_result -ne 0 ]]; then
-    echo "error getting bucket policy: $info"
+  if ! info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
+    log 2 "error getting bucket policy: $info"
     return 1
   fi
 
+  log 5 "policy info: $info"
   bucket_policy=""
   policy_brackets=false
+  # NOTE:  versitygw sends policies back in multiple lines here, direct in single line
   while IFS= read -r line; do
     if [[ $policy_brackets == false ]]; then
       policy_line=$(echo "$line" | grep 'Policy: ')
       if [[ $policy_line != "" ]]; then
-        if [[ $policy_line != *'{' ]]; then
+        if [[ $policy_line != *'{'* ]]; then
           break
         fi
-        policy_brackets=true
-        bucket_policy+="{"
+        if [[ $policy_line == *'}'* ]]; then
+          log 5 "policy on single line"
+          bucket_policy=${policy_line//Policy:/}
+          break
+        else
+          policy_brackets=true
+          bucket_policy+="{"
+        fi
       fi
     else
       bucket_policy+=$line
@@ -78,6 +104,7 @@ get_bucket_policy_s3cmd() {
       fi
     fi
   done <<< "$info"
+  log 5 "bucket policy: $bucket_policy"
   export bucket_policy
   return 0
 }

tests/commands/get_object.sh

@@ -39,3 +39,24 @@ get_object_with_range() {
   fi
   return 0
 }
+
+get_object_with_user() {
+  if [ $# -ne 6 ]; then
+    log 2 "'get object with user' command requires command type, bucket, key, save location, aws ID, aws secret key"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+  else
+    log 2 "'get object with user' command not implemented for '$1'"
+    return 1
+  fi
+  log 5 "put object exit code: $exit_code"
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error getting object: $get_object_error"
+    export get_object_error
+    return 1
+  fi
+  return 0
+}

tests/commands/list_buckets.sh

@@ -7,7 +7,6 @@ list_buckets() {
   fi
 
   local exit_code=0
-  local error
   if [[ $1 == 's3' ]]; then
     buckets=$(aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then

tests/commands/list_multipart_uploads.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+list_multipart_uploads() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'list multipart uploads' command requires bucket name"
+    return 1
+  fi
+  if ! uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+    log 2 "error listing uploads: $uploads"
+    return 1
+  fi
+  export uploads
+}
+
+list_multipart_uploads_with_user() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'list multipart uploads' command requires bucket name, username, password"
+    return 1
+  fi
+  if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+    log 2 "error listing uploads: $uploads"
+    list_multipart_uploads_error=$uploads
+    export list_multipart_uploads_error
+    return 1
+  fi
+  export uploads
+}

tests/commands/put_bucket_acl.sh

@@ -2,7 +2,7 @@
 
 put_bucket_acl() {
   if [[ $# -ne 3 ]]; then
-    log 2 "put bucket acl command requires command type, bucket name, acls"
+    log 2 "put bucket acl command requires command type, bucket name, acls or username"
     return 1
   fi
   local error=""
@@ -11,7 +11,7 @@ put_bucket_acl() {
     log 5 "bucket name: $2, acls: $3"
     error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$2" --access-control-policy "file://$3" 2>&1) || put_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$2" --acl-grant=read:ABCDEFG 2>&1) || put_result=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$2" --acl-grant=read:"$3" 2>&1) || put_result=$?
   else
     log 2 "put_bucket_acl not implemented for '$1'"
     return 1
@@ -21,4 +21,28 @@ put_bucket_acl() {
     return 1
   fi
   return 0
-}
+}
+
+put_bucket_canned_acl() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "'put bucket canned acl' command requires bucket name, canned ACL"
+    return 1
+  fi
+  if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2"); then
+    log 2 "error re-setting bucket acls: $error"
+    return 1
+  fi
+  return 0
+}
+
+put_bucket_canned_acl_with_user() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "'put bucket canned acl with user' command requires bucket name, canned ACL, username, password"
+    return 1
+  fi
+  if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2"); then
+    log 2 "error re-setting bucket acls: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/put_bucket_ownership_controls.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+put_bucket_ownership_controls() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "'put bucket ownership controls' command requires bucket name, control"
+    return 1
+  fi
+  if ! controls_error=$(aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" \
+      --ownership-controls="Rules=[{ObjectOwnership=$2}]" 2>&1); then
+    log 2 "error putting bucket ownership controls: $controls_error"
+    return 1
+  fi
+  return 0
+}

tests/commands/put_bucket_policy.sh

@@ -2,22 +2,39 @@
 
 put_bucket_policy() {
   if [[ $# -ne 3 ]]; then
-    log 2 "get bucket policy command requires command type, bucket, policy file"
+    log 2 "'put bucket policy' command requires command type, bucket, policy file"
     return 1
   fi
+  local put_policy_result=0
   if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_result=$?
+    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_result=$?
+    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
-    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_result=$?
+    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$?
   else
     log 2 "command 'put bucket policy' not implemented for '$1'"
     return 1
   fi
-  if [[ $put_result -ne 0 ]]; then
-    log 2 "error putting policy: $policy"
+  if [[ $put_policy_result -ne 0 ]]; then
+    put_bucket_policy_error=$policy
+    log 2 "error putting policy: $put_bucket_policy_error"
+    export put_bucket_policy_error
     return 1
   fi
   return 0
-}
+}
+
+put_bucket_policy_with_user() {
+  if [[ $# -ne 4 ]]; then
+    log 2 "'put bucket policy with user' command requires bucket, policy file, username, password"
+    return 1
+  fi
+  if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then
+    log 2 "error putting bucket policy with user $3: $policy"
+    put_bucket_policy_error=$policy
+    export put_bucket_policy_error
+    return 1
+  fi
+  return 0
+}

tests/commands/put_object.sh

@@ -34,15 +34,15 @@ put_object_with_user() {
   fi
   local exit_code=0
   if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+    put_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
   else
     log 2 "'put object with user' command not implemented for '$1'"
     return 1
   fi
   log 5 "put object exit code: $exit_code"
   if [ $exit_code -ne 0 ]; then
-    log 2 "error putting object into bucket: $error"
-    export error
+    log 2 "error putting object into bucket: $put_object_error"
+    export put_object_error
     return 1
   fi
   return 0

tests/commands/upload_part_copy.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+upload_part_copy() {
+  if [ $# -ne 5 ]; then
+    echo "upload multipart part copy function must have bucket, key, upload ID, file name, part number"
+    return 1
+  fi
+  local etag_json
+  echo "$1 $2 $3 $4 $5"
+  etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
+  if [[ $uploaded -ne 0 ]]; then
+    echo "Error uploading part $5: $etag_json"
+    return 1
+  fi
+  etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag')
+  export etag
+}
+
+upload_part_copy_with_range() {
+  if [ $# -ne 6 ]; then
+    log 2 "upload multipart part copy function must have bucket, key, upload ID, file name, part number, range"
+    return 1
+  fi
+  local etag_json
+  log 5 "bucket: $1, key: $2, upload ID: $3, file name: $4, range: $5, copy source range: $6"
+  etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$?
+  if [[ $uploaded -ne 0 ]]; then
+    log 2 "Error uploading part $5: $etag_json"
+    export upload_part_copy_error=$etag_json
+    return 1
+  fi
+  etag=$(echo "$etag_json" | grep -v "InsecureRequestWarning" | jq '.CopyPartResult.ETag')
+  export etag
+}

tests/env.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+
+check_env_vars() {
+  if ! check_universal_vars; then
+    log 2 "error checking universal params"
+    return 1
+  fi
+  if [[ $RUN_VERSITYGW == "true" ]]; then
+    if ! check_versity_vars; then
+      log 2 "error checking versity params"
+      return 1
+    fi
+  fi
+  if [[ $RUN_S3CMD == "true" ]]; then
+    if [[ -z "$S3CMD_CONFIG" ]]; then
+      log 2 "running s3cmd commands requires S3CMD_CONFIG param"
+      return 1
+    fi
+    export S3CMD_CONFIG
+  fi
+  if [[ $RUN_MC == "true" ]]; then
+    if [ -z "$MC_ALIAS" ]; then
+      log 2 "running mc tests requires MC_ALIAS param"
+      return 1
+    fi
+    export MC_ALIAS
+  fi
+  return 0
+}
+
+check_universal_vars() {
+  if [[ $BYPASS_ENV_FILE != "true" ]]; then
+    if [ -z "$VERSITYGW_TEST_ENV" ]; then
+      if [ -r tests/.env ]; then
+        source tests/.env
+      else
+        log 3 "Warning: no .env file found in tests folder"
+      fi
+    else
+      # shellcheck source=./tests/.env.default
+      source "$VERSITYGW_TEST_ENV"
+    fi
+  fi
+  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
+    # shellcheck source=./tests/.secrets
+    source "$SECRETS_FILE"
+  else
+    log 3 "Warning: no secrets file found"
+  fi
+  if [[ -n "$LOG_LEVEL" ]]; then
+    export LOG_LEVEL_INT=$LOG_LEVEL
+  fi
+  if [ -z "$AWS_ACCESS_KEY_ID" ]; then
+    log 2 "No AWS access key set"
+    return 1
+  elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+    log 2 "No AWS secret access key set"
+    return 1
+  elif [ -z "$AWS_REGION" ]; then
+    log 2 "No AWS region set"
+    return 1
+  elif [ -z "$AWS_PROFILE" ]; then
+    log 2 "No AWS profile set"
+    return 1
+  elif [ "$DIRECT" != "true" ] && [ -z "$AWS_ENDPOINT_URL" ]; then
+    log 2 "No AWS endpoint URL set"
+    return 1
+  elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then
+    log 2 "RUN_VERSITYGW must be 'true' or 'false'"
+    return 1
+  elif [ -z "$BUCKET_ONE_NAME" ]; then
+    log 2 "No bucket one name set"
+    return 1
+  elif [ -z "$BUCKET_TWO_NAME" ]; then
+    log 2 "No bucket two name set"
+    return 1
+  elif [ -z "$RECREATE_BUCKETS" ]; then
+    log 2 "No recreate buckets parameter set"
+    return 1
+  elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then
+    log 2 "RECREATE_BUCKETS must be 'true' or 'false'"
+    return 1
+  fi
+  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL RUN_VERSITYGW \
+    BUCKET_ONE_NAME BUCKET_TWO_NAME RECREATE_BUCKETS
+  if [[ -n "$TEST_LOG_FILE" ]]; then
+    export TEST_LOG_FILE
+  fi
+  if [[ -n "$VERSITY_LOG_FILE" ]]; then
+    export VERSITY_LOG_FILE
+  fi
+  if [[ -n "$DIRECT" ]]; then
+    export DIRECT
+  fi
+}
+
+check_versity_vars() {
+  if [ -z "$LOCAL_FOLDER" ]; then
+    log 2 "No local storage folder set"
+    return 1
+  elif [ -z "$VERSITY_EXE" ]; then
+    log 2 "No versity executable location set"
+    return 1
+  elif [ -z "$BACKEND" ]; then
+    log 2 "No backend parameter set (options: 'posix', 's3')"
+    return 1
+  fi
+  export LOCAL_FOLDER VERSITY_EXE BACKEND
+  if [ "$BACKEND" == 's3' ]; then
+    if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then
+      log 2 "missing second AWS access key ID for s3 backend"
+      return 1
+    fi
+    if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then
+      log 2 "missing second AWS secret access key for s3 backend"
+      return 1
+    fi
+    export AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY_TWO
+  fi
+  if [[ -r $GOCOVERDIR ]]; then
+    export GOCOVERDIR=$GOCOVERDIR
+  fi
+  if [[ $RUN_USERS == "true" ]]; then
+    if ! check_user_vars; then
+      log 2 "error setting user vars"
+      return 1
+    fi
+  fi
+}
+
+check_user_vars() {
+  if [[ -z "$IAM_TYPE" ]]; then
+    export IAM_TYPE="folder"
+  fi
+  if [[ "$IAM_TYPE" == "folder" ]]; then
+    if [[ -z "$USERS_FOLDER" ]]; then
+      log 2 "if IAM type is folder (or not set), USERS_FOLDER parameter is required"
+      return 1
+    fi
+    if [ ! -d "$USERS_FOLDER" ]; then
+      if mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1); then
+        log 2 "error creating users folder: $mkdir_error"
+        return 1
+      fi
+    fi
+    IAM_PARAMS="--iam-dir=$USERS_FOLDER"
+    export IAM_PARAMS
+    return 0
+  fi
+  if [[ $IAM_TYPE == "s3" ]]; then
+    if [[ -z "$USERS_BUCKET" ]]; then
+      log 2 "if IAM type is s3, USERS_BUCKET is required"
+      return 1
+    fi
+    IAM_PARAMS="--s3-iam-access $AWS_ACCESS_KEY_ID --s3-iam-secret $AWS_SECRET_ACCESS_KEY \
+      --s3-iam-region us-east-1 --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \
+      --s3-iam-noverify"
+    export IAM_PARAMS
+    return 0
+  fi
+  log 2 "unrecognized IAM_TYPE value: $IAM_TYPE"
+  return 1
+}

tests/iam.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+get_iam_parameters() {
+  if [[ -z "$IAM_TYPE" ]]; then
+    export IAM_TYPE="folder"
+  fi
+  if [[ "$IAM_TYPE" == "folder" ]]; then
+    if [[ -z "$USERS_FOLDER" ]]; then
+      log 2 "if IAM type is folder (or not set), USERS_FOLDER parameter is required"
+      return 1
+    fi
+    if [ ! -d "$USERS_FOLDER" ]; then
+      if mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1); then
+        log 2 "error creating users folder: $mkdir_error"
+        return 1
+      fi
+    fi
+    iam_params="--iam-dir=$USERS_FOLDER"
+    export iam_params
+    return 0
+  fi
+  if [[ $IAM_TYPE == "s3" ]]; then
+    if [[ -z "$USERS_BUCKET" ]]; then
+      log 2 "if IAM type is s3, USERS_BUCKET is required"
+      return 1
+    fi
+    log 4 "$USERS_BUCKET"
+    if ! bucket_exists "s3api" "$USERS_BUCKET"; then
+      log 4 "bucket doesn't exist"
+      if [[ $? == 2 ]]; then
+        log 2 "error checking if users bucket exists"
+        return 1
+      fi
+      if ! create_bucket "s3api" "$USERS_BUCKET"; then
+        log 2 "error creating bucket"
+        return 1
+      fi
+      log 4 "bucket create successful"
+    else
+      log 4 "bucket exists"
+    fi
+    iam_params="--s3-iam-access $AWS_ACCESS_KEY_ID --s3-iam-secret $AWS_SECRET_ACCESS_KEY \
+      --s3-iam-region us-east-1 --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \
+      --s3-iam-noverify"
+    export iam_params
+    return 0
+  fi
+  log 2 "unrecognized IAM_TYPE value: $IAM_TYPE"
+  return 1
+}

tests/integration/group-tests.go

@@ -65,6 +65,8 @@ func TestCreateBucket(s *S3Conf) {
 	CreateBucket_invalid_bucket_name(s)
 	CreateBucket_existing_bucket(s)
 	CreateBucket_owned_by_you(s)
+	CreateBucket_invalid_ownership(s)
+	CreateBucket_ownership_with_acl(s)
 	CreateBucket_as_user(s)
 	CreateBucket_default_acl(s)
 	CreateBucket_non_default_acl(s)
@@ -89,6 +91,24 @@ func TestDeleteBucket(s *S3Conf) {
 	DeleteBucket_success_status_code(s)
 }
 
+func TestPutBucketOwnershipControls(s *S3Conf) {
+	PutBucketOwnershipControls_non_existing_bucket(s)
+	PutBucketOwnershipControls_multiple_rules(s)
+	PutBucketOwnershipControls_invalid_ownership(s)
+	PutBucketOwnershipControls_success(s)
+}
+
+func TestGetBucketOwnershipControls(s *S3Conf) {
+	GetBucketOwnershipControls_non_existing_bucket(s)
+	GetBucketOwnershipControls_default_ownership(s)
+	GetBucketOwnershipControls_success(s)
+}
+
+func TestDeleteBucketOwnershipControls(s *S3Conf) {
+	DeleteBucketOwnershipControls_non_existing_bucket(s)
+	DeleteBucketOwnershipControls_success(s)
+}
+
 func TestPutBucketTagging(s *S3Conf) {
 	PutBucketTagging_non_existing_bucket(s)
 	PutBucketTagging_long_tags(s)
@@ -236,6 +256,7 @@ func TestUploadPartCopy(s *S3Conf) {
 func TestListParts(s *S3Conf) {
 	ListParts_incorrect_uploadId(s)
 	ListParts_incorrect_object_key(s)
+	ListParts_truncated(s)
 	ListParts_success(s)
 }
 
@@ -266,6 +287,7 @@ func TestCompleteMultipartUpload(s *S3Conf) {
 
 func TestPutBucketAcl(s *S3Conf) {
 	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_disabled(s)
 	PutBucketAcl_invalid_acl_canned_and_acp(s)
 	PutBucketAcl_invalid_acl_canned_and_grants(s)
 	PutBucketAcl_invalid_acl_acp_and_grants(s)
@@ -279,12 +301,16 @@ func TestPutBucketAcl(s *S3Conf) {
 
 func TestGetBucketAcl(s *S3Conf) {
 	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_translation_canned_public_read(s)
+	GetBucketAcl_translation_canned_public_read_write(s)
+	GetBucketAcl_translation_canned_private(s)
 	GetBucketAcl_access_denied(s)
 	GetBucketAcl_success(s)
 }
 
 func TestPutBucketPolicy(s *S3Conf) {
 	PutBucketPolicy_non_existing_bucket(s)
+	PutBucketPolicy_empty_statement(s)
 	PutBucketPolicy_invalid_effect(s)
 	PutBucketPolicy_empty_actions_string(s)
 	PutBucketPolicy_empty_actions_array(s)
@@ -393,6 +419,9 @@ func TestFullFlow(s *S3Conf) {
 	TestHeadBucket(s)
 	TestListBuckets(s)
 	TestDeleteBucket(s)
+	TestPutBucketOwnershipControls(s)
+	TestGetBucketOwnershipControls(s)
+	TestDeleteBucketOwnershipControls(s)
 	TestPutBucketTagging(s)
 	TestGetBucketTagging(s)
 	TestDeleteBucketTagging(s)
@@ -503,6 +532,8 @@ func GetIntTests() IntTests {
 		"CreateBucket_invalid_bucket_name":                                    CreateBucket_invalid_bucket_name,
 		"CreateBucket_existing_bucket":                                        CreateBucket_existing_bucket,
 		"CreateBucket_owned_by_you":                                           CreateBucket_owned_by_you,
+		"CreateBucket_invalid_ownership":                                      CreateBucket_invalid_ownership,
+		"CreateBucket_ownership_with_acl":                                     CreateBucket_ownership_with_acl,
 		"CreateBucket_as_user":                                                CreateBucket_as_user,
 		"CreateDeleteBucket_success":                                          CreateDeleteBucket_success,
 		"CreateBucket_default_acl":                                            CreateBucket_default_acl,
@@ -516,6 +547,15 @@ func GetIntTests() IntTests {
 		"DeleteBucket_non_existing_bucket":                                    DeleteBucket_non_existing_bucket,
 		"DeleteBucket_non_empty_bucket":                                       DeleteBucket_non_empty_bucket,
 		"DeleteBucket_success_status_code":                                    DeleteBucket_success_status_code,
+		"PutBucketOwnershipControls_non_existing_bucket":                      PutBucketOwnershipControls_non_existing_bucket,
+		"PutBucketOwnershipControls_multiple_rules":                           PutBucketOwnershipControls_multiple_rules,
+		"PutBucketOwnershipControls_invalid_ownership":                        PutBucketOwnershipControls_invalid_ownership,
+		"PutBucketOwnershipControls_success":                                  PutBucketOwnershipControls_success,
+		"GetBucketOwnershipControls_non_existing_bucket":                      GetBucketOwnershipControls_non_existing_bucket,
+		"GetBucketOwnershipControls_default_ownership":                        GetBucketOwnershipControls_default_ownership,
+		"GetBucketOwnershipControls_success":                                  GetBucketOwnershipControls_success,
+		"DeleteBucketOwnershipControls_non_existing_bucket":                   DeleteBucketOwnershipControls_non_existing_bucket,
+		"DeleteBucketOwnershipControls_success":                               DeleteBucketOwnershipControls_success,
 		"PutBucketTagging_non_existing_bucket":                                PutBucketTagging_non_existing_bucket,
 		"PutBucketTagging_long_tags":                                          PutBucketTagging_long_tags,
 		"PutBucketTagging_success":                                            PutBucketTagging_success,
@@ -605,6 +645,7 @@ func GetIntTests() IntTests {
 		"UploadPartCopy_by_range_success":                                     UploadPartCopy_by_range_success,
 		"ListParts_incorrect_uploadId":                                        ListParts_incorrect_uploadId,
 		"ListParts_incorrect_object_key":                                      ListParts_incorrect_object_key,
+		"ListParts_truncated":                                                 ListParts_truncated,
 		"ListParts_success":                                                   ListParts_success,
 		"ListMultipartUploads_non_existing_bucket":                            ListMultipartUploads_non_existing_bucket,
 		"ListMultipartUploads_empty_result":                                   ListMultipartUploads_empty_result,
@@ -633,9 +674,13 @@ func GetIntTests() IntTests {
 		"PutBucketAcl_success_canned_acl":                                     PutBucketAcl_success_canned_acl,
 		"PutBucketAcl_success_acp":                                            PutBucketAcl_success_acp,
 		"GetBucketAcl_non_existing_bucket":                                    GetBucketAcl_non_existing_bucket,
+		"GetBucketAcl_translation_canned_public_read":                         GetBucketAcl_translation_canned_public_read,
+		"GetBucketAcl_translation_canned_public_read_write":                   GetBucketAcl_translation_canned_public_read_write,
+		"GetBucketAcl_translation_canned_private":                             GetBucketAcl_translation_canned_private,
 		"GetBucketAcl_access_denied":                                          GetBucketAcl_access_denied,
 		"GetBucketAcl_success":                                                GetBucketAcl_success,
 		"PutBucketPolicy_non_existing_bucket":                                 PutBucketPolicy_non_existing_bucket,
+		"PutBucketPolicy_empty_statement":                                     PutBucketPolicy_empty_statement,
 		"PutBucketPolicy_invalid_effect":                                      PutBucketPolicy_invalid_effect,
 		"PutBucketPolicy_empty_actions_string":                                PutBucketPolicy_empty_actions_string,
 		"PutBucketPolicy_empty_actions_array":                                 PutBucketPolicy_empty_actions_array,

tests/integration/s3conf.go

@@ -93,13 +93,8 @@ func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
 	return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "")
 }
 
-func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
-	return aws.Endpoint{
-		PartitionID:       "aws",
-		URL:               c.endpoint,
-		SigningRegion:     c.awsRegion,
-		HostnameImmutable: true,
-	}, nil
+func (c *S3Conf) GetClient() *s3.Client {
+	return s3.NewFromConfig(c.Config())
 }
 
 func (c *S3Conf) Config() aws.Config {
@@ -114,11 +109,6 @@ func (c *S3Conf) Config() aws.Config {
 		config.WithHTTPClient(client),
 	}
 
-	if c.endpoint != "" && c.endpoint != "aws" {
-		opts = append(opts,
-			config.WithEndpointResolverWithOptions(c))
-	}
-
 	if c.checksumDisable {
 		opts = append(opts,
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -135,11 +125,15 @@ func (c *S3Conf) Config() aws.Config {
 		log.Fatalln("error:", err)
 	}
 
+	if c.endpoint != "" && c.endpoint != "aws" {
+		cfg.BaseEndpoint = &c.endpoint
+	}
+
 	return cfg
 }
 
 func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
-	uploader := manager.NewUploader(s3.NewFromConfig(c.Config()))
+	uploader := manager.NewUploader(c.GetClient())
 	uploader.PartSize = c.PartSize
 	uploader.Concurrency = c.Concurrency
 
@@ -154,7 +148,7 @@ func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
 }
 
 func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
-	downloader := manager.NewDownloader(s3.NewFromConfig(c.Config()))
+	downloader := manager.NewDownloader(c.GetClient())
 	downloader.PartSize = c.PartSize
 	downloader.Concurrency = c.Concurrency
 

tests/integration/tests.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
@@ -1719,47 +1720,76 @@ func CreateBucket_owned_by_you(s *S3Conf) error {
 	})
 }
 
-func CreateBucket_default_acl(s *S3Conf) error {
-	testName := "CreateBucket_default_acl"
+func CreateBucket_invalid_ownership(s *S3Conf) error {
+	testName := "CreateBucket_invalid_ownership"
 	runF(testName)
 
-	bucket := getBucketName()
-	client := s3.NewFromConfig(s.Config())
-
-	err := setup(s, bucket)
-	if err != nil {
+	invalidOwnership := types.ObjectOwnership("invalid_ownership")
+	err := setup(s, getBucketName(), withOwnership(invalidOwnership))
+	if err := checkApiErr(err, s3err.APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Invalid x-amz-object-ownership header: %v", invalidOwnership),
+		HTTPStatusCode: http.StatusBadRequest,
+	}); err != nil {
 		failF("%v: %v", testName, err)
 		return fmt.Errorf("%v: %w", testName, err)
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-	out, err := client.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &bucket})
-	cancel()
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
-
-	if *out.Owner.ID != s.awsID {
-		failF("%v: expected bucket owner to be %v, instead got %v", testName, s.awsID, *out.Owner.ID)
-		return fmt.Errorf("%v: expected bucket owner to be %v, instead got %v", testName, s.awsID, *out.Owner.ID)
-	}
-
-	if len(out.Grants) != 0 {
-		failF("%v: expected grants to be empty instead got %v", testName, len(out.Grants))
-		return fmt.Errorf("%v: expected grants to be empty instead got %v", testName, len(out.Grants))
-	}
-
-	err = teardown(s, bucket)
-	if err != nil {
-		failF("%v: %v", err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
-
 	passF(testName)
 	return nil
 }
 
+func CreateBucket_ownership_with_acl(s *S3Conf) error {
+	testName := "CreateBucket_ownership_with_acl"
+
+	runF(testName)
+	client := s3.NewFromConfig(s.Config())
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket:          getPtr(getBucketName()),
+		ObjectOwnership: types.ObjectOwnershipBucketOwnerEnforced,
+		ACL:             types.BucketCannedACLPublicRead,
+	})
+	cancel()
+	if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership)); err != nil {
+		failF("%v: %v", testName, err)
+		return fmt.Errorf("%v: %w", testName, err)
+	}
+
+	passF(testName)
+	return nil
+}
+
+func CreateBucket_default_acl(s *S3Conf) error {
+	testName := "CreateBucket_default_acl"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &bucket})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if *out.Owner.ID != s.awsID {
+			return fmt.Errorf("expected bucket owner to be %v, instead got %v", s.awsID, *out.Owner.ID)
+		}
+		if len(out.Grants) != 1 {
+			return fmt.Errorf("expected grants length to be 1, instead got %v", len(out.Grants))
+		}
+		grt := out.Grants[0]
+		if grt.Permission != types.PermissionFullControl {
+			return fmt.Errorf("expected the grantee to have full-control permission, instead got %v", grt.Permission)
+		}
+		if *grt.Grantee.ID != s.awsID {
+			return fmt.Errorf("expected the grantee id to be %v, instead got %v", s.awsID, *grt.Grantee.ID)
+		}
+
+		return nil
+	})
+}
+
 func CreateBucket_non_default_acl(s *S3Conf) error {
 	testName := "CreateBucket_non_default_acl"
 	runF(testName)
@@ -1775,6 +1805,12 @@ func CreateBucket_non_default_acl(s *S3Conf) error {
 	}
 
 	grants := []types.Grant{
+		{
+			Grantee: &types.Grantee{
+				ID: &s.awsID,
+			},
+			Permission: types.PermissionFullControl,
+		},
 		{
 			Grantee: &types.Grantee{
 				ID: getPtr("grt1"),
@@ -1799,7 +1835,13 @@ func CreateBucket_non_default_acl(s *S3Conf) error {
 	client := s3.NewFromConfig(s.Config())
 
 	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-	_, err = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &bucket, GrantFullControl: getPtr("grt1"), GrantReadACP: getPtr("grt2"), GrantWrite: getPtr("grt3")})
+	_, err = client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket:           &bucket,
+		GrantFullControl: getPtr("grt1"),
+		GrantReadACP:     getPtr("grt2"),
+		GrantWrite:       getPtr("grt3"),
+		ObjectOwnership:  types.ObjectOwnershipBucketOwnerPreferred,
+	})
 	cancel()
 	if err != nil {
 		failF("%v: %v", err)
@@ -2194,6 +2236,220 @@ func DeleteBucket_success_status_code(s *S3Conf) error {
 	return nil
 }
 
+func PutBucketOwnershipControls_non_existing_bucket(s *S3Conf) error {
+	testName := "PutBucketOwnershipControls_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+			Bucket: getPtr(getBucketName()),
+			OwnershipControls: &types.OwnershipControls{
+				Rules: []types.OwnershipControlsRule{
+					{
+						ObjectOwnership: types.ObjectOwnershipBucketOwnerPreferred,
+					},
+				},
+			},
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func PutBucketOwnershipControls_multiple_rules(s *S3Conf) error {
+	testName := "PutBucketOwnershipControls_multiple_rules"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+			Bucket: &bucket,
+			OwnershipControls: &types.OwnershipControls{
+				Rules: []types.OwnershipControlsRule{
+					{
+						ObjectOwnership: types.ObjectOwnershipBucketOwnerPreferred,
+					},
+					{
+						ObjectOwnership: types.ObjectOwnershipObjectWriter,
+					},
+				},
+			},
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func PutBucketOwnershipControls_invalid_ownership(s *S3Conf) error {
+	testName := "PutBucketOwnershipControls_invalid_ownership"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+			Bucket: &bucket,
+			OwnershipControls: &types.OwnershipControls{
+				Rules: []types.OwnershipControlsRule{
+					{
+						ObjectOwnership: types.ObjectOwnership("invalid_ownership"),
+					},
+				},
+			},
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func PutBucketOwnershipControls_success(s *S3Conf) error {
+	testName := "PutBucketOwnershipControls_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+			Bucket: &bucket,
+			OwnershipControls: &types.OwnershipControls{
+				Rules: []types.OwnershipControlsRule{
+					{
+						ObjectOwnership: types.ObjectOwnershipObjectWriter,
+					},
+				},
+			},
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func GetBucketOwnershipControls_non_existing_bucket(s *S3Conf) error {
+	testName := "GetBucketOwnershipControls_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+			Bucket: getPtr(getBucketName()),
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func GetBucketOwnershipControls_default_ownership(s *S3Conf) error {
+	testName := "GetBucketOwnershipControls_default_ownership"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		resp, err := s3client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if len(resp.OwnershipControls.Rules) != 1 {
+			return fmt.Errorf("expected ownership control rules length to be 1, instead got %v", len(resp.OwnershipControls.Rules))
+		}
+		if resp.OwnershipControls.Rules[0].ObjectOwnership != types.ObjectOwnershipBucketOwnerEnforced {
+			return fmt.Errorf("expected the bucket ownership to be %v, instead got %v", types.ObjectOwnershipBucketOwnerEnforced, resp.OwnershipControls.Rules[0].ObjectOwnership)
+		}
+
+		return nil
+	})
+}
+
+func GetBucketOwnershipControls_success(s *S3Conf) error {
+	testName := "GetBucketOwnershipControls_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+			Bucket: &bucket,
+			OwnershipControls: &types.OwnershipControls{
+				Rules: []types.OwnershipControlsRule{
+					{
+						ObjectOwnership: types.ObjectOwnershipObjectWriter,
+					},
+				},
+			},
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		resp, err := s3client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if len(resp.OwnershipControls.Rules) != 1 {
+			return fmt.Errorf("expected ownership control rules length to be 1, instead got %v", len(resp.OwnershipControls.Rules))
+		}
+		if resp.OwnershipControls.Rules[0].ObjectOwnership != types.ObjectOwnershipObjectWriter {
+			return fmt.Errorf("expected the bucket ownership to be %v, instead got %v", types.ObjectOwnershipObjectWriter, resp.OwnershipControls.Rules[0].ObjectOwnership)
+		}
+
+		return nil
+	})
+}
+
+func DeleteBucketOwnershipControls_non_existing_bucket(s *S3Conf) error {
+	testName := "DeleteBucketOwnershipControls_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
+			Bucket: getPtr(getBucketName()),
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func DeleteBucketOwnershipControls_success(s *S3Conf) error {
+	testName := "DeleteBucketOwnershipControls_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func PutBucketTagging_non_existing_bucket(s *S3Conf) error {
 	testName := "PutBucketTagging_non_existing_bucket"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -3034,7 +3290,7 @@ func GetObject_invalid_ranges(s *S3Conf) error {
 		_, err = s3client.GetObject(ctx, &s3.GetObjectInput{
 			Bucket: &bucket,
 			Key:    &obj,
-			Range:  getPtr("bytes=1000000000-999999999999"),
+			Range:  getPtr("bytes=0-0"),
 		})
 		cancel()
 		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil {
@@ -3042,15 +3298,19 @@ func GetObject_invalid_ranges(s *S3Conf) error {
 		}
 
 		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
-		_, err = s3client.GetObject(ctx, &s3.GetObjectInput{
+		resp, err := s3client.GetObject(ctx, &s3.GetObjectInput{
 			Bucket: &bucket,
 			Key:    &obj,
-			Range:  getPtr("bytes=0-0"),
+			Range:  getPtr("bytes=1500-999999999999"),
 		})
 		cancel()
-		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil {
+		if err != nil {
 			return err
 		}
+
+		if *resp.ContentLength != dataLength-1500 {
+			return fmt.Errorf("expected content-length to be %v, instead got %v", dataLength-1500, *resp.ContentLength)
+		}
 		return nil
 	})
 }
@@ -3159,7 +3419,7 @@ func GetObject_by_range_success(s *S3Conf) error {
 			return fmt.Errorf("expected accept range: %v, instead got: %v", rangeString, getString(out.AcceptRanges))
 		}
 		b, err := io.ReadAll(out.Body)
-		if err != nil {
+		if err != nil && !errors.Is(err, io.EOF) {
 			return err
 		}
 
@@ -3183,7 +3443,7 @@ func GetObject_by_range_success(s *S3Conf) error {
 		defer out.Body.Close()
 
 		b, err = io.ReadAll(out.Body)
-		if err != nil {
+		if err != nil && !errors.Is(err, io.EOF) {
 			return err
 		}
 
@@ -5145,7 +5405,7 @@ func UploadPartCopy_greater_range_than_obj_size(s *S3Conf) error {
 			PartNumber:      &partNumber,
 		})
 		cancel()
-		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil {
+		if err := checkApiErr(err, backend.CreateExceedingRangeErr(int64(srcObjSize))); err != nil {
 			return err
 		}
 
@@ -5270,6 +5530,70 @@ func ListParts_incorrect_object_key(s *S3Conf) error {
 	})
 }
 
+func ListParts_truncated(s *S3Conf) error {
+	testName := "ListParts_truncated"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		obj := "my-obj"
+		out, err := createMp(s3client, bucket, obj)
+		if err != nil {
+			return err
+		}
+
+		parts, err := uploadParts(s3client, 5*1024*1024, 5, bucket, obj, *out.UploadId)
+		if err != nil {
+			return err
+		}
+
+		maxParts := int32(3)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		res, err := s3client.ListParts(ctx, &s3.ListPartsInput{
+			Bucket:   &bucket,
+			Key:      &obj,
+			UploadId: out.UploadId,
+			MaxParts: &maxParts,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if !*res.IsTruncated {
+			return fmt.Errorf("expected the result to be truncated")
+		}
+		if *res.MaxParts != maxParts {
+			return fmt.Errorf("expected max-parts to be %v, instead got %v", maxParts, *res.MaxParts)
+		}
+		if *res.NextPartNumberMarker != fmt.Sprint(*parts[2].PartNumber) {
+			return fmt.Errorf("expected next part number marker to be %v, instead got %v", fmt.Sprint(*parts[2].PartNumber), *res.NextPartNumberMarker)
+		}
+		if ok := compareParts(res.Parts, parts[:3]); !ok {
+			return fmt.Errorf("expected the parts data to be %v, instead got %v", parts[:3], res.Parts)
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		res2, err := s3client.ListParts(ctx, &s3.ListPartsInput{
+			Bucket:           &bucket,
+			Key:              &obj,
+			UploadId:         out.UploadId,
+			PartNumberMarker: res.NextPartNumberMarker,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if *res2.PartNumberMarker != *res.NextPartNumberMarker {
+			return fmt.Errorf("expected part number marker to be %v, instead got %v", *res.NextPartNumberMarker, *res2.PartNumberMarker)
+		}
+		if ok := compareParts(parts[3:], res2.Parts); !ok {
+			return fmt.Errorf("expected the parts data to be %v, instead got %v", parts[3:], res2.Parts)
+		}
+
+		return nil
+	})
+}
+
 func ListParts_success(s *S3Conf) error {
 	testName := "ListParts_success"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -5467,7 +5791,7 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error {
 }
 
 func ListMultipartUploads_success(s *S3Conf) error {
-	testName := "ListMultipartUploads_max_uploads"
+	testName := "ListMultipartUploads_success"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
 		obj1, obj2 := "my-obj-1", "my-obj-2"
 		out1, err := createMp(s3client, bucket, obj1)
@@ -5824,6 +6148,23 @@ func PutBucketAcl_non_existing_bucket(s *S3Conf) error {
 	})
 }
 
+func PutBucketAcl_disabled(s *S3Conf) error {
+	testName := "PutBucketAcl_disabled"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{
+			Bucket:    &bucket,
+			ACL:       types.BucketCannedACLPublicRead,
+			GrantRead: &s.awsID,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAclNotSupported)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
 func PutBucketAcl_invalid_acl_canned_and_acp(s *S3Conf) error {
 	testName := "PutBucketAcl_invalid_acl_canned_and_acp"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -5839,7 +6180,7 @@ func PutBucketAcl_invalid_acl_canned_and_acp(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_invalid_acl_canned_and_grants(s *S3Conf) error {
@@ -5869,7 +6210,7 @@ func PutBucketAcl_invalid_acl_canned_and_grants(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_invalid_acl_acp_and_grants(s *S3Conf) error {
@@ -5899,7 +6240,7 @@ func PutBucketAcl_invalid_acl_acp_and_grants(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_invalid_owner(s *S3Conf) error {
@@ -5928,7 +6269,7 @@ func PutBucketAcl_invalid_owner(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_invalid_owner_not_in_body(s *S3Conf) error {
@@ -5954,7 +6295,7 @@ func PutBucketAcl_invalid_owner_not_in_body(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_success_access_denied(s *S3Conf) error {
@@ -5999,7 +6340,7 @@ func PutBucketAcl_success_access_denied(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_success_canned_acl(s *S3Conf) error {
@@ -6031,7 +6372,7 @@ func PutBucketAcl_success_canned_acl(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_success_acp(s *S3Conf) error {
@@ -6072,7 +6413,7 @@ func PutBucketAcl_success_acp(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketAcl_success_grants(s *S3Conf) error {
@@ -6117,7 +6458,7 @@ func PutBucketAcl_success_grants(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func GetBucketAcl_non_existing_bucket(s *S3Conf) error {
@@ -6136,6 +6477,156 @@ func GetBucketAcl_non_existing_bucket(s *S3Conf) error {
 	})
 }
 
+func GetBucketAcl_translation_canned_public_read(s *S3Conf) error {
+	testName := "GetBucketAcl_translation_canned_public_read"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		grants := []types.Grant{
+			{
+				Grantee: &types.Grantee{
+					ID:   &s.awsID,
+					Type: types.TypeCanonicalUser,
+				},
+				Permission: types.PermissionFullControl,
+			},
+			{
+				Grantee: &types.Grantee{
+					ID:   getPtr("all-users"),
+					Type: types.TypeCanonicalUser,
+				},
+				Permission: types.PermissionRead,
+			},
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{
+			Bucket: &bucket,
+			ACL:    types.BucketCannedACLPublicRead,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if ok := compareGrants(out.Grants, grants); !ok {
+			return fmt.Errorf("expected grants to be %v, instead got %v", grants, out.Grants)
+		}
+		if *out.Owner.ID != s.awsID {
+			return fmt.Errorf("expected bucket owner to be %v, instead got %v", s.awsID, *out.Owner.ID)
+		}
+
+		return nil
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
+}
+
+func GetBucketAcl_translation_canned_public_read_write(s *S3Conf) error {
+	testName := "GetBucketAcl_translation_canned_public_read_write"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		grants := []types.Grant{
+			{
+				Grantee: &types.Grantee{
+					ID:   &s.awsID,
+					Type: types.TypeCanonicalUser,
+				},
+				Permission: types.PermissionFullControl,
+			},
+			{
+				Grantee: &types.Grantee{
+					ID:   getPtr("all-users"),
+					Type: types.TypeCanonicalUser,
+				},
+				Permission: types.PermissionRead,
+			},
+			{
+				Grantee: &types.Grantee{
+					ID:   getPtr("all-users"),
+					Type: types.TypeCanonicalUser,
+				},
+				Permission: types.PermissionWrite,
+			},
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{
+			Bucket: &bucket,
+			ACL:    types.BucketCannedACLPublicReadWrite,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if ok := compareGrants(out.Grants, grants); !ok {
+			return fmt.Errorf("expected grants to be %v, instead got %v", grants, out.Grants)
+		}
+		if *out.Owner.ID != s.awsID {
+			return fmt.Errorf("expected bucket owner to be %v, instead got %v", s.awsID, *out.Owner.ID)
+		}
+
+		return nil
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
+}
+
+func GetBucketAcl_translation_canned_private(s *S3Conf) error {
+	testName := "GetBucketAcl_translation_canned_private"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		grants := []types.Grant{
+			{
+				Grantee: &types.Grantee{
+					ID:   &s.awsID,
+					Type: types.TypeCanonicalUser,
+				},
+				Permission: types.PermissionFullControl,
+			},
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{
+			Bucket: &bucket,
+			ACL:    types.BucketCannedACLPrivate,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if ok := compareGrants(out.Grants, grants); !ok {
+			return fmt.Errorf("expected grants to be %v, instead got %v", grants, out.Grants)
+		}
+		if *out.Owner.ID != s.awsID {
+			return fmt.Errorf("expected bucket owner to be %v, instead got %v", s.awsID, *out.Owner.ID)
+		}
+
+		return nil
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
+}
+
 func GetBucketAcl_access_denied(s *S3Conf) error {
 	testName := "GetBucketAcl_access_denied"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -6222,6 +6713,15 @@ func GetBucketAcl_success(s *S3Conf) error {
 			return err
 		}
 
+		grants = append([]types.Grant{
+			{
+				Grantee: &types.Grantee{
+					ID: &s.awsID,
+				},
+				Permission: types.PermissionFullControl,
+			},
+		}, grants...)
+
 		if ok := compareGrants(out.Grants, grants); !ok {
 			return fmt.Errorf("expected grants to be %v, instead got %v", grants, out.Grants)
 		}
@@ -6230,7 +6730,7 @@ func GetBucketAcl_success(s *S3Conf) error {
 		}
 
 		return nil
-	})
+	}, withOwnership(types.ObjectOwnershipBucketOwnerPreferred))
 }
 
 func PutBucketPolicy_non_existing_bucket(s *S3Conf) error {
@@ -6251,6 +6751,28 @@ func PutBucketPolicy_non_existing_bucket(s *S3Conf) error {
 	})
 }
 
+func PutBucketPolicy_empty_statement(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_statement"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := `
+		{
+			"Statement": []
+		}
+		`
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err := checkApiErr(err, getMalformedPolicyError("Could not parse the policy: Statement is empty!")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
 func PutBucketPolicy_invalid_effect(s *S3Conf) error {
 	testName := "PutBucketPolicy_invalid_effect"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -6263,7 +6785,7 @@ func PutBucketPolicy_invalid_effect(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("invalid effect: invalid_effect")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid effect: invalid_effect")); err != nil {
 			return err
 		}
 		return nil
@@ -6282,7 +6804,7 @@ func PutBucketPolicy_empty_actions_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6301,7 +6823,7 @@ func PutBucketPolicy_empty_actions_array(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6320,7 +6842,7 @@ func PutBucketPolicy_invalid_action(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("invalid action: ListObjects")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6339,7 +6861,7 @@ func PutBucketPolicy_unsupported_action(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("unsupported action: s3:PutLifecycleConfiguration")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6358,7 +6880,7 @@ func PutBucketPolicy_incorrect_action_wildcard_usage(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("invalid wildcard usage: s3:hello prefix is not in the supported actions list")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6377,7 +6899,7 @@ func PutBucketPolicy_empty_principals_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6396,7 +6918,7 @@ func PutBucketPolicy_empty_principals_array(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6415,7 +6937,7 @@ func PutBucketPolicy_principals_aws_struct_empty_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6434,7 +6956,7 @@ func PutBucketPolicy_principals_aws_struct_empty_string_slice(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6453,7 +6975,7 @@ func PutBucketPolicy_principals_incorrect_wildcard_usage(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals should either contain * or user access keys")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6472,14 +6994,8 @@ func PutBucketPolicy_non_existing_principals(s *S3Conf) error {
 		})
 		cancel()
 
-		apiErr1 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"}))
-		apiErr2 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_2", "a_rarely_existing_user_account_1"}))
-
-		err1 := checkApiErr(err, apiErr1)
-		err2 := checkApiErr(err, apiErr2)
-
-		if err1 != nil && err2 != nil {
-			return err1
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
+			return err
 		}
 
 		return nil
@@ -6498,7 +7014,7 @@ func PutBucketPolicy_empty_resources_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6517,7 +7033,7 @@ func PutBucketPolicy_empty_resources_array(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6537,7 +7053,7 @@ func PutBucketPolicy_invalid_resource_prefix(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6557,7 +7073,7 @@ func PutBucketPolicy_invalid_resource_with_starting_slash(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6576,10 +7092,10 @@ func PutBucketPolicy_duplicate_resource(s *S3Conf) error {
 			Policy: &doc,
 		})
 		cancel()
-
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("duplicate resource: %v", resource[1:len(resource)-1]))); err != nil {
+		if err != nil {
 			return err
 		}
+
 		return nil
 	})
 }
@@ -6597,7 +7113,7 @@ func PutBucketPolicy_incorrect_bucket_name(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("incorrect bucket name in prefix-%v", bucket))); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6617,7 +7133,7 @@ func PutBucketPolicy_object_action_on_bucket_resource(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("unsupported object action 's3:PutObjectTagging' on the specified resources")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Action does not apply to any resource(s) in statement")); err != nil {
 			return err
 		}
 		return nil
@@ -6637,7 +7153,7 @@ func PutBucketPolicy_bucket_action_on_object_resource(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("unsupported bucket action 's3:DeleteBucket' on the specified resources")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Action does not apply to any resource(s) in statement")); err != nil {
 			return err
 		}
 		return nil

tests/integration/utils.go

@@ -66,6 +66,7 @@ func setup(s *S3Conf, bucket string, opts ...setupOpt) error {
 	_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
 		Bucket:                     &bucket,
 		ObjectLockEnabledForBucket: &cfg.LockEnabled,
+		ObjectOwnership:            cfg.Ownership,
 	})
 	cancel()
 	return err
@@ -121,6 +122,7 @@ func teardown(s *S3Conf, bucket string) error {
 
 type setupCfg struct {
 	LockEnabled bool
+	Ownership   types.ObjectOwnership
 }
 
 type setupOpt func(*setupCfg)
@@ -128,6 +130,9 @@ type setupOpt func(*setupCfg)
 func withLock() setupOpt {
 	return func(s *setupCfg) { s.LockEnabled = true }
 }
+func withOwnership(o types.ObjectOwnership) setupOpt {
+	return func(s *setupCfg) { s.Ownership = o }
+}
 
 func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, opts ...setupOpt) error {
 	runF(testName)

tests/logger.sh

@@ -2,12 +2,15 @@
 
 # levels:  1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace
 
+export LOG_LEVEL_INT=4
+
 log() {
   if [[ $# -ne 2 ]]; then
     echo "log function requires level, message"
     return 1
   fi
-  if [[ $1 -gt $LOG_LEVEL ]]; then
+  # shellcheck disable=SC2153
+  if [[ $1 -gt $LOG_LEVEL_INT ]]; then
     return 0
   fi
   log_level=""
@@ -18,9 +21,51 @@ log() {
     4) log_level="INFO";;
     5) log_level="DEBUG";;
     6) log_level="TRACE";;
+    *) echo "invalid log level $1"; return 1
   esac
-  echo "$log_level $2"
-  if [[ -n "$TEST_LOG_FILE" ]]; then
-    echo "$2" >> "$TEST_LOG_FILE"
+  if [[ "$2" == *"secret"* ]]; then
+    log_mask "$log_level" "$2"
+    return 0
   fi
-}
+  log_message "$log_level" "$2"
+}
+
+log_mask() {
+  if [[ $# -ne 2 ]]; then
+    echo "mask and log requires level, string"
+    return 1
+  fi
+  local masked_args=()  # Initialize an array to hold the masked arguments
+
+  IFS=' ' read -r -a array <<< "$2"
+
+  mask_next=false
+  for arg in "${array[@]}"; do
+    if [[ $mask_next == true ]]; then
+      masked_args+=("********")
+      mask_next=false
+    elif [[ "$arg" == --secret_key=* ]]; then
+      masked_args+=("--secret_key=********")
+    elif [[ "$arg" == --secret=* ]]; then
+      masked_args+=("--secret=********")
+    else
+      if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]] || [[ "$arg" == "--s3-iam-secret" ]]; then
+        mask_next=true
+      fi
+      masked_args+=("$arg")
+    fi
+  done
+  log_message "$log_level" "${masked_args[*]}"
+}
+
+log_message() {
+  if [[ $# -ne 2 ]]; then
+    echo "log message requires level, message"
+    return 1
+  fi
+  now="$(date "+%Y-%m-%d %H:%M:%S")"
+  echo "$now $1 $2"
+  if [[ -n "$TEST_LOG_FILE" ]]; then
+    echo "$now $1 $2" >> "$TEST_LOG_FILE"
+  fi
+}

tests/remove_static.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+source ./tests/setup.sh
+source ./tests/util.sh
+
+delete_bucket_if_exists() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "delete_bucket_if_exists command missing command type, name"
+    return 1
+  fi
+  bucket_exists "$1" "$2" || local exists_result=$?
+  if [[ $exists_result -eq 2 ]]; then
+    log 2 "error checking if bucket exists"
+    return 1
+  fi
+  if [[ $exists_result -eq 1 ]]; then
+    log 5 "bucket '$2' doesn't exist, skipping"
+    return 0
+  fi
+  if ! delete_bucket_recursive "$1" "$2"; then
+    log 2 "error deleting bucket"
+    return 1
+  fi
+  log 5 "bucket '$2' successfully deleted"
+  return 0
+}
+
+if ! setup; then
+  log 2 "error starting versity to set up static buckets"
+  exit 1
+fi
+if ! delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then
+  log 2 "error deleting static bucket one"
+elif ! delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then
+  log 2 "error deleting static bucket two"
+fi
+if ! teardown; then
+  log 2 "error stopping versity"
+fi

tests/run.sh

@@ -5,9 +5,12 @@ show_help() {
     echo "Usage: $0 [option...]"
     echo "   -h, --help          Display this help message and exit"
     echo "   -s, --static        Don't remove buckets between tests"
-    echo "   aws                 Run tests with aws cli"
+    echo "   aws                 Run tests with aws (s3api) cli"
+    echo "   s3api               Run tests with s3api cli"
+    echo "   s3                  Run tests with s3 cli"
     echo "   s3cmd               Run tests with s3cmd utility"
     echo "   mc                  Run tests with mc utility"
+    echo "   aws-user            Run user tests with aws cli"
 }
 
 handle_param() {
@@ -16,10 +19,7 @@ handle_param() {
           show_help
           exit 0
           ;;
-      -s|--static)
-          export RECREATE_BUCKETS=false
-          ;;
-      s3|s3api|aws|s3cmd|mc|user)
+      s3|s3api|aws|s3cmd|mc|aws-user)
           set_command_type "$1"
           ;;
       *) # Handle unrecognized options or positional arguments
@@ -38,31 +38,16 @@ set_command_type() {
   export command_type
 }
 
-if [[ -z $RECREATE_BUCKETS ]]; then
-  export RECREATE_BUCKETS=true
-elif [[ $RECREATE_BUCKETS != true ]] && [[ $RECREATE_BUCKETS != false ]]; then
-  echo "Invalid RECREATE_BUCKETS value: $RECREATE_BUCKETS"
-  exit 1
-else
-  export RECREATE_BUCKETS=$RECREATE_BUCKETS
-fi
 while [[ "$#" -gt 0 ]]; do
   handle_param "$1"
   shift # past argument or value
 done
 
-if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
-  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set, or BYPASS_ENV_FILE must be set to true"
   exit 1
 fi
 
-if [[ $RECREATE_BUCKETS == false ]]; then
-  ./tests/setup_static.sh || exit_code=$?
-  if [[ exit_code -ne 0 ]]; then
-    exit 1
-  fi
-fi
-
 case $command_type in
   s3api|aws)
     echo "Running aws tests ..."
@@ -86,6 +71,10 @@ case $command_type in
     echo "Running mc tests ..."
     "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
     ;;
+  aws-user)
+    echo "Running aws user tests ..."
+    "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
 esac
 
+# shellcheck disable=SC2086
 exit $exit_code

tests/run_all.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
-if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
-  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set, or BYPASS_ENV_FILE must be set to true"
   exit 1
 fi
 
-# shellcheck source=./.env.default
-source "$VERSITYGW_TEST_ENV"
-export RECREATE_BUCKETS
-
 if ! ./tests/run.sh aws; then
   exit 1
 fi
@@ -21,7 +17,4 @@ fi
 if ! ./tests/run.sh mc; then
   exit 1
 fi
-if ! ./tests/run.sh user; then
-  exit 1
-fi
 exit 0

tests/setup.sh

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
 
+source ./tests/env.sh
 source ./tests/setup_mc.sh
 source ./tests/versity.sh
 
 # bats setup function
 setup() {
-  start_versity || start_result=$?
-  if [[ $start_result -ne 0 ]]; then
-    echo "error starting versity executable"
+  if ! check_env_vars; then
+    log 2 "error checking env values"
     return 1
   fi
-
-  check_params || check_result=$?
-  if [[ $check_result -ne 0 ]]; then
-    echo "parameter check failed"
-    return 1
+  if [ "$RUN_VERSITYGW" == "true" ]; then
+    if ! run_versity_app; then
+      log 2 "error starting versity apps"
+      return 1
+    fi
   fi
 
   log 4 "Running test $BATS_TEST_NAME"
@@ -32,9 +32,8 @@ setup() {
   fi
 
   if [[ $RUN_MC == true ]]; then
-    check_add_mc_alias || check_result=$?
-    if [[ $check_result -ne 0 ]]; then
-      echo "mc alias check/add failed"
+    if ! check_add_mc_alias; then
+      log 2 "mc alias check/add failed"
       return 1
     fi
   fi
@@ -44,33 +43,6 @@ setup() {
     BUCKET_TWO_NAME
 }
 
-# make sure required environment variables for tests are defined properly
-# return 0 for yes, 1 for no
-check_params() {
-  if [ -z "$BUCKET_ONE_NAME" ]; then
-    echo "No bucket one name set"
-    return 1
-  elif [ -z "$BUCKET_TWO_NAME" ]; then
-    echo "No bucket two name set"
-    return 1
-  elif [ -z "$RECREATE_BUCKETS" ]; then
-    echo "No recreate buckets parameter set"
-    return 1
-  elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then
-    echo "RECREATE_BUCKETS must be 'true' or 'false'"
-    return 1
-  fi
-  if [[ -z "$LOG_LEVEL" ]]; then
-    export LOG_LEVEL=2
-  else
-    export LOG_LEVEL
-  fi
-  if [[ -n "$TEST_LOG_FILE" ]]; then
-    export TEST_LOG_FILE
-  fi
-  return 0
-}
-
 # fail a test
 # param:  error message
 fail() {

tests/setup_mc.sh

@@ -8,8 +8,7 @@ check_for_alias() {
     return 2
   fi
   while IFS= read -r line; do
-    error=$(echo "$line" | grep -w "$MC_ALIAS ")
-    if [[ $? -eq 0 ]]; then
+    if echo "$line" | grep -w "$MC_ALIAS "; then
       return 0
     fi
   done <<< "$aliases"

tests/setup_static.sh

@@ -1,7 +1,40 @@
 #!/bin/bash
 
 source ./tests/setup.sh
-setup
-aws --no-verify-ssl s3 mb s3://"$BUCKET_ONE_NAME"
-aws --no-verify-ssl s3 mb s3://"$BUCKET_TWO_NAME"
-teardown
+source ./tests/util.sh
+source ./tests/commands/create_bucket.sh
+
+create_bucket_if_not_exists() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "create_bucket_if_not_exists command missing command type, name"
+    return 1
+  fi
+  bucket_exists "$1" "$2" || local exists_result=$?
+  if [[ $exists_result -eq 2 ]]; then
+    log 2 "error checking if bucket exists"
+    return 1
+  fi
+  if [[ $exists_result -eq 0 ]]; then
+    log 5 "bucket '$2' already exists, skipping"
+    return 0
+  fi
+  if ! create_bucket_object_lock_enabled "$2"; then
+    log 2 "error creating bucket"
+    return 1
+  fi
+  log 5 "bucket '$2' successfully created"
+  return 0
+}
+
+if ! setup; then
+  log 2 "error starting versity to set up static buckets"
+  exit 1
+fi
+if ! create_bucket_if_not_exists "s3api" "$BUCKET_ONE_NAME"; then
+  log 2 "error creating static bucket one"
+elif ! create_bucket_if_not_exists "s3api" "$BUCKET_TWO_NAME"; then
+  log 2 "error creating static bucket two"
+fi
+if ! teardown; then
+  log 2 "error stopping versity"
+fi

tests/test_common.sh

@@ -47,14 +47,11 @@ test_common_create_delete_bucket() {
     fail "create/delete bucket test requires command type"
   fi
 
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Failed to create bucket"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "failed to create bucket"
 
-  bucket_exists "$1" "$BUCKET_ONE_NAME" || local exists_three=$?
-  [[ $exists_three -eq 0 ]] || fail "Failed bucket existence check"
+  bucket_exists "$1" "$BUCKET_ONE_NAME" || fail "failed bucket existence check"
 
-  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" || local delete_result_two=$?
-  [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" || fail "failed to delete bucket"
 }
 
 test_common_copy_object() {
@@ -62,32 +59,25 @@ test_common_copy_object() {
     fail "copy object test requires command type"
   fi
   local object_name="test-object"
-  create_test_files "$object_name" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+  create_test_files "$object_name" || fail "error creating test file"
   echo "test data" > "$test_file_folder/$object_name"
 
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket one"
-  setup_bucket "$1" "$BUCKET_TWO_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket two"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket one"
+  setup_bucket "$1" "$BUCKET_TWO_NAME" || fail "error setting up bucket two"
 
   if [[ $1 == 's3' ]]; then
-    copy_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || local put_result=$?
+    copy_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to copy object to bucket one"
   else
-    put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || local put_result=$?
+    put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to put object to bucket one"
   fi
-  [[ $put_result -eq 0 ]] || fail "Failed to add object to bucket"
   if [[ $1 == 's3' ]]; then
-    copy_object "$1" "s3://$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name" || local copy_result_one=$?
+    copy_object "$1" "s3://$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name" || fail "object not copied to bucket two"
   else
-    copy_object "$1" "$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name" || local copy_result_one=$?
+    copy_object "$1" "$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name" || fail "object not copied to bucket two"
   fi
-  [[ $copy_result_one -eq 0 ]] || fail "Object not added to bucket"
-  get_object "$1" "$BUCKET_TWO_NAME" "$object_name" "$test_file_folder/$object_name-copy" || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "failed to retrieve object"
+  get_object "$1" "$BUCKET_TWO_NAME" "$object_name" "$test_file_folder/$object_name-copy" || fail "failed to retrieve object"
 
-  compare_files "$test_file_folder/$object_name" "$test_file_folder/$object_name-copy" || local compare_result=$?
-  [[ $compare_result -eq 0 ]] || fail "files not the same"
+  compare_files "$test_file_folder/$object_name" "$test_file_folder/$object_name-copy" || fail "files not the same"
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
   delete_bucket_or_contents "$1" "$BUCKET_TWO_NAME"
@@ -144,28 +134,24 @@ test_common_put_get_object() {
   fi
 
   local object_name="test-object"
-  create_test_files "$object_name" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+
+  create_test_files "$object_name" || fail "error creating test file"
   echo "test data" > "$test_file_folder"/"$object_name"
 
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
 
-  put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || local copy_result=$?
-  [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket"
-  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || local exists_result_one=$?
-  [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
+  if [[ $1 == 's3' ]]; then
+    copy_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to add object to bucket"
+  else
+    put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to add object to bucket"
+  fi
+  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || fail "object not added to bucket"
 
-  get_object "$1" "$BUCKET_ONE_NAME" "$object_name" "$test_file_folder/${object_name}_copy" || local delete_result=$?
-  [[ $delete_result -eq 0 ]] || fail "Failed to delete object"
-  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || local exists_result_two=$?
-  [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
-
-  compare_files "$test_file_folder"/"$object_name" "$test_file_folder/${object_name}_copy" || compare_result=$?
-  [[ $compare_result -ne 0 ]] || fail "objects are different"
+  get_object "$1" "$BUCKET_ONE_NAME" "$object_name" "$test_file_folder/${object_name}_copy" || fail "failed to get object"
+  compare_files "$test_file_folder"/"$object_name" "$test_file_folder/${object_name}_copy" || fail "objects are different"
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
-  delete_test_files "$test_file_folder/$object_name" "$test_file_folder/${object_name}_copy"
+  delete_test_files "$object_name" "${object_name}_copy"
 }
 
 test_common_get_set_versioning() {
@@ -274,18 +260,14 @@ test_common_set_get_delete_bucket_tags() {
   local key="test_key"
   local value="test_value"
 
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
-  get_bucket_tagging "$1" "$BUCKET_ONE_NAME" || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting bucket tags first time"
+  get_bucket_tagging "$1" "$BUCKET_ONE_NAME" || fail "Error getting bucket tags first time"
 
-  check_bucket_tags_empty "$1" "$BUCKET_ONE_NAME" || local check_result=$?
-  [[ $check_result -eq 0 ]] || fail "error checking if bucket tags are empty"
+  check_bucket_tags_empty "$1" "$BUCKET_ONE_NAME" || fail "error checking if bucket tags are empty"
 
   put_bucket_tag "$1" "$BUCKET_ONE_NAME" $key $value
-  get_bucket_tagging "$1" "$BUCKET_ONE_NAME" || local get_result_two=$?
-  [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags second time"
+  get_bucket_tagging "$1" "$BUCKET_ONE_NAME" || fail "Error getting bucket tags second time"
 
   local tag_set_key
   local tag_set_value
@@ -302,11 +284,9 @@ test_common_set_get_delete_bucket_tags() {
   fi
   delete_bucket_tags "$1" "$BUCKET_ONE_NAME"
 
-  get_bucket_tagging "$1" "$BUCKET_ONE_NAME" || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting bucket tags third time"
+  get_bucket_tagging "$1" "$BUCKET_ONE_NAME" || fail "Error getting bucket tags third time"
 
-  check_bucket_tags_empty "$1" "$BUCKET_ONE_NAME" || local check_result=$?
-  [[ $check_result -eq 0 ]] || fail "error checking if bucket tags are empty"
+  check_bucket_tags_empty "$1" "$BUCKET_ONE_NAME" || fail "error checking if bucket tags are empty"
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
 }
 
@@ -362,10 +342,9 @@ test_common_presigned_url_utf8_chars() {
 
   local bucket_file="my-$%^&*;"
   local bucket_file_copy="bucket-file-copy"
-  bucket_file_data="test file\n"
 
   create_test_files "$bucket_file" || local created=$?
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/"$bucket_file"
+  dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
   setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
@@ -453,27 +432,23 @@ test_common_get_bucket_location() {
 
 test_common_put_bucket_acl() {
   [[ $# -eq 1 ]] || fail "test common put bucket acl missing command type"
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating bucket"
+  setup_bucket  "$1" "$BUCKET_ONE_NAME" || fail "error creating bucket"
+  put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls"
 
-  if ! user_exists "ABCDEFG"; then
-    create_user "ABCDEFG" "HIJKLMN" user || create_result=$?
-    [[ $create_result -eq 0 ]] || fail "Error creating user"
-  fi
+  setup_user "ABCDEFG" "HIJKLMN" "user" || fail "error creating user"
 
-  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || fail "error retrieving acl"
 
   log 5 "Initial ACLs: $acl"
-  id=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq '.Owner.ID')
+  id=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq '.Owner.ID' 2>&1) || fail "error getting ID: $id"
   if [[ $id != '"'"$AWS_ACCESS_KEY_ID"'"' ]]; then
-    # in some cases, ID is canonical user ID rather than AWS_ACCESS_KEY_ID
-    canonical_id=$(aws --no-verify-ssl s3api list-buckets --query 'Owner.ID') || local list_result=$?
-    [[ $list_result -eq 0 ]] || fail "error getting canonical ID: $canonical_id"
+    # for direct, ID is canonical user ID rather than AWS_ACCESS_KEY_ID
+    canonical_id=$(aws --no-verify-ssl s3api list-buckets --query 'Owner.ID' 2>&1) || fail "error getting caononical ID: $canonical_id"
     [[ $id == "$canonical_id" ]] || fail "acl ID doesn't match AWS key or canonical ID"
   fi
 
   acl_file="test-acl"
+  create_test_files "$acl_file"
 
 cat <<EOF > "$test_file_folder"/"$acl_file"
   {
@@ -492,15 +467,18 @@ cat <<EOF > "$test_file_folder"/"$acl_file"
   }
 EOF
 
-  put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || local put_result=$?
-  [[ $put_result -eq 0 ]] || fail "Error putting acl"
+  log 6 "before 1st put acl"
+  if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || fail "error putting first acl"
+  else
+    put_bucket_acl "$1" "$BUCKET_ONE_NAME" "ABCDEFG" || fail "error putting first acl"
+  fi
 
-  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || fail "error retrieving second ACL"
 
   log 5 "Acls after 1st put: $acl"
-  public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants[0]')
-  permission=$(echo "$public_grants" | jq -r '.Permission')
+  public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants[1]' 2>&1) || fail "error getting public grants: $public_grants"
+  permission=$(echo "$public_grants" | jq -r '.Permission' 2>&1) || fail "error getting permission: $permission"
   [[ $permission == "READ" ]] || fail "incorrect permission ($permission)"
 
 cat <<EOF > "$test_file_folder"/"$acl_file"
@@ -520,17 +498,15 @@ cat <<EOF > "$test_file_folder"/"$acl_file"
   }
 EOF
 
-  put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || local put_result=$?
-  [[ $put_result -eq 0 ]] || fail "Error putting acl"
+  put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || fail "error putting second acl"
 
-  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || fail "error retrieving second ACL"
 
   log 5 "Acls after 2nd put: $acl"
-  public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants')
-  public_grant_length=$(echo "$public_grants" | jq 'length')
-  [[ $public_grant_length -eq 1 ]] || fail "incorrect grant length for private ACL ($public_grant_length)"
-  permission=$(echo "$public_grants" | jq -r '.[0].Permission')
+  public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants' 2>&1) || fail "error retrieving public grants: $public_grants"
+  public_grant_length=$(echo "$public_grants" | jq -r 'length' 2>&1) || fail "Error retrieving public grant length: $public_grant_length"
+  [[ $public_grant_length -eq 2 ]] || fail "incorrect grant length for private ACL ($public_grant_length)"
+  permission=$(echo "$public_grants" | jq -r '.[0].Permission' 2>&1) || fail "Error retrieving permission: $permission"
   [[ $permission == "FULL_CONTROL" ]] || fail "incorrect permission ($permission)"
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
@@ -541,47 +517,55 @@ test_common_get_put_delete_bucket_policy() {
 
   policy_file="policy_file"
 
-  create_test_files "$policy_file" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating policy file"
+  create_test_files "$policy_file" || fail "error creating policy file"
 
   effect="Allow"
-  principal="*"
+  #principal="*"
+  if [[ $DIRECT == "true" ]]; then
+    principal="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/s3user\"}"
+  else
+    principal="\"*\""
+  fi
   action="s3:GetObject"
   resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"
 
   cat <<EOF > "$test_file_folder"/$policy_file
+{
+  "Version": "2012-10-17",
+  "Statement": [
     {
-      "Version": "2012-10-17",
-      "Statement": [
-        {
-           "Effect": "$effect",
-           "Principal": "$principal",
-           "Action": "$action",
-           "Resource": "$resource"
-        }
-      ]
+       "Effect": "$effect",
+       "Principal": $principal,
+       "Action": "$action",
+       "Resource": "$resource"
     }
+  ]
+}
 EOF
+  log 5 "POLICY: $(cat "$test_file_folder/$policy_file")"
 
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
 
-  check_for_empty_policy "$1" "$BUCKET_ONE_NAME" || check_result=$?
-  [[ $get_result -eq 0 ]] || fail "policy not empty"
+  check_for_empty_policy "$1" "$BUCKET_ONE_NAME" || fail "policy not empty"
 
-  put_bucket_policy "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$policy_file" || put_result=$?
-  [[ $put_result -eq 0 ]] || fail "error putting bucket"
+  put_bucket_policy "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$policy_file" || fail "error putting bucket policy"
 
-  get_bucket_policy "$1" "$BUCKET_ONE_NAME" || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "error getting bucket policy after setting"
+  get_bucket_policy "$1" "$BUCKET_ONE_NAME" || fail "error getting bucket policy after setting"
 
-  returned_effect=$(echo "$bucket_policy" | jq -r '.Statement[0].Effect')
+  # shellcheck disable=SC2154
+  log 5 "POLICY:  $bucket_policy"
+  statement=$(echo "$bucket_policy" | jq -r '.Statement[0]' 2>&1) || fail "error getting statement value: $statement"
+  returned_effect=$(echo "$statement" | jq -r '.Effect' 2>&1) || fail "error getting effect: $returned_effect"
   [[ $effect == "$returned_effect" ]] || fail "effect mismatch ($effect, $returned_effect)"
-  returned_principal=$(echo "$bucket_policy" | jq -r '.Statement[0].Principal')
-  [[ $principal == "$returned_principal" ]] || fail "principal mismatch ($principal, $returned_principal)"
-  returned_action=$(echo "$bucket_policy" | jq -r '.Statement[0].Action')
+  returned_principal=$(echo "$statement" | jq -r '.Principal')
+  if [[ -n $DIRECT ]] && arn=$(echo "$returned_principal" | jq -r '.AWS' 2>&1); then
+    [[ $arn == "arn:aws:iam::$DIRECT_AWS_USER_ID:user/s3user" ]] || fail "arn mismatch"
+  else
+    [[ $principal == "\"$returned_principal\"" ]] || fail "principal mismatch ($principal, $returned_principal)"
+  fi
+  returned_action=$(echo "$statement" | jq -r '.Action')
   [[ $action == "$returned_action" ]] || fail "action mismatch ($action, $returned_action)"
-  returned_resource=$(echo "$bucket_policy" | jq -r '.Statement[0].Resource')
+  returned_resource=$(echo "$statement" | jq -r '.Resource')
   [[ $resource == "$returned_resource" ]] || fail "resource mismatch ($resource, $returned_resource)"
 
   delete_bucket_policy "$1" "$BUCKET_ONE_NAME" || delete_result=$?

tests/test_mc.sh

@@ -36,6 +36,32 @@ export RUN_MC=true
   test_common_set_get_delete_bucket_tags "mc"
 }
 
+# delete-object - put-object tests
+
+# delete-object-tagging
+@test "test_delete_object_tagging" {
+  test_common_delete_object_tagging "mc"
+}
+
+# delete-objects - test setup/teardown
+
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "mc"
+}
+
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-bucket-tagging
+@test "test_set_get_object_tags_mc" {
+  test_common_set_get_object_tags "mc"
+}
+
+# get-object
+@test "test_put_get_object" {
+  test_common_put_get_object "mc"
+}
+
 @test "test_put_object-with-data-mc" {
   test_common_put_object_with_data "mc"
 }
@@ -52,9 +78,6 @@ export RUN_MC=true
   test_common_list_objects "mc"
 }
 
-@test "test_set_get_object_tags_mc" {
-  test_common_set_get_object_tags "mc"
-}
 
 @test "test_presigned_url_utf8_chars_mc" {
   test_common_presigned_url_utf8_chars "mc"
@@ -94,10 +117,4 @@ export RUN_MC=true
   delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
 }
 
-@test "test_delete_object_tagging" {
-  test_common_delete_object_tagging "mc"
-}
 
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "mc"
-}

tests/test_s3.sh

@@ -19,6 +19,15 @@ source ./tests/test_common.sh
 
 # delete-bucket - test_create_delete_bucket
 
+# delete-object - test_put_object
+
+# delete-objects - tested with recursive bucket delete
+
+# get-object
+@test "test_copy_get_object" {
+  test_common_put_get_object "s3"
+}
+
 @test "test_put_object" {
   test_common_put_object_no_data "s3"
 }
@@ -30,7 +39,3 @@ source ./tests/test_common.sh
 @test "test_list_objects_file_count" {
   test_common_list_objects_file_count "s3"
 }
-
-@test "test_put_get_object" {
-  test_common_put_get_object "s3"
-}

tests/test_s3cmd.sh

@@ -10,6 +10,7 @@ source ./tests/commands/get_bucket_policy.sh
 source ./tests/commands/put_bucket_policy.sh
 
 export RUN_S3CMD=true
+export RUN_USERS=true
 
 # complete-multipart-upload
 @test "test_complete_multipart_upload" {
@@ -17,20 +18,28 @@ export RUN_S3CMD=true
 }
 
 # copy-object
-@test "test_copy_object_with_data" {
-  test_common_put_object_with_data "s3cmd"
-}
-
-# copy-object
-@test "test_copy_object_no_data" {
-  test_common_put_object_no_data "s3cmd"
-}
+#@test "test_copy_object" {
+#  test_common_copy_object "s3cmd"
+#}
 
 # create-bucket
 @test "test_create_delete_bucket" {
   test_common_create_delete_bucket "s3cmd"
 }
 
+@test "test_create_bucket_invalid_name_s3cmd" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "s3cmd" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"just the bucket name"* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}
+
 # delete-bucket - test_create_delete_bucket
 
 # delete-bucket-policy
@@ -38,6 +47,32 @@ export RUN_S3CMD=true
   test_common_get_put_delete_bucket_policy "s3cmd"
 }
 
+# delete-object - test_put_object
+
+# delete-objects - tested with cleanup before or after tests
+
+# get-bucket-acl - test_put_bucket_acl
+
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "s3cmd"
+}
+
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-object
+@test "test_put_get_object" {
+  test_common_put_get_object "s3cmd"
+}
+
+@test "test_put_object_with_data" {
+  test_common_put_object_with_data "s3cmd"
+}
+
+@test "test_put_object_no_data" {
+  test_common_put_object_no_data "s3cmd"
+}
+
 #@test "test_put_bucket_acl" {
 #  test_common_put_bucket_acl "s3cmd"
 #}
@@ -59,19 +94,6 @@ export RUN_S3CMD=true
   test_common_list_objects_file_count "s3cmd"
 }
 
-@test "test_create_bucket_invalid_name_s3cmd" {
-  if [[ $RECREATE_BUCKETS != "true" ]]; then
-    return
-  fi
-
-  create_bucket_invalid_name "s3cmd" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
-
-  [[ "$bucket_create_error" == *"just the bucket name"* ]] || fail "unexpected error:  $bucket_create_error"
-
-  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
-}
-
 @test "test_get_bucket_info_s3cmd" {
   setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
   [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
@@ -89,6 +111,3 @@ export RUN_S3CMD=true
   delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
 }
 
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "s3cmd"
-}

tests/test_user_aws.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bats
 
 source ./tests/test_user_common.sh
+source ./tests/util_users.sh
+
+export RUN_USERS=true
 
 @test "test_admin_user_aws" {
   test_admin_user "aws"
@@ -10,6 +13,12 @@ source ./tests/test_user_common.sh
   test_create_user_already_exists "aws"
 }
 
+@test "test_delete_user_no_access_key" {
+  if delete_user ""; then
+    fail "delete user with empty access key succeeded"
+  fi
+}
+
 @test "test_user_user_aws" {
   test_user_user "aws"
 }

tests/test_user_common.sh

@@ -15,31 +15,21 @@ test_admin_user() {
   admin_password="123456"
   user_password="789012"
 
-  user_exists "$admin_username" || local admin_exists_result=$?
-  if [[ $admin_exists_result -eq 0 ]]; then
-    delete_user "$admin_username" || local delete_admin_result=$?
-    [[ $delete_admin_result -eq 0 ]] || fail "failed to delete admin user"
-  fi
-  create_user "$admin_username" "$admin_password" "admin" || create_admin_result=$?
-  [[ $create_admin_result -eq 0 ]] || fail "failed to create admin user"
+  setup_user "$admin_username" "$admin_password" "admin" || fail "error setting up admin user"
 
-  user_exists "$user_username" || local user_exists_result=$?
-  if [[ $user_exists_result -eq 0 ]]; then
-    delete_user "$user_username" || local delete_user_result=$?
-    [[ $delete_user_result -eq 0 ]] || fail "failed to delete user user"
+  if user_exists "$user_username"; then
+    delete_user "$user_username" || fail "failed to delete user '$user_username'"
   fi
-  create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user"
+  create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user" || fail "failed to create user '$user_username'"
 
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
-  delete_bucket "aws" "versity-gwtest-admin-bucket" || local delete_result=$?
-  [[ $delete_result -eq 0 ]] || fail "error deleting bucket if it exists"
-  create_bucket_with_user "aws" "versity-gwtest-admin-bucket" "$admin_username" "$admin_password" || create_result_two=$?
-  [[ $create_result_two -eq 0 ]] || fail "error creating bucket with user"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  delete_bucket "aws" "versity-gwtest-admin-bucket" || fail "error deleting bucket if it exists"
+  create_bucket_with_user "aws" "versity-gwtest-admin-bucket" "$admin_username" "$admin_password" || fail "error creating bucket with admin user"
 
   bucket_one_found=false
   bucket_two_found=false
-  list_buckets_with_user "aws" "$admin_username" "$admin_password"
+  list_buckets_with_user "aws" "$admin_username" "$admin_password" || fail "error listing buckets with admin user"
+  # shellcheck disable=SC2154
   for bucket in "${bucket_array[@]}"; do
     if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
       bucket_one_found=true
@@ -53,8 +43,7 @@ test_admin_user() {
   if [ $bucket_one_found == false ] || [ $bucket_two_found == false ]; then
     fail "not all expected buckets listed"
   fi
-  change_bucket_owner "$admin_username" "$admin_password" "versity-gwtest-admin-bucket" "$user_username" || local change_result=$?
-  [[ $change_result -eq 0 ]] || fail "error changing bucket owner"
+  change_bucket_owner "$admin_username" "$admin_password" "versity-gwtest-admin-bucket" "$user_username" || fail "error changing bucket owner"
 
   delete_bucket "aws" "versity-gwtest-admin-bucket"
   delete_user "$user_username"
@@ -69,17 +58,11 @@ test_create_user_already_exists() {
   username="ABCDEG"
   password="123456"
 
-  user_exists "$username" || local exists_result=$?
-  if [[ $exists_result -eq 0 ]]; then
-    delete_user "$username" || local delete_result=$?
-    [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'"
+  setup_user "$username" "123456" "admin" || fail "error setting up user"
+  if create_user "$username" "123456" "admin"; then
+    fail "'user already exists' error not returned"
   fi
 
-  create_user "$username" "123456" "admin" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "error creating user"
-  create_user "$username" "123456" "admin" || local create_result=$?
-  [[ $create_result -eq 1 ]] || fail "'user already exists' error not returned"
-
   delete_bucket "aws" "versity-gwtest-admin-bucket"
   delete_user "$username"
 }
@@ -92,31 +75,24 @@ test_user_user() {
   username="ABCDEG"
   password="123456"
 
-  user_exists "$username" || local exists_result=$?
-  if [[ $exists_result -eq 0 ]]; then
-    delete_user "$username" || local delete_result=$?
-    [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'"
-  fi
+  setup_user "$username" "$password" "user" || fail "error setting up user"
   delete_bucket "aws" "versity-gwtest-user-bucket"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket '$BUCKET_ONE_NAME'"
 
-  create_user "$username" "123456" "user" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "error creating user"
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
-
-  create_bucket_with_user "aws" "versity-gwtest-user-bucket" "$username" "$password" || create_result_two=$?
-  [[ $create_result_two -eq 1 ]] || fail "creating bucket with 'user' account failed to return error"
+  if create_bucket_with_user "aws" "versity-gwtest-user-bucket" "$username" "$password"; then
+    fail "creating bucket with 'user' account failed to return error"
+  fi
+  # shellcheck disable=SC2154
   [[ $error == *"Access Denied"* ]] || fail "error message '$error' doesn't contain 'Access Denied'"
 
-  create_bucket "aws" "versity-gwtest-user-bucket" || create_result_three=$?
-  [[ $create_result_three -eq 0 ]] || fail "creating bucket account returned error"
+  create_bucket "aws" "versity-gwtest-user-bucket" || fail "error creating bucket"
 
-  change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "versity-gwtest-user-bucket" "$username" || local change_result=$?
-  [[ $change_result -eq 0 ]] || fail "error changing bucket owner"
-  change_bucket_owner "$username" "$password" "versity-gwtest-user-bucket" "admin" || local change_result_two=$?
-  [[ $change_result_two -eq 1 ]] || fail "user shouldn't be able to change bucket owner"
+  change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "versity-gwtest-user-bucket" "$username" || fail "error changing bucket owner"
+  if change_bucket_owner "$username" "$password" "versity-gwtest-user-bucket" "admin"; then
+    fail "user shouldn't be able to change bucket owner"
+  fi
 
-  list_buckets_with_user "aws" "$username" "$password"
+  list_buckets_with_user "aws" "$username" "$password" || fail "error listing buckets with user '$username'"
   bucket_found=false
   for bucket in "${bucket_array[@]}"; do
     if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
@@ -141,22 +117,13 @@ test_userplus_operation() {
   username="ABCDEG"
   password="123456"
 
-  user_exists "$username" || local exists_result=$?
-  if [[ $exists_result -eq 0 ]]; then
-    delete_user "$username" || local delete_result=$?
-    [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'"
-  fi
   delete_bucket "aws" "versity-gwtest-userplus-bucket"
+  setup_user "$username" "$password" "userplus" || fail "error creating user '$username'"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket '$BUCKET_ONE_NAME'"
 
-  create_user "$username" "123456" "userplus" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "error creating user"
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  create_bucket_with_user "aws" "versity-gwtest-userplus-bucket" "$username" "$password" || fail "error creating bucket with user '$username'"
 
-  create_bucket_with_user "aws" "versity-gwtest-userplus-bucket" "$username" "$password" || create_result_two=$?
-  [[ $create_result_two -eq 0 ]] || fail "error creating bucket"
-
-  list_buckets_with_user "aws" "$username" "$password"
+  list_buckets_with_user "aws" "$username" "$password" || fail "error listing buckets with user '$username'"
   bucket_found=false
   for bucket in "${bucket_array[@]}"; do
     if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
@@ -169,10 +136,10 @@ test_userplus_operation() {
     fail "userplus-owned bucket not found in user list"
   fi
 
-  change_bucket_owner "$username" "$password" "versity-gwtest-userplus-bucket" "admin" || local change_result_two=$?
-  [[ $change_result_two -eq 1 ]] || fail "userplus shouldn't be able to change bucket owner"
+  if change_bucket_owner "$username" "$password" "versity-gwtest-userplus-bucket" "admin"; then
+    fail "userplus shouldn't be able to change bucket owner"
+  fi
 
   delete_bucket "aws" "versity-gwtest-admin-bucket"
-  delete_user "$username" || delete_result=$?
-  [[ $delete_result -eq 0 ]] || fail "error deleting user"
+  delete_user "$username"
 }

tests/test_user_s3cmd.sh

@@ -2,6 +2,9 @@
 
 source ./tests/test_user_common.sh
 
+export RUN_S3CMD=true
+export RUN_USERS=true
+
 @test "test_admin_user_s3cmd" {
   test_admin_user "s3cmd"
 }

tests/util.sh

@@ -8,12 +8,17 @@ source ./tests/commands/complete_multipart_upload.sh
 source ./tests/commands/create_multipart_upload.sh
 source ./tests/commands/create_bucket.sh
 source ./tests/commands/delete_bucket.sh
+source ./tests/commands/delete_bucket_policy.sh
 source ./tests/commands/delete_object.sh
+source ./tests/commands/get_bucket_ownership_controls.sh
 source ./tests/commands/get_bucket_tagging.sh
 source ./tests/commands/get_object_tagging.sh
 source ./tests/commands/head_bucket.sh
 source ./tests/commands/head_object.sh
 source ./tests/commands/list_objects.sh
+source ./tests/commands/put_bucket_acl.sh
+source ./tests/commands/put_bucket_ownership_controls.sh
+source ./tests/commands/upload_part_copy.sh
 
 # recursively delete an AWS bucket
 # param:  bucket name
@@ -29,7 +34,7 @@ delete_bucket_recursive() {
   if [[ $1 == 's3' ]]; then
     error=$(aws --no-verify-ssl s3 rb s3://"$2" --force 2>&1) || exit_code="$?"
   elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
-    delete_bucket_recursive_s3api "$2" 2>&1 || exit_code="$?"
+    delete_bucket_recursive_s3api "$2" || exit_code="$?"
   elif [[ $1 == "s3cmd" ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?"
   elif [[ $1 == "mc" ]]; then
@@ -55,16 +60,26 @@ delete_bucket_recursive_s3api() {
     log 2 "delete bucket recursive command for s3api requires bucket name"
     return 1
   fi
-  list_objects 's3api' "$1" || local list_result=$?
-  if [[ $list_result -ne 0 ]]; then
+  if ! list_objects 's3api' "$1"; then
     log 2 "error listing objects"
     return 1
   fi
   # shellcheck disable=SC2154
   for object in "${object_array[@]}"; do
-    delete_object 's3api' "$1" "$object" || local delete_object_result=$?
-    if [[ $delete_object_result -ne 0 ]]; then
+    if ! delete_object 's3api' "$1" "$object"; then
       log 2 "error deleting object $object"
+      if [[ $delete_object_error == *"WORM"* ]]; then
+        log 5 "WORM protection found"
+        if ! put_object_legal_hold "$1" "$object" "OFF"; then
+          log 2 "error removing object legal hold"
+          return 1
+        fi
+        if ! delete_object 's3api' "$1" "$object"; then
+          log 2 "error deleting object after legal hold removal"
+          return 1
+        fi
+        continue
+      fi
       return 1
     fi
   done
@@ -114,8 +129,7 @@ bucket_exists() {
     return 2
   fi
 
-  head_bucket "$1" "$2" || local check_result=$?
-  if [[ $check_result -ne 0 ]]; then
+  if ! head_bucket "$1" "$2"; then
     # shellcheck disable=SC2154
     bucket_info=$(echo "$bucket_info" | grep -v "InsecureRequestWarning")
     log 5 "$bucket_info"
@@ -134,26 +148,39 @@ bucket_exists() {
 # return:  0 for success, 1 for failure
 delete_bucket_or_contents() {
   if [ $# -ne 2 ]; then
-    echo "delete bucket or contents function requires command type, bucket name"
+    log 2 "delete bucket or contents function requires command type, bucket name"
     return 1
   fi
   if [[ $RECREATE_BUCKETS == "false" ]]; then
-    delete_bucket_contents "$1" "$2" || local delete_result=$?
-    if [[ $delete_result -ne 0 ]]; then
-      echo "error deleting bucket contents"
+    if ! delete_bucket_contents "$1" "$2"; then
+      log 2 "error deleting bucket contents"
       return 1
     fi
+    if ! delete_bucket_policy "s3api" "$2"; then
+      log 2 "error deleting bucket policies"
+      return 1
+    fi
+    if ! get_object_ownership_rule "$2"; then
+      log 2 "error getting object ownership rule"
+      return 1
+    fi
+    # shellcheck disable=SC2154
+    if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$2" "private"; then
+      log 2 "error resetting bucket ACLs"
+      return 1
+    fi
+    log 5 "bucket contents, policy, ACL deletion success"
     return 0
   fi
-  delete_bucket_recursive "$1" "$2" || local delete_result=$?
-  if [[ $delete_result -ne 0 ]]; then
-    echo "Bucket deletion error"
+  if ! delete_bucket_recursive "$1" "$2"; then
+    log 2 "Bucket deletion error"
     return 1
   fi
+  log 5 "bucket deletion success"
   return 0
 }
 
-delete_bucket_if_exists() {
+delete_bucket_or_contents_if_exists() {
   if [ $# -ne 2 ]; then
     log 2 "bucket creation function requires command type, bucket name"
     return 1
@@ -170,10 +197,8 @@ delete_bucket_if_exists() {
       log 2 "error deleting bucket or contents"
       return 1
     fi
-    #if [[ $RECREATE_BUCKETS == "false" ]]; then
     log 5 "bucket and/or bucket data deletion success"
     return 0
-    #fi
   fi
   if [[ $RECREATE_BUCKETS == "false" ]]; then
     log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
@@ -191,18 +216,29 @@ setup_bucket() {
     log 2 "bucket creation function requires command type, bucket name"
     return 1
   fi
-  delete_bucket_if_exists "$1" "$2" || local delete_bucket_result=$?
-  if [[ $delete_bucket_result -ne 0 ]]; then
+  if [[ $1 == "s3cmd" ]]; then
+    log 5 "putting bucket ownership controls"
+    put_bucket_ownership_controls "$2" "BucketOwnerPreferred"
+  fi
+  if ! delete_bucket_or_contents_if_exists "$1" "$2"; then
     log 2 "error deleting bucket, or checking for bucket existence"
     return 1
   fi
   local create_result
-  create_bucket "$1" "$2" || create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    log 2 "Error creating bucket"
-    return 1
+  log 5 "util.setup_bucket: command type: $1, bucket name: $2"
+  if [[ $RECREATE_BUCKETS == "true" ]]; then
+    if ! create_bucket "$1" "$2"; then
+      log 2 "Error creating bucket"
+      return 1
+    fi
+    log 5 "bucket creation success"
+    if [[ $1 == "s3cmd" ]]; then
+      log 5 "putting bucket ownership controls"
+      put_bucket_ownership_controls "$2" "BucketOwnerPreferred" || fail "putting bucket ownership controls failed"
+    fi
+  else
+    log 5 "skipping bucket re-creation"
   fi
-  log 5 "Bucket creation success"
   return 0
 }
 
@@ -219,35 +255,8 @@ object_exists() {
     echo "error checking if object exists"
     return 2
   fi
+  # shellcheck disable=SC2086
   return $head_result
-
-  return 0
-  local exit_code=0
-  local error=""
-  if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 ls "s3://$2/$3" 2>&1) || exit_code="$?"
-  elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api head-object --bucket "$2" --prefix "$3" 2>&1) || exit_code="$?"
-  elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || exit_code="$?"
-  elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
-  else
-    echo "invalid command type $1"
-    return 2
-  fi
-  if [ $exit_code -ne 0 ]; then
-    if [[ "$error" == "" ]] || [[ $error == *"InsecureRequestWarning"* ]]; then
-      return 1
-    else
-      echo "error checking if object exists: $error"
-      return 2
-    fi
-  # s3cmd, mc return empty when object doesn't exist, rather than error
-  elif [[ ( $1 == 's3cmd' ) || ( $1 == 'mc' ) ]] && [[ $error == "" ]]; then
-    return 1
-  fi
-  return 0
 }
 
 put_object_with_metadata() {
@@ -289,9 +298,9 @@ get_object_metadata() {
     echo "error copying object to bucket: $error"
     return 1
   fi
-  log 5 "$metadata_struct"
+  log 5 "raw metadata: $metadata_struct"
   metadata=$(echo "$metadata_struct" | jq '.Metadata')
-  echo $metadata
+  log 5 "metadata: $metadata"
   export metadata
   return 0
 }
@@ -504,12 +513,12 @@ check_object_tags_empty() {
     echo "bucket tags empty check requires command type, bucket, and key"
     return 2
   fi
-  get_object_tagging "$1" "$2" "$3" || get_result=$?
-  if [[ $get_result -ne 0 ]]; then
+  if ! get_object_tagging "$1" "$2" "$3"; then
     echo "failed to get tags"
     return 2
   fi
   check_tags_empty "$1" || local check_result=$?
+  # shellcheck disable=SC2086
   return $check_result
 }
 
@@ -518,12 +527,12 @@ check_bucket_tags_empty() {
     echo "bucket tags empty check requires command type, bucket"
     return 2
   fi
-  get_bucket_tagging "$1" "$2" || get_result=$?
-  if [[ $get_result -ne 0 ]]; then
+  if ! get_bucket_tagging "$1" "$2"; then
     echo "failed to get tags"
     return 2
   fi
   check_tags_empty "$1" || local check_result=$?
+  # shellcheck disable=SC2086
   return $check_result
 }
 
@@ -661,14 +670,12 @@ multipart_upload_before_completion() {
     return 1
   fi
 
-  split_file "$3" "$4" || split_result=$?
-  if [[ $split_result -ne 0 ]]; then
+  if ! split_file "$3" "$4"; then
     log 2 "error splitting file"
     return 1
   fi
 
-  create_multipart_upload "$1" "$2" || create_result=$?
-  if [[ $create_result -ne 0 ]]; then
+  if ! create_multipart_upload "$1" "$2"; then
     log 2 "error creating multpart upload"
     return 1
   fi
@@ -676,8 +683,7 @@ multipart_upload_before_completion() {
   parts="["
   for ((i = 1; i <= $4; i++)); do
     # shellcheck disable=SC2154
-    upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$?
-    if [[ $upload_result -ne 0 ]]; then
+    if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then
       echo "error uploading part $i"
       return 1
     fi
@@ -738,7 +744,7 @@ multipart_upload_before_completion_custom() {
     return 1
   fi
 
-  # shellcheck disable=SC2048
+  # shellcheck disable=SC2086 disable=SC2048
   create_multipart_upload_custom "$1" "$2" ${*:5} || local create_result=$?
   if [[ $create_result -ne 0 ]]; then
     log 2 "error creating multipart upload"
@@ -769,7 +775,7 @@ multipart_upload_custom() {
     return 1
   fi
 
-  # shellcheck disable=SC2048
+  # shellcheck disable=SC2086 disable=SC2048
   multipart_upload_before_completion_custom "$1" "$2" "$3" "$4" ${*:5} || local result=$?
   if [[ $result -ne 0 ]]; then
     log 2 "error performing pre-completion multipart upload"
@@ -835,18 +841,20 @@ multipart_upload_with_params() {
 # return 0 for success, 1 for failure
 run_then_abort_multipart_upload() {
   if [ $# -ne 4 ]; then
-    echo "run then abort multipart upload command missing bucket, key, file, and/or part count"
+    log 2 "run then abort multipart upload command missing bucket, key, file, and/or part count"
     return 1
   fi
 
-  multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
-  if [[ $result -ne 0 ]]; then
-    echo "error performing pre-completion multipart upload"
+  if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
+    log 2 "error performing pre-completion multipart upload"
     return 1
   fi
 
-  abort_multipart_upload "$1" "$2" "$upload_id"
-  return $?
+  if ! abort_multipart_upload "$1" "$2" "$upload_id"; then
+    log 2 "error aborting multipart upload"
+    return 1
+  fi
+  return 0
 }
 
 # copy a file to/from S3
@@ -872,19 +880,17 @@ copy_file() {
 # export parts on success, return 1 for error
 list_parts() {
   if [ $# -ne 4 ]; then
-    echo "list multipart upload parts command missing bucket, key, file, and/or part count"
+    log 2 "list multipart upload parts command requires bucket, key, file, and part count"
     return 1
   fi
 
-  multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
-  if [[ $result -ne 0 ]]; then
-    echo "error performing pre-completion multipart upload"
+  if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
+    log 2 "error performing pre-completion multipart upload"
     return 1
   fi
 
-  listed_parts=$(aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$upload_id") || local listed=$?
-  if [[ $listed -ne 0 ]]; then
-    echo "Error aborting upload: $parts"
+  if ! listed_parts=$(aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$upload_id" 2>&1); then
+    log 2 "Error listing multipart upload parts: $listed_parts"
     return 1
   fi
   export listed_parts
@@ -893,35 +899,29 @@ list_parts() {
 # list unfinished multipart uploads
 # params:  bucket, key one, key two
 # export current two uploads on success, return 1 for error
-list_multipart_uploads() {
+create_and_list_multipart_uploads() {
   if [ $# -ne 3 ]; then
-    echo "list multipart uploads command requires bucket and two keys"
+    log 2 "list multipart uploads command requires bucket and two keys"
     return 1
   fi
 
-  create_multipart_upload "$1" "$2" || local create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    echo "error creating multpart upload"
+  if ! create_multipart_upload "$1" "$2"; then
+    log 2 "error creating multpart upload"
     return 1
   fi
 
-  create_multipart_upload "$1" "$3" || local create_result_two=$?
-  if [[ $create_result_two -ne 0 ]]; then
-    echo "error creating multpart upload two"
+  if ! create_multipart_upload "$1" "$3"; then
+    log 2 "error creating multpart upload two"
     return 1
   fi
 
-  uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1") || local list_result=$?
-  if [[ $list_result -ne 0 ]]; then
-    echo "error listing uploads: $uploads"
+  if ! list_multipart_uploads "$1"; then
+    echo "error listing uploads"
     return 1
   fi
-  export uploads
+  return 0
 }
 
-# perform a multi-part upload within bucket
-# params:  bucket, key, file, number of parts
-# return 0 for success, 1 for failure
 multipart_upload_from_bucket() {
   if [ $# -ne 4 ]; then
     echo "multipart upload from bucket command missing bucket, copy source, key, and/or part count"
@@ -964,29 +964,62 @@ multipart_upload_from_bucket() {
   parts+="]"
 
   error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}') || local completed=$?
-    if [[ $completed -ne 0 ]]; then
-      echo "Error completing upload: $error"
-      return 1
-    fi
-    return 0
-
-  parts+="]"
+  if [[ $completed -ne 0 ]]; then
+    echo "Error completing upload: $error"
+    return 1
+  fi
+  return 0
 }
 
-upload_part_copy() {
+multipart_upload_from_bucket_range() {
   if [ $# -ne 5 ]; then
-    echo "upload multipart part copy function must have bucket, key, upload ID, file name, part number"
+    echo "multipart upload from bucket with range command requires bucket, copy source, key, part count, and range"
     return 1
   fi
-  local etag_json
-  echo "$1 $2 $3 $4 $5"
-  etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
-  if [[ $uploaded -ne 0 ]]; then
-    echo "Error uploading part $5: $etag_json"
+
+  split_file "$3" "$4" || local split_result=$?
+  if [[ $split_result -ne 0 ]]; then
+    echo "error splitting file"
     return 1
   fi
-  etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag')
-  export etag
+
+  for ((i=0;i<$4;i++)) {
+    echo "key: $3"
+    log 5 "file info: $(ls -l "$3"-"$i")"
+    put_object "s3api" "$3-$i" "$1" "$2-$i" || local copy_result=$?
+    if [[ $copy_result -ne 0 ]]; then
+      echo "error copying object"
+      return 1
+    fi
+  }
+
+  create_multipart_upload "$1" "$2-copy" || local create_multipart_result=$?
+  if [[ $create_multipart_result -ne 0 ]]; then
+    echo "error running first multpart upload"
+    return 1
+  fi
+
+  parts="["
+  for ((i = 1; i <= $4; i++)); do
+    upload_part_copy_with_range "$1" "$2-copy" "$upload_id" "$2" "$i" "$5" || local upload_part_copy_result=$?
+    if [[ $upload_part_copy_result -ne 0 ]]; then
+      # shellcheck disable=SC2154
+      echo "error uploading part $i: $upload_part_copy_error"
+      return 1
+    fi
+    parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
+    if [[ $i -ne $4 ]]; then
+      parts+=","
+    fi
+  done
+  parts+="]"
+
+  error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}') || local completed=$?
+  if [[ $completed -ne 0 ]]; then
+    echo "Error completing upload: $error"
+    return 1
+  fi
+  return 0
 }
 
 create_presigned_url() {

tests/util_aws.sh

@@ -29,6 +29,7 @@ abort_all_multipart_uploads() {
 
     log 5 "$lines"
     while read -r line; do
+      # shellcheck disable=SC2086
       error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" $line 2>&1) || abort_result=$?
       if [[ $abort_result -ne 0 ]]; then
         echo "error aborting multipart upload: $error"

tests/util_file.sh

@@ -157,9 +157,36 @@ create_test_file_count() {
       return 1
     fi
   }
+  # shellcheck disable=SC2153
   if [[ $LOG_LEVEL -ge 5 ]]; then
     ls_result=$(ls "$test_file_folder"/file_*)
     log 5 "$ls_result"
   fi
   return 0
 }
+
+download_and_compare_file() {
+  if [[ $# -ne 5 ]]; then
+    log 2 "'download and compare file' requires command type, original file, bucket, key, local file"
+    return 1
+  fi
+  download_and_compare_file_with_user "$1" "$2" "$3" "$4" "$5" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
+  return "$?"
+}
+
+download_and_compare_file_with_user() {
+  if [[ $# -ne 7 ]]; then
+    log 2 "'download and compare file with user' command requires command type, original file, bucket, key, local file, user, password"
+    return 1
+  fi
+  if ! get_object_with_user "$1" "$3" "$4" "$5" "$6" "$7"; then
+    log 2 "error retrieving file"
+    return 1
+  fi
+  log 5 "files: $2, $5"
+  if ! compare_files "$2" "$5"; then
+    log 2 "files don't match"
+    return 1
+  fi
+  return 0
+}

tests/util_policy.sh

@@ -6,23 +6,44 @@ check_for_empty_policy() {
     return 1
   fi
 
-  local get_result=0
-  get_bucket_policy "$1" "$2" || get_result=$?
-  if [[ $get_result -ne 0 ]]; then
-    echo "error getting bucket policy"
+  if ! get_bucket_policy "$1" "$2"; then
+    log 2 "error getting bucket policy"
     return 1
   fi
+  # shellcheck disable=SC2154
+  log 5 "bucket policy: $bucket_policy"
 
   # shellcheck disable=SC2154
   if [[ $bucket_policy == "" ]]; then
     return 0
   fi
 
-  policy=$(echo "$bucket_policy" | jq -r '.Policy')
-  statement=$(echo "$policy" | jq -r '.Statement[0]')
+  #policy=$(echo "$bucket_policy" | jq -r '.Policy')
+  statement=$(echo "$bucket_policy" | jq -r '.Statement[0]')
+  log 5 "statement: $statement"
   if [[ "" != "$statement" ]] && [[ "null" != "$statement" ]]; then
     echo "policy should be empty (actual value: '$statement')"
     return 1
   fi
   return 0
-}
+}
+
+setup_policy_with_single_statement() {
+  if [[ $# -ne 6 ]]; then
+    "'setup single policy' command requires file, version, effect, principal, action, resource"
+  fi
+  cat <<EOF > "$1"
+{
+  "Version": "$2",
+  "Statement": [
+    {
+       "Effect": "$3",
+       "Principal": "$4",
+       "Action": "$5",
+       "Resource": "$6"
+    }
+  ]
+}
+EOF
+log 5 "$(cat "$1")"
+}

tests/util_users.sh

@@ -1,13 +1,48 @@
 #!/usr/bin/env bash
 
-create_user() {
+setup_user() {
   if [[ $# -ne 3 ]]; then
-    echo "create user command requires user ID, key, and role"
+    log 2 "'setup user' command requires username, password, and role"
     return 1
   fi
-  create_user_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3" || create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    echo "error creating user: $error"
+  if user_exists "$1"; then
+    if ! delete_user "$1"; then
+      log 2 "error deleting user '$1'"
+      return 1
+    fi
+  fi
+  if ! create_user_versitygw "$1" "$2" "$3"; then
+    log 2 "error creating user '$1'"
+    return 1
+  fi
+  return 0
+}
+
+setup_user_direct() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'setup user direct' command requires username, role, and bucket"
+    return 1
+  fi
+  if user_exists "$1"; then
+    if ! delete_user "$1"; then
+      log 2 "error deleting user '$1'"
+      return 1
+    fi
+  fi
+  if ! create_user_direct "$1" "$2" "$3"; then
+    log 2 "error creating user"
+    return 1
+  fi
+  return 0
+}
+
+create_user_versitygw() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "create user command requires user ID, key, and role"
+    return 1
+  fi
+  if ! create_user_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3"; then
+    log 2 "error creating user"
     return 1
   fi
   return 0
@@ -26,20 +61,117 @@ create_user_if_nonexistent() {
   return $?
 }
 
-create_user_with_user() {
-  if [[ $# -ne 5 ]]; then
-    echo "create user with user command requires creator ID, key, and new user ID, key, and role"
+put_user_policy() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "attaching user policy requires user ID, role, bucket name"
     return 1
   fi
-  error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5") || local create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    echo "error creating user: $error"
+  if [[ -z "$test_file_folder" ]]; then
+    log 2 "no test folder defined"
+    return 1
+  fi
+
+  # TODO add other roles
+  if [[ $2 != "user" ]]; then
+    log 2 "role for '$2' not currently supported"
+    return 1
+  fi
+
+cat <<EOF > "$test_file_folder"/user_policy_file
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "*",
+      "Resource": "arn:aws:s3:::$3/*"
+    }
+  ]
+}
+EOF
+
+  if ! error=$(aws iam put-user-policy --user-name "$1" --policy-name "UserPolicy" --policy-document "file://$test_file_folder/user_policy_file" 2>&1); then
+    log 2 "error putting user policy: $error"
     return 1
   fi
   return 0
 }
 
+create_user_direct() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "create user direct command requires desired username, role, bucket name"
+    return 1
+  fi
+  if ! error=$(aws iam create-user --user-name "$1" 2>&1); then
+    log 2 "error creating new user: $error"
+    return 1
+  fi
+  if ! put_user_policy "$1" "$2" "$3"; then
+    log 2 "error attaching user policy"
+    return 1
+  fi
+  if ! keys=$(aws iam create-access-key --user-name "$1" 2>&1); then
+    log 2 "error creating keys for new user: $keys"
+    return 1
+  fi
+  key_id=$(echo "$keys" | jq -r ".AccessKey.AccessKeyId")
+  export key_id
+  secret_key=$(echo "$keys" | jq -r ".AccessKey.SecretAccessKey")
+  export secret_key
+
+  # propagation delay occurs when user is added to IAM, so wait a few seconds
+  sleep 5
+
+  return 0
+}
+
+create_user_with_user() {
+  if [[ $# -ne 5 ]]; then
+    log 2 "create user with user command requires creator ID, key, and new user ID, key, and role"
+    return 1
+  fi
+  if ! error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5" 2>&1); then
+    log 2 "error creating user: $error"
+    return 1
+  fi
+  return 0
+}
+
+list_users_direct() {
+  # AWS_ENDPOINT_URL of s3.amazonaws.com doesn't work here
+  if ! users=$(aws --profile="$AWS_PROFILE" iam list-users 2>&1); then
+    log 2 "error listing users via direct s3 call: $users"
+    return 1
+  fi
+  parsed_users=()
+  if ! users_list=$(echo "$users" | jq -r ".Users[].UserName" 2>&1); then
+    log 2 "error parsing users array: $users_list"
+    return 1
+  fi
+  while IFS= read -r line; do
+    parsed_users+=("$line")
+  done <<< "$users_list"
+  log 5 "parsed users: ${parsed_users[*]}"
+  export parsed_users
+  return 0
+}
+
 list_users() {
+  if [[ $DIRECT == "true" ]]; then
+    if ! list_users_direct; then
+      log 2 "error listing users via direct s3 call"
+      return 1
+    fi
+    return 0
+  fi
+  if ! list_users_versitygw; then
+    log 2 "error listing versitygw users"
+    return 1
+  fi
+  return 0
+}
+
+list_users_versitygw() {
   users=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users) || local list_result=$?
   if [[ $list_result -ne 0 ]]; then
     echo "error listing users: $users"
@@ -55,15 +187,15 @@ list_users() {
 
 user_exists() {
   if [[ $# -ne 1 ]]; then
-    echo "user exists command requires username"
+    log 2 "user exists command requires username"
     return 2
   fi
-  list_users || local list_result=$?
-  if [[ $list_result -ne 0 ]]; then
-    echo "error listing user"
+  if ! list_users; then
+    log 2 "error listing user"
     return 2
   fi
   for element in "${parsed_users[@]}"; do
+    log 5 "user: $element"
     if [[ $element == "$1" ]]; then
       return 0
     fi
@@ -71,28 +203,109 @@ user_exists() {
   return 1
 }
 
+delete_user_direct() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "delete user direct command requires username"
+    return 1
+  fi
+  if ! policies=$(aws iam list-user-policies --user-name "$1" --query 'PolicyNames' --output text 2>&1); then
+    log 2 "error getting user policies: $error"
+    return 1
+  fi
+  for policy_name in $policies; do
+    if ! user_policy_delete_error=$(aws iam delete-user-policy --user-name "$1" --policy-name "$policy_name" 2>&1); then
+      log 2 "error deleting user policy: $user_policy_delete_error"
+      return 1
+    fi
+  done
+  if ! keys=$(aws iam list-access-keys --user-name "$1" 2>&1); then
+    log 2 "error getting keys: $keys"
+    return 1
+  fi
+  if ! key=$(echo "$keys" | jq -r ".AccessKeyMetadata[0].AccessKeyId" 2>&1); then
+    log 2 "error getting key ID: $key"
+    return 1
+  fi
+  if [[ $key != "null" ]]; then
+    if ! error=$(aws iam delete-access-key --user-name "$1" --access-key-id "$key" 2>&1); then
+      log 2 "error deleting access key: $error"
+      return 1
+    fi
+  fi
+  if ! error=$(aws --profile="$AWS_PROFILE" iam delete-user --user-name "$1" 2>&1); then
+    log 2 "error deleting user: $error"
+    return 1
+  fi
+  return 0
+}
+
+delete_user_versitygw() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "delete user via versitygw command requires user ID or username"
+    return 1
+  fi
+  log 5 "$VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access $1"
+  if ! error=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1" 2>&1); then
+    log 2 "error deleting user: $error"
+    export error
+    return 1
+  fi
+  return 0
+}
+
 delete_user() {
   if [[ $# -ne 1 ]]; then
-      echo "delete user command requires user ID"
-      return 1
-    fi
-    error=$($VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access "$1") || local delete_result=$?
-    if [[ $delete_result -ne 0 ]]; then
-      echo "error deleting user: $error"
+    log 2 "delete user command requires user ID"
+    return 1
+  fi
+  if [[ $DIRECT == "true" ]]; then
+    if ! delete_user_direct "$1"; then
+      log 2 "error deleting user direct via s3"
       return 1
     fi
+    log 5 "user '$1' deleted successfully"
     return 0
+  fi
+  if ! delete_user_versitygw "$1"; then
+    log 2 "error deleting user via versitygw"
+    return 1
+  fi
 }
 
 change_bucket_owner() {
   if [[ $# -ne 4 ]]; then
-      echo "change bucket owner command requires ID, key, bucket name, and new owner"
-      return 1
-    fi
-    error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$?
-    if [[ $change_result -ne 0 ]]; then
-      echo "error changing bucket owner: $error"
-      return 1
-    fi
-    return 0
+    echo "change bucket owner command requires ID, key, bucket name, and new owner"
+    return 1
+  fi
+  error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$?
+  if [[ $change_result -ne 0 ]]; then
+    echo "error changing bucket owner: $error"
+    return 1
+  fi
+  return 0
 }
+
+get_bucket_owner() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get bucket owner' command requires bucket name"
+    return 1
+  fi
+  if ! buckets=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-buckets 2>&1); then
+    log 2 "error listing buckets: $buckets"
+    return 1
+  fi
+  log 5 "BUCKET DATA:  $buckets"
+  bucket_vals=$(echo "$buckets" | awk 'NR > 2')
+  while IFS= read -r line; do
+    log 5 "bucket line: $line"
+    bucket=$(echo "$line" | awk '{print $1}')
+    if [[ $bucket == "$1" ]]; then
+      bucket_owner=$(echo "$line" | awk '{print $2}')
+      export bucket_owner
+      return 0
+    fi
+  done <<< "$bucket_vals"
+  log 3 "bucket owner for bucket '$1' not found"
+  bucket_owner=
+  return 0
+}

tests/versity.sh

@@ -1,126 +1,47 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 source ./tests/util_file.sh
-
-check_exe_params_versity() {
-  if [ -z "$LOCAL_FOLDER" ]; then
-    echo "No local storage folder set"
-    return 1
-  elif [ -z "$VERSITY_EXE" ]; then
-    echo "No versity executable location set"
-    return 1
-  elif [ -z "$BACKEND" ]; then
-    echo "No backend parameter set (options: 'posix')"
-    return 1
-  fi
-  if [ "$BACKEND" == 's3' ]; then
-    if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then
-      echo "missing second AWS access key ID for s3 backend"
-      return 1
-    fi
-    if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then
-      echo "missing second AWS secret access key for s3 backend"
-      return 1
-    fi
-  fi
-}
-
-check_exe_params() {
-  if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-    echo "No AWS access key set"
-    return 1
-  elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
-    echo "No AWS secret access key set"
-    return 1
-  elif [ -z "$AWS_PROFILE" ]; then
-    echo "No AWS profile set"
-    return 1
-  elif [ -z "$AWS_ENDPOINT_URL" ]; then
-    echo "No AWS endpoint URL set"
-    return 1
-  elif [ -z "$MC_ALIAS" ]; then
-    echo "No mc alias set"
-    return 1
-  elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then
-    echo "RUN_VERSITYGW must be 'true' or 'false'"
-    return 1
-  elif [ -z "$USERS_FOLDER" ]; then
-    echo "No users folder parameter set"
-    return 1
-  fi
-  if [[ -r $GOCOVERDIR ]]; then
-    export GOCOVERDIR=$GOCOVERDIR
-  fi
-  if [[ $RUN_VERSITYGW == "true" ]]; then
-    local check_result
-    check_exe_params_versity || check_result=$?
-    if [[ $check_result -ne 0 ]]; then
-      return 1
-    fi
-  fi
-}
-
-start_versity() {
-  if [ -z "$VERSITYGW_TEST_ENV" ]; then
-    if [ -r tests/.env ]; then
-      source tests/.env
-    else
-      echo "Warning: no .env file found in tests folder"
-    fi
-  else
-    # shellcheck source=./.env.default
-    source "$VERSITYGW_TEST_ENV"
-  fi
-  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
-    # shellcheck source=/.secrets
-    source "$SECRETS_FILE"
-  else
-    echo "Warning: no secrets file found"
-  fi
-
-  check_exe_params || check_result=$?
-  if [[ $check_result -ne 0 ]]; then
-    echo "error checking for parameters"
-    return 1
-  fi
-
-  if [ "$RUN_VERSITYGW" == "true" ]; then
-    run_versity_app || run_result=$?
-    if [[ $run_result -ne 0 ]]; then
-      echo "error starting versity apps"
-      return 1
-    fi
-  fi
-
-  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL VERSITY_EXE
-}
+source ./tests/iam.sh
 
 start_versity_process() {
   if [[ $# -ne 1 ]]; then
-    echo "start versity process function requires number"
+    log 2 "start versity process function requires number"
     return 1
   fi
-  create_test_file_folder || create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    echo "error creating test log folder"
+  if ! create_test_file_folder; then
+    log 2 "error creating test log folder"
     return 1
   fi
-  base_command+=(">" "$test_file_folder/versity_log_$1.txt" "2>&1")
-  ("${base_command[@]}") &
+  IFS=' ' read -r -a full_command <<< "${base_command[@]}"
+  log 5 "versity command: ${full_command[*]}"
+  if [ -n "$VERSITY_LOG_FILE" ]; then
+    "${full_command[@]}" >> "$VERSITY_LOG_FILE" 2>&1 &
+  else
+    "${full_command[@]}" 2>&1 &
+  fi
+  # shellcheck disable=SC2181
   if [[ $? -ne 0 ]]; then
-    echo "error running versitygw command: $(cat "$test_file_folder/versity_log_$1.txt")"
+    sleep 1
+    if [ -n "$VERSITY_LOG_FILE" ]; then
+      log 2 "error running versitygw command: $(cat "$VERSITY_LOG_FILE")"
+    fi
     return 1
   fi
   eval versitygw_pid_"$1"=$!
+  if [ -n "$VERSITY_LOG_FILE" ]; then
+    process_info="Versity process $1, PID $!"
+    echo "$process_info" >> "$VERSITY_LOG_FILE"
+  fi
+  log 4 "$process_info"
   local pid
   eval pid=\$versitygw_pid_"$1"
   sleep 1
 
-  local proc_check
-  check_result=$(kill -0 $pid 2>&1) || proc_check=$?
-  if [[ $proc_check -ne 0 ]]; then
-    echo "versitygw failed to start: $check_result"
-    echo "log data: $(cat "$test_file_folder/versity_log_$1.txt")"
+  if ! check_result=$(kill -0 "$pid" 2>&1); then
+    log 2 "versitygw failed to start: $check_result"
+    if [ -n "$VERSITY_LOG_FILE" ]; then
+      log 2 "log data: $(cat "$VERSITY_LOG_FILE")"
+    fi
     return 1
   fi
   export versitygw_pid_"$1"
@@ -128,14 +49,44 @@ start_versity_process() {
 
 run_versity_app_posix() {
   if [[ $# -ne 3 ]]; then
-    echo "run versity app w/posix command requires access ID, secret key, process number"
+    log 2 "run versity app w/posix command requires access ID, secret key, process number"
+    return 1
+  fi
+  base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION")
+  if [ -n "$RUN_USERS" ]; then
+    # shellcheck disable=SC2153
+    IFS=' ' read -r -a iam_array <<< "$IAM_PARAMS"
+  fi
+  base_command+=("${iam_array[@]}")
+  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
+    base_command+=(--cert "$CERT" --key "$KEY")
+  fi
+  if [ -n "$PORT" ]; then
+    base_command+=(--port ":$PORT")
+  fi
+  base_command+=(posix "$LOCAL_FOLDER")
+  export base_command
+
+  if ! start_versity_process "$3"; then
+    log 2 "error starting versity process"
+    return 1
+  fi
+  return 0
+}
+
+run_versity_app_scoutfs() {
+  if [[ $# -ne 3 ]]; then
+    echo "run versity app w/scoutfs command requires access ID, secret key, process number"
     return 1
   fi
   base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION"  --iam-dir="$USERS_FOLDER")
   if [ -n "$CERT" ] && [ -n "$KEY" ]; then
     base_command+=(--cert "$CERT" --key "$KEY")
   fi
-  base_command+=(posix "$LOCAL_FOLDER")
+  if [ -n "$PORT" ]; then
+    base_command+=(--port ":$PORT")
+  fi
+  base_command+=(scoutfs "$LOCAL_FOLDER")
   export base_command
 
   local versity_result
@@ -149,20 +100,23 @@ run_versity_app_posix() {
 
 run_versity_app_s3() {
   if [[ $# -ne 1 ]]; then
-    echo "run versity app w/s3 command requires process number"
+    log 2 "run versity app w/s3 command requires process number"
     return 1
   fi
-  base_command=("$VERSITY_EXE" --port=":7071" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY")
+  base_command=("$VERSITY_EXE" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY")
   if [ -n "$CERT" ] && [ -n "$KEY" ]; then
     base_command+=(--cert "$CERT" --key "$KEY")
   fi
+  if [ -n "$PORT_TWO" ]; then
+    base_command+=(--port ":$PORT_TWO")
+  else
+    base_command+=(--port ":7071")
+  fi
   base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com)
   export base_command
 
-  local versity_result
-  start_versity_process "$1" || versity_result=$?
-  if [[ $versity_result -ne 0 ]]; then
-    echo "error starting versity process"
+  if ! start_versity_process "$1"; then
+    log 2 "error starting versity process"
     return 1
   fi
   return 0
@@ -170,38 +124,51 @@ run_versity_app_s3() {
 
 run_versity_app() {
   if [[ $BACKEND == 'posix' ]]; then
-    run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
+    if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
+      log 2 "error starting versity app"
+      return 1
+    fi
+  elif [[ $BACKEND == 'scoutfs' ]]; then
+    run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
     if [[ $result_one -ne 0 ]]; then
       echo "error starting versity app"
       return 1
     fi
   elif [[ $BACKEND == 's3' ]]; then
-    run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
-    if [[ $result_one -ne 0 ]]; then
-      echo "error starting versity app"
+    if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
+      log 2 "error starting versity app"
       return 1
     fi
-    run_versity_app_s3 "2" || result_two=$?
-    if [[ $result_two -ne 0 ]]; then
-      echo "error starting second versity app"
+    if ! run_versity_app_s3 "2"; then
+      log 2 "error starting second versity app"
       return 1
     fi
   else
-    echo "unrecognized backend type $BACKEND"
+    log 2 "unrecognized backend type $BACKEND"
     return 1
   fi
+  if [[ $IAM_TYPE == "s3" ]]; then
+    if ! bucket_exists "s3api" "$USERS_BUCKET"; then
+      if ! create_bucket "s3api" "$USERS_BUCKET"; then
+        log 2 "error creating IAM bucket"
+        return 1
+      fi
+    fi
+  fi
 }
 
 stop_single_process() {
   if [[ $# -ne 1 ]]; then
-    echo "stop single process function requires process ID"
+    log 2 "stop single process function requires process ID"
     return 1
   fi
-  if ps -p "$1" > /dev/null; then
+  log 5 "stop process with ID: $1"
+  # shellcheck disable=SC2086
+  if ps_result=$(ps -p $1 2>&1) > /dev/null; then
     kill "$1"
     wait "$1" || true
   else
-    echo "Process with PID $1 does not exist."
+    log 3 "error stopping versity app: $ps_result"
   fi
 }
 
@@ -209,18 +176,14 @@ stop_versity() {
   if [ "$RUN_VERSITYGW" == "false" ]; then
     return
   fi
-  local result_one
-  local result_two
   # shellcheck disable=SC2154
-  stop_single_process "$versitygw_pid_1" || result_one=$?
-  if [[ $result_one -ne 0 ]]; then
-    echo "error stopping versity process"
+  if ! stop_single_process "$versitygw_pid_1"; then
+    log 2 "error stopping versity process"
   fi
   if [[ $BACKEND == 's3' ]]; then
     # shellcheck disable=SC2154
-    stop_single_process "$versitygw_pid_2" || result_two=$?
-    if [[ $result_two -ne 0 ]]; then
-      echo "error stopping versity process two"
+    if ! stop_single_process "$versitygw_pid_2"; then
+      log 2 "error stopping versity process two"
     fi
   fi
 }