Merge pull request #663 from versity/test_cmdline_acl_correction

Test cmdline acl correction

.github/workflows/shellcheck.yml

@@ -0,0 +1,16 @@
+name: shellcheck
+on: pull_request
+jobs:
+
+  build:
+    name: Run shellcheck
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Run checks
+      run: |
+        shellcheck --version
+        shellcheck -e SC1091 tests/*.sh tests/*/*.sh

.github/workflows/system.yml

@@ -4,16 +4,91 @@ jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          #- set: 1
+          #  LOCAL_FOLDER: /tmp/gw1
+          #  BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
+          #  BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+          #  IAM_TYPE: folder
+          #  USERS_FOLDER: /tmp/iam1
+          #  AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          #  RUN_SET: "s3cmd"
+          #  RECREATE_BUCKETS: "true"
+          #  PORT: 7070
+          #  BACKEND: "posix"
+          - set: 2
+            LOCAL_FOLDER: /tmp/gw2
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam2
+            AWS_ENDPOINT_URL: https://127.0.0.1:7071
+            RUN_SET: "s3"
+            RECREATE_BUCKETS: "true"
+            PORT: 7071
+            BACKEND: "posix"
+          - set: 3
+            LOCAL_FOLDER: /tmp/gw3
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam3
+            AWS_ENDPOINT_URL: https://127.0.0.1:7072
+            RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
+            PORT: 7072
+            BACKEND: "posix"
+          - set: 4
+            LOCAL_FOLDER: /tmp/gw4
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam4
+            AWS_ENDPOINT_URL: https://127.0.0.1:7073
+            RUN_SET: "mc"
+            RECREATE_BUCKETS: "true"
+            PORT: 7073
+            BACKEND: "posix"
+          - set: 5
+            LOCAL_FOLDER: /tmp/gw5
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
+            IAM_TYPE: s3
+            USERS_BUCKET: versity-gwtest-iam
+            AWS_ENDPOINT_URL: https://127.0.0.1:7074
+            RUN_SET: "aws-user"
+            RECREATE_BUCKETS: "true"
+            PORT: 7074
+            BACKEND: "posix"
+          - set: 6
+            LOCAL_FOLDER: /tmp/gw6
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam6
+            AWS_ENDPOINT_URL: https://127.0.0.1:7075
+            RUN_SET: "aws"
+            RECREATE_BUCKETS: "false"
+            PORT: 7075
+            BACKEND: "posix"
+          - set: 7
+            LOCAL_FOLDER: /tmp/gw7
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam7
+            AWS_ENDPOINT_URL: https://127.0.0.1:7076
+            RUN_SET: "aws"
+            RECREATE_BUCKETS: "true"
+            PORT: 7076
+            BACKEND: "s3"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
 
-      - name: Install ShellCheck
-        run: sudo apt-get install shellcheck
-
-      - name: Run ShellCheck
-        run: shellcheck -S warning ./tests/*.sh
-
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -39,34 +114,46 @@ jobs:
           chmod 755 /usr/local/bin/mc
 
       - name: Build and run, posix backend
+        env:
+          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
+          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
+          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
+          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
+          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
+          IAM_TYPE: ${{ matrix.IAM_TYPE }}
+          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
+          RUN_SET: ${{ matrix.RUN_SET }}
+          PORT: ${{ matrix.PORT }}
+          AWS_PROFILE: versity
+          VERSITY_EXE: ${{ github.workspace }}/versitygw
+          RUN_VERSITYGW: true
+          BACKEND: ${{ matrix.BACKEND }}
+          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
+          CERT: ${{ github.workspace }}/cert.pem
+          KEY: ${{ github.workspace }}/versitygw.pem
+          S3CMD_CONFIG: tests/s3cfg.local.default
+          MC_ALIAS: versity
+          LOG_LEVEL: 4
+          GOCOVERDIR: ${{ github.workspace }}/cover
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
+          export AWS_ACCESS_KEY_ID_TWO=user
+          export AWS_SECRET_ACCESS_KEY_TWO=pass
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity
-          mkdir /tmp/gw
+          mkdir $LOCAL_FOLDER
           export WORKSPACE=$GITHUB_WORKSPACE
-          openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
-          openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir cover iam
-          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
-
-      #- name: Build and run, s3 backend
-      #  run: |
-      #    make testbin
-      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export AWS_REGION=us-east-1
-      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
-      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
-      #    aws configure set aws_region $AWS_REGION --profile versity_s3
-      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export WORKSPACE=$GITHUB_WORKSPACE
-      #    VERSITYGW_TEST_ENV=./tests/.env.s3 GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          mkdir $GOCOVERDIR $USERS_FOLDER
+          if [[ $RECREATE_BUCKETS == "false" ]]; then
+            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
+          fi
+          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
       - name: Coverage report
         run: |

README.md

@@ -6,7 +6,7 @@
   <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
 </picture>
 
- [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
 
 ### Binary release builds
 Download [latest release](https://github.com/versity/versitygw/releases)
@@ -22,8 +22,7 @@ Download [latest release](https://github.com/versity/versitygw/releases)
 * Simplified interface for adding new storage system support
 
 ### News
-* New performance (scale up) analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
-* New performance (scale out) Part 2 analysis article [https://github.com/versity/versitygw/wiki/Performance-Part-2](https://github.com/versity/versitygw/wiki/Performance-Part-2)
+Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
 
 ### Mailing List
 Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).

auth/acl.go

@@ -17,6 +17,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 
@@ -27,7 +28,6 @@ import (
 )
 
 type ACL struct {
-	ACL      types.BucketCannedACL
 	Owner    string
 	Grantees []Grantee
 }
@@ -93,12 +93,34 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
 	}
 
+	defaultGrantees := []Grantee{
+		{
+			Permission: types.PermissionFullControl,
+			Access:     acl.Owner,
+		},
+	}
+
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
-		acl.ACL = input.ACL
-		acl.Grantees = []Grantee{}
+		switch input.ACL {
+		case types.BucketCannedACLPublicRead:
+			defaultGrantees = append(defaultGrantees, Grantee{
+				Permission: types.PermissionRead,
+				Access:     "all-users",
+			})
+		case types.BucketCannedACLPublicReadWrite:
+			defaultGrantees = append(defaultGrantees, []Grantee{
+				{
+					Permission: types.PermissionRead,
+					Access:     "all-users",
+				},
+				{
+					Permission: types.PermissionWrite,
+					Access:     "all-users",
+				},
+			}...)
+		}
 	} else {
-		grantees := []Grantee{}
 		accs := []string{}
 
 		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -107,31 +129,31 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
 				}
 			}
 			if input.GrantRead != nil && *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "READ"})
 				}
 			}
 			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "READ_ACP"})
 				}
 			}
 			if input.GrantWrite != nil && *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "WRITE"})
 				}
 			}
 			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{Access: str, Permission: "WRITE_ACP"})
 				}
 			}
 
@@ -142,7 +164,7 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
 					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 				}
-				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
+				defaultGrantees = append(defaultGrantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
 				if _, ok := cache[*grt.Grantee.ID]; !ok {
 					cache[*grt.Grantee.ID] = true
 					accs = append(accs, *grt.Grantee.ID)
@@ -158,11 +180,10 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		if len(accList) > 0 {
 			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
 		}
-
-		acl.Grantees = grantees
-		acl.ACL = ""
 	}
 
+	acl.Grantees = defaultGrantees
+
 	result, err := json.Marshal(acl)
 	if err != nil {
 		return nil, err
@@ -206,34 +227,21 @@ func splitUnique(s, divider string) []string {
 }
 
 func verifyACL(acl ACL, access string, permission types.Permission) error {
-	if acl.ACL != "" {
-		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
+	grantee := Grantee{Access: access, Permission: permission}
+	granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
+	granteeAllUsers := Grantee{Access: "all-users", Permission: permission}
 
+	isFound := false
+
+	for _, grt := range acl.Grantees {
+		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
+			isFound = true
+			break
+		}
+	}
+
+	if isFound {
 		return nil
-	} else {
-		if len(acl.Grantees) == 0 {
-			return nil
-		}
-		grantee := Grantee{Access: access, Permission: permission}
-		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
-
-		isFound := false
-
-		for _, grt := range acl.Grantees {
-			if grt == grantee || grt == granteeFullCtrl {
-				isFound = true
-				break
-			}
-		}
-
-		if isFound {
-			return nil
-		}
 	}
 
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -279,32 +287,31 @@ type AccessOptions struct {
 	Bucket        string
 	Object        string
 	Action        Action
+	Readonly      bool
 }
 
 func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	if opts.Readonly {
+		if opts.AclPermission == types.PermissionWrite || opts.AclPermission == types.PermissionWriteAcp {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+	}
 	if opts.IsRoot {
 		return nil
 	}
 	if opts.Acc.Role == RoleAdmin {
 		return nil
 	}
-	if opts.Acc.Access == opts.Acl.Owner {
-		return nil
+
+	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
 	}
 
-	policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
-	if err != nil {
-		return err
-	}
-
-	// If bucket policy is not set and the ACL is default, only the owner has access
-	if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
-		return err
-	}
 	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
 		return err
 	}

auth/bucket_policy.go

@@ -16,12 +16,22 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
+	"errors"
 	"net/http"
 
 	"github.com/versity/versitygw/s3err"
 )
 
+var (
+	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidResource = errors.New("Policy has invalid resource")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidPrincipal = errors.New("Invalid principal in policy")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidAction = errors.New("Policy has invalid action")
+)
+
 type BucketPolicy struct {
 	Statement []BucketPolicyItem `json:"Statement"`
 }
@@ -75,11 +85,14 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 
 	for action := range bpi.Actions {
 		isObjectAction := action.IsObjectAction()
-		if isObjectAction && !containsObjectAction {
-			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		if isObjectAction == nil {
+			break
 		}
-		if !isObjectAction && !containsBucketAction {
-			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		if *isObjectAction && !containsObjectAction {
+			return errResourceMismatch
+		}
+		if !*isObjectAction && !containsBucketAction {
+			return errResourceMismatch
 		}
 	}
 
@@ -108,6 +121,11 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 		return getMalformedPolicyError(err)
 	}
 
+	if len(policy.Statement) == 0 {
+		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
+	}
+
 	if err := policy.Validate(bucket, iam); err != nil {
 		return getMalformedPolicyError(err)
 	}
@@ -115,12 +133,7 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 	return nil
 }
 
-func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
-	// If bucket policy is not set
-	if len(policy) == 0 {
-		return nil
-	}
-
+func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
 	var bucketPolicy BucketPolicy
 	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
 		return err

auth/bucket_policy_actions.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -56,6 +55,8 @@ const (
 	GetObjectRetentionAction               Action = "s3:GetObjectRetention"
 	PutObjectRetentionAction               Action = "s3:PutObjectRetention"
 	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
+	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
+	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
 	AllActions                             Action = "s3:*"
 )
 
@@ -92,6 +93,8 @@ var supportedActionList = map[Action]struct{}{
 	GetObjectRetentionAction:               {},
 	PutObjectRetentionAction:               {},
 	BypassGovernanceRetentionAction:        {},
+	PutBucketOwnershipControlsAction:       {},
+	GetBucketOwnershipControlsAction:       {},
 	AllActions:                             {},
 }
 
@@ -119,7 +122,7 @@ var supportedObjectActionList = map[Action]struct{}{
 // Validates Action: it should either wildcard match with supported actions list or be in it
 func (a Action) IsValid() error {
 	if !strings.HasPrefix(string(a), "s3:") {
-		return fmt.Errorf("invalid action: %v", a)
+		return errInvalidAction
 	}
 
 	if a == AllActions {
@@ -134,31 +137,39 @@ func (a Action) IsValid() error {
 			}
 		}
 
-		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+		return errInvalidAction
 	}
 
 	_, found := supportedActionList[a]
 	if !found {
-		return fmt.Errorf("unsupported action: %v", a)
+		return errInvalidAction
 	}
 	return nil
 }
 
+func getBoolPtr(bl bool) *bool {
+	return &bl
+}
+
 // Checks if the action is object action
-func (a Action) IsObjectAction() bool {
+// nil points to 's3:*'
+func (a Action) IsObjectAction() *bool {
+	if a == AllActions {
+		return nil
+	}
 	if a[len(a)-1] == '*' {
 		pattern := strings.TrimSuffix(string(a), "*")
 		for act := range supportedObjectActionList {
 			if strings.HasPrefix(string(act), pattern) {
-				return true
+				return getBoolPtr(true)
 			}
 		}
 
-		return false
+		return getBoolPtr(false)
 	}
 
 	_, found := supportedObjectActionList[a]
-	return found
+	return &found
 }
 
 func (a Action) WildCardMatch(act Action) bool {
@@ -177,7 +188,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("actions can't be empty")
+			return errInvalidAction
 		}
 		*a = make(Actions)
 		for _, s := range ss {
@@ -190,7 +201,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("actions can't be empty")
+				return errInvalidAction
 			}
 			*a = make(Actions)
 			err = a.Add(s)

auth/bucket_policy_effect.go

@@ -30,5 +30,6 @@ func (bpat BucketPolicyAccessType) Validate() error {
 		return nil
 	}
 
-	return fmt.Errorf("invalid effect: %v", bpat)
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	return fmt.Errorf("Invalid effect: %v", bpat)
 }

auth/bucket_policy_principals.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 )
 
 type Principals map[string]struct{}
@@ -28,23 +27,50 @@ func (p Principals) Add(key string) {
 // Override UnmarshalJSON method to decode both []string and string properties
 func (p *Principals) UnmarshalJSON(data []byte) error {
 	ss := []string{}
+	var s string
+	var k struct {
+		AWS string
+	}
+
 	var err error
+
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		for _, s := range ss {
 			p.Add(s)
 		}
+		return nil
+	} else if err = json.Unmarshal(data, &s); err == nil {
+		if s == "" {
+			return errInvalidPrincipal
+		}
+		*p = make(Principals)
+		p.Add(s)
+
+		return nil
+	} else if err = json.Unmarshal(data, &k); err == nil {
+		if k.AWS == "" {
+			return errInvalidPrincipal
+		}
+		*p = make(Principals)
+		p.Add(k.AWS)
+
+		return nil
 	} else {
-		var s string
-		if err = json.Unmarshal(data, &s); err == nil {
-			if s == "" {
-				return fmt.Errorf("principals can't be empty")
+		var sk struct {
+			AWS []string
+		}
+		if err = json.Unmarshal(data, &sk); err == nil {
+			if len(sk.AWS) == 0 {
+				return errInvalidPrincipal
 			}
 			*p = make(Principals)
-			p.Add(s)
+			for _, s := range sk.AWS {
+				p.Add(s)
+			}
 		}
 	}
 
@@ -71,7 +97,7 @@ func (p Principals) Validate(iam IAMService) error {
 		if len(p) == 1 {
 			return nil
 		}
-		return fmt.Errorf("principals should either contain * or user access keys")
+		return errInvalidPrincipal
 	}
 
 	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -79,7 +105,7 @@ func (p Principals) Validate(iam IAMService) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user accounts don't exist: %v", accs)
+		return errInvalidPrincipal
 	}
 
 	return nil

auth/bucket_policy_resources.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -30,7 +29,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("resources can't be empty")
+			return errInvalidResource
 		}
 		*r = make(Resources)
 		for _, s := range ss {
@@ -43,7 +42,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("resources can't be empty")
+				return errInvalidResource
 			}
 			*r = make(Resources)
 			err = r.Add(s)
@@ -60,12 +59,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 func (r Resources) Add(rc string) error {
 	ok, pattern := isValidResource(rc)
 	if !ok {
-		return fmt.Errorf("invalid resource: %v", rc)
-	}
-
-	_, found := r[pattern]
-	if found {
-		return fmt.Errorf("duplicate resource: %v", rc)
+		return errInvalidResource
 	}
 
 	r[pattern] = struct{}{}
@@ -99,7 +93,7 @@ func (r Resources) ContainsBucketPattern() bool {
 func (r Resources) Validate(bucket string) error {
 	for resource := range r {
 		if !strings.HasPrefix(resource, bucket) {
-			return fmt.Errorf("incorrect bucket name in %v", resource)
+			return errInvalidResource
 		}
 	}
 

auth/iam.go

@@ -30,12 +30,30 @@ const (
 
 // Account is a gateway IAM account
 type Account struct {
-	Access    string `json:"access"`
-	Secret    string `json:"secret"`
-	Role      Role   `json:"role"`
-	UserID    int    `json:"userID"`
-	GroupID   int    `json:"groupID"`
-	ProjectID int    `json:"projectID"`
+	Access  string `json:"access"`
+	Secret  string `json:"secret"`
+	Role    Role   `json:"role"`
+	UserID  int    `json:"userID"`
+	GroupID int    `json:"groupID"`
+}
+
+// Mutable props, which could be changed when updating an IAM account
+type MutableProps struct {
+	Secret  *string `json:"secret"`
+	UserID  *int    `json:"userID"`
+	GroupID *int    `json:"groupID"`
+}
+
+func updateAcc(acc *Account, props MutableProps) {
+	if props.Secret != nil {
+		acc.Secret = *props.Secret
+	}
+	if props.GroupID != nil {
+		acc.GroupID = *props.GroupID
+	}
+	if props.UserID != nil {
+		acc.UserID = *props.UserID
+	}
 }
 
 // IAMService is the interface for all IAM service implementations
@@ -44,33 +62,50 @@ type Account struct {
 type IAMService interface {
 	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
+	UpdateUserAccount(access string, props MutableProps) error
 	DeleteUserAccount(access string) error
 	ListUserAccounts() ([]Account, error)
 	Shutdown() error
 }
 
-var ErrNoSuchUser = errors.New("user not found")
+var (
+	// ErrUserExists is returned when the user already exists
+	ErrUserExists = errors.New("user already exists")
+	// ErrNoSuchUser is returned when the user does not exist
+	ErrNoSuchUser = errors.New("user not found")
+)
 
 type Opts struct {
-	Dir                string
-	LDAPServerURL      string
-	LDAPBindDN         string
-	LDAPPassword       string
-	LDAPQueryBase      string
-	LDAPObjClasses     string
-	LDAPAccessAtr      string
-	LDAPSecretAtr      string
-	LDAPRoleAtr        string
-	S3Access           string
-	S3Secret           string
-	S3Region           string
-	S3Bucket           string
-	S3Endpoint         string
-	S3DisableSSlVerfiy bool
-	S3Debug            bool
-	CacheDisable       bool
-	CacheTTL           int
-	CachePrune         int
+	Dir                    string
+	LDAPServerURL          string
+	LDAPBindDN             string
+	LDAPPassword           string
+	LDAPQueryBase          string
+	LDAPObjClasses         string
+	LDAPAccessAtr          string
+	LDAPSecretAtr          string
+	LDAPRoleAtr            string
+	LDAPUserIdAtr          string
+	LDAPGroupIdAtr         string
+	VaultEndpointURL       string
+	VaultSecretStoragePath string
+	VaultMountPath         string
+	VaultRootToken         string
+	VaultRoleId            string
+	VaultRoleSecret        string
+	VaultServerCert        string
+	VaultClientCert        string
+	VaultClientCertKey     string
+	S3Access               string
+	S3Secret               string
+	S3Region               string
+	S3Bucket               string
+	S3Endpoint             string
+	S3DisableSSlVerfiy     bool
+	S3Debug                bool
+	CacheDisable           bool
+	CacheTTL               int
+	CachePrune             int
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -83,14 +118,19 @@ func New(o *Opts) (IAMService, error) {
 		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
 		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
-			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
-			o.LDAPObjClasses)
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
+			o.LDAPGroupIdAtr, o.LDAPObjClasses)
 		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
 	case o.S3Endpoint != "":
 		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
 			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
 		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
 			o.S3Endpoint, o.S3Bucket)
+	case o.VaultEndpointURL != "":
+		svc, err = NewVaultIAMService(o.VaultEndpointURL, o.VaultSecretStoragePath,
+			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
+			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
+		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")

auth/iam_cache.go

@@ -66,6 +66,21 @@ func (i *icache) get(k string) (Account, bool) {
 	return v.value, true
 }
 
+func (i *icache) update(k string, props MutableProps) {
+	i.Lock()
+	defer i.Unlock()
+
+	item, found := i.items[k]
+	if found {
+		updateAcc(&item.value, props)
+
+		// refresh the expiration date
+		item.exp = time.Now().Add(i.expire)
+
+		i.items[k] = item
+	}
+}
+
 func (i *icache) Delete(k string) {
 	i.Lock()
 	delete(i.items, k)
@@ -166,6 +181,16 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
 	return nil
 }
 
+func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
+	err := c.service.UpdateUserAccount(access, props)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.update(access, props)
+	return nil
+}
+
 // ListUserAccounts is a passthrough to the underlying service and
 // does not make use of the cache
 func (c *IAMCache) ListUserAccounts() ([]Account, error) {

auth/iam_internal.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"sync"
 	"time"
 )
 
@@ -32,6 +33,13 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
 	dir string
 }
 
@@ -62,6 +70,9 @@ func NewInternal(dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -70,7 +81,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 
 		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
-			return nil, fmt.Errorf("account already exists")
+			return nil, ErrUserExists
 		}
 		conf.AccessAccounts[account.Access] = account
 
@@ -86,6 +97,9 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -99,9 +113,41 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+// UpdateUserAccount updates the specified user account fields. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		acc, found := conf.AccessAccounts[access]
+		if !found {
+			return nil, ErrNoSuchUser
+		}
+
+		updateAcc(&acc, props)
+		conf.AccessAccounts[access] = acc
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -121,6 +167,9 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 
 // ListUserAccounts lists all the user accounts stored.
 func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return []Account{}, fmt.Errorf("get iam data: %w", err)
@@ -135,12 +184,11 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
 	var accs []Account
 	for _, k := range keys {
 		accs = append(accs, Account{
-			Access:    k,
-			Secret:    conf.AccessAccounts[k].Secret,
-			Role:      conf.AccessAccounts[k].Role,
-			UserID:    conf.AccessAccounts[k].UserID,
-			GroupID:   conf.AccessAccounts[k].GroupID,
-			ProjectID: conf.AccessAccounts[k].ProjectID,
+			Access:  k,
+			Secret:  conf.AccessAccounts[k].Secret,
+			Role:    conf.AccessAccounts[k].Role,
+			UserID:  conf.AccessAccounts[k].UserID,
+			GroupID: conf.AccessAccounts[k].GroupID,
 		})
 	}
 
@@ -189,6 +237,10 @@ func parseIAM(b []byte) (iAMConfig, error) {
 		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
 	}
 
+	if conf.AccessAccounts == nil {
+		conf.AccessAccounts = make(map[string]Account)
+	}
+
 	return conf, nil
 }
 

auth/iam_ldap.go

@@ -1,7 +1,22 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package auth
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/go-ldap/ldap/v3"
@@ -14,12 +29,15 @@ type LdapIAMService struct {
 	accessAtr  string
 	secretAtr  string
 	roleAtr    string
+	groupIdAtr string
+	userIdAtr  string
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
-	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
+		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
 	conn, err := ldap.DialURL(url)
@@ -38,15 +56,19 @@ func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objCl
 		accessAtr:  accAtr,
 		secretAtr:  secAtr,
 		roleAtr:    roleAtr,
+		userIdAtr:  userIdAtr,
+		groupIdAtr: groupIdAtr,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
-	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
 	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
+	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
+	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -65,7 +87,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
 		nil,
 	)
 
@@ -74,14 +96,48 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		return Account{}, err
 	}
 
+	if len(result.Entries) == 0 {
+		return Account{}, ErrNoSuchUser
+	}
+
 	entry := result.Entries[0]
+	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
+	}
+	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
+	}
 	return Account{
-		Access: entry.GetAttributeValue(ld.accessAtr),
-		Secret: entry.GetAttributeValue(ld.secretAtr),
-		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
+		Access:  entry.GetAttributeValue(ld.accessAtr),
+		Secret:  entry.GetAttributeValue(ld.secretAtr),
+		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
+		GroupID: groupId,
+		UserID:  userId,
 	}, nil
 }
 
+func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+	if props.Secret != nil {
+		req.Replace(ld.secretAtr, []string{*props.Secret})
+	}
+	if props.GroupID != nil {
+		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
+	}
+	if props.UserID != nil {
+		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
+	}
+
+	err := ld.conn.Modify(req)
+	//TODO: Handle non existing user case
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (ld *LdapIAMService) DeleteUserAccount(access string) error {
 	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
 
@@ -106,7 +162,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(&%v)", searchFilter),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
 		nil,
 	)
 
@@ -117,10 +173,20 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 
 	result := []Account{}
 	for _, el := range resp.Entries {
+		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
+		}
+		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
+		}
 		result = append(result, Account{
-			Access: el.GetAttributeValue(ld.accessAtr),
-			Secret: el.GetAttributeValue(ld.secretAtr),
-			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
+			Access:  el.GetAttributeValue(ld.accessAtr),
+			Secret:  el.GetAttributeValue(ld.secretAtr),
+			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
+			GroupID: groupId,
+			UserID:  userId,
 		})
 	}
 

auth/iam_s3_object.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"sort"
+	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -41,6 +42,14 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+
 	access        string
 	secret        string
 	region        string
@@ -85,11 +94,21 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 		return nil, fmt.Errorf("init s3 IAM: %v", err)
 	}
 
+	if endpoint != "" {
+		i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &endpoint
+		})
+		return i, nil
+	}
+
 	i.client = s3.NewFromConfig(cfg)
 	return i, nil
 }
 
 func (s *IAMServiceS3) CreateAccount(account Account) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -97,7 +116,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 
 	_, ok := conf.AccessAccounts[account.Access]
 	if ok {
-		return fmt.Errorf("account already exists")
+		return ErrUserExists
 	}
 	conf.AccessAccounts[account.Access] = account
 
@@ -105,6 +124,9 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 }
 
 func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return Account{}, err
@@ -118,7 +140,30 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	acc, ok := conf.AccessAccounts[access]
+	if !ok {
+		return ErrNoSuchUser
+	}
+
+	updateAcc(&acc, props)
+	conf.AccessAccounts[access] = acc
+
+	return s.storeAccts(conf)
+}
+
 func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -134,6 +179,9 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
 }
 
 func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return nil, err
@@ -148,28 +196,17 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
 	var accs []Account
 	for _, k := range keys {
 		accs = append(accs, Account{
-			Access:    k,
-			Secret:    conf.AccessAccounts[k].Secret,
-			Role:      conf.AccessAccounts[k].Role,
-			UserID:    conf.AccessAccounts[k].UserID,
-			GroupID:   conf.AccessAccounts[k].GroupID,
-			ProjectID: conf.AccessAccounts[k].ProjectID,
+			Access:  k,
+			Secret:  conf.AccessAccounts[k].Secret,
+			Role:    conf.AccessAccounts[k].Role,
+			UserID:  conf.AccessAccounts[k].UserID,
+			GroupID: conf.AccessAccounts[k].GroupID,
 		})
 	}
 
 	return accs, nil
 }
 
-// ResolveEndpoint is used for on prem or non-aws endpoints
-func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
-	return aws.Endpoint{
-		PartitionID:       "aws",
-		URL:               s.endpoint,
-		SigningRegion:     s.region,
-		HostnameImmutable: true,
-	}, nil
-}
-
 func (s *IAMServiceS3) Shutdown() error {
 	return nil
 }
@@ -188,11 +225,6 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
 		config.WithHTTPClient(client),
 	}
 
-	if s.endpoint != "" {
-		opts = append(opts,
-			config.WithEndpointResolverWithOptions(s))
-	}
-
 	if s.debug {
 		opts = append(opts,
 			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
@@ -213,12 +245,12 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
 		// init empty accounts stuct and return that
 		var nsk *types.NoSuchKey
 		if errors.As(err, &nsk) {
-			return iAMConfig{}, nil
+			return iAMConfig{AccessAccounts: map[string]Account{}}, nil
 		}
 		var apiErr smithy.APIError
 		if errors.As(err, &apiErr) {
 			if apiErr.ErrorCode() == "NotFound" {
-				return iAMConfig{}, nil
+				return iAMConfig{AccessAccounts: map[string]Account{}}, nil
 			}
 		}
 

auth/iam_single.go

@@ -35,6 +35,11 @@ func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
 	return Account{}, ErrNoSuchUser
 }
 
+// UpdateUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
+	return ErrNotSupported
+}
+
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
 	return ErrNotSupported

auth/iam_vault.go

@@ -0,0 +1,248 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	vault "github.com/hashicorp/vault-client-go"
+	"github.com/hashicorp/vault-client-go/schema"
+)
+
+type VaultIAMService struct {
+	client            *vault.Client
+	reqOpts           []vault.RequestOption
+	secretStoragePath string
+}
+
+var _ IAMService = &VaultIAMService{}
+
+func NewVaultIAMService(endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
+	opts := []vault.ClientOption{
+		vault.WithAddress(endpoint),
+		// set request timeout to 10 secs
+		vault.WithRequestTimeout(10 * time.Second),
+	}
+	if serverCert != "" {
+		tls := vault.TLSConfiguration{}
+
+		tls.ServerCertificate.FromBytes = []byte(serverCert)
+		if clientCert != "" {
+			if clientCertKey == "" {
+				return nil, fmt.Errorf("client certificate and client certificate should both be specified")
+			}
+
+			tls.ClientCertificate.FromBytes = []byte(clientCert)
+			tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
+		}
+
+		opts = append(opts, vault.WithTLS(tls))
+	}
+
+	client, err := vault.New(opts...)
+	if err != nil {
+		return nil, fmt.Errorf("init vault client: %w", err)
+	}
+
+	reqOpts := []vault.RequestOption{}
+	// if mount path is not specified, it defaults to "approle"
+	if mountPath != "" {
+		reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
+	}
+
+	// Authentication
+	switch {
+	case rootToken != "":
+		err := client.SetToken(rootToken)
+		if err != nil {
+			return nil, fmt.Errorf("root token authentication failure: %w", err)
+		}
+	case roleID != "":
+		if roleSecret == "" {
+			return nil, fmt.Errorf("role id and role secret must both be specified")
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
+			RoleId:   roleID,
+			SecretId: roleSecret,
+		}, reqOpts...)
+		cancel()
+		if err != nil {
+			return nil, fmt.Errorf("approle authentication failure: %w", err)
+		}
+
+		if err := client.SetToken(resp.Auth.ClientToken); err != nil {
+			return nil, fmt.Errorf("approle authentication set token failure: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
+	}
+
+	return &VaultIAMService{
+		client:            client,
+		reqOpts:           reqOpts,
+		secretStoragePath: secretStoragePath,
+	}, nil
+}
+
+func (vt *VaultIAMService) CreateAccount(account Account) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
+		Data: map[string]any{
+			account.Access: account,
+		},
+		Options: map[string]interface{}{
+			"cas": 0,
+		},
+	}, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		if strings.Contains(err.Error(), "check-and-set") {
+			return ErrUserExists
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acc, err := parseVaultUserAccount(resp.Data.Data, access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	return acc, nil
+}
+
+func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	//TODO: We need something like a transaction here ?
+	acc, err := vt.GetUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	updateAcc(&acc, props)
+
+	err = vt.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	err = vt.CreateAccount(acc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (vt *VaultIAMService) DeleteUserAccount(access string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		if vault.IsErrorStatus(err, 404) {
+			return []Account{}, nil
+		}
+		return nil, err
+	}
+
+	accs := []Account{}
+
+	for _, acss := range resp.Data.Keys {
+		acc, err := vt.GetUserAccount(acss)
+		if err != nil {
+			return nil, err
+		}
+		accs = append(accs, acc)
+	}
+
+	return accs, nil
+}
+
+// the client doesn't have explicit shutdown, as it uses http.Client
+func (vt *VaultIAMService) Shutdown() error {
+	return nil
+}
+
+var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
+
+func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
+	usrAcc, ok := data[access].(map[string]interface{})
+	if !ok {
+		return acc, errInvalidUser
+	}
+
+	acss, ok := usrAcc["access"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	secret, ok := usrAcc["secret"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	role, ok := usrAcc["role"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userIdJson, ok := usrAcc["userID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userId, err := userIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+	groupIdJson, ok := usrAcc["groupID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	groupId, err := groupIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+
+	return Account{
+		Access:  acss,
+		Secret:  secret,
+		Role:    Role(role),
+		UserID:  int(userId),
+		GroupID: int(groupId),
+	}, nil
+}

auth/object_lock.go

@@ -33,15 +33,14 @@ type BucketLockConfig struct {
 	CreatedAt        *time.Time
 }
 
-type ObjectLockConfig struct {
-	LegalHoldEnabled bool
-	Retention        *types.ObjectLockRetention
-}
-
 func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
 	var lockConfig types.ObjectLockConfiguration
 	if err := xml.Unmarshal(input, &lockConfig); err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
 	}
 
 	config := BucketLockConfig{
@@ -50,8 +49,19 @@ func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
 
 	if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
 		retention := lockConfig.Rule.DefaultRetention
+
+		if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
+			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+		}
 		if retention.Years != nil && retention.Days != nil {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+		}
+
+		if retention.Days != nil && *retention.Days <= 0 {
+			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
+		}
+		if retention.Years != nil && *retention.Years <= 0 {
+			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
 		}
 
 		config.DefaultRetention = retention
@@ -94,7 +104,7 @@ func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
 	case types.ObjectLockRetentionModeCompliance:
 	case types.ObjectLockRetentionModeGovernance:
 	default:
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
 	}
 
 	return json.Marshal(retention)
@@ -125,7 +135,7 @@ func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
 	}
 }
 
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, isAdminOrRoot bool, be backend.Backend) error {
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, bypass bool, be backend.Backend) error {
 	data, err := be.GetObjectLockConfiguration(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -144,13 +154,26 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 		return nil
 	}
 
-	objExists := true
+	checkDefaultRetention := false
+
+	if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
+		expirationDate := *bucketLockConfig.CreatedAt
+		if bucketLockConfig.DefaultRetention.Days != nil {
+			expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
+		}
+		if bucketLockConfig.DefaultRetention.Years != nil {
+			expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
+		}
+
+		if expirationDate.After(time.Now()) {
+			checkDefaultRetention = true
+		}
+	}
 
 	for _, obj := range objects {
-		var checkRetention bool = true
+		checkRetention := true
 		retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
-			objExists = false
 			continue
 		}
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
@@ -170,15 +193,17 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 				if retention.RetainUntilDate.After(time.Now()) {
 					switch retention.Mode {
 					case types.ObjectLockRetentionModeGovernance:
-						if !isAdminOrRoot {
+						if !bypass {
+							return s3err.GetAPIError(s3err.ErrObjectLocked)
+						} else {
 							policy, err := be.GetBucketPolicy(ctx, bucket)
+							if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+								return s3err.GetAPIError(s3err.ErrObjectLocked)
+							}
 							if err != nil {
 								return err
 							}
-							if len(policy) == 0 {
-								return s3err.GetAPIError(s3err.ErrObjectLocked)
-							}
-							err = verifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
+							err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
 							if err != nil {
 								return s3err.GetAPIError(s3err.ErrObjectLocked)
 							}
@@ -190,37 +215,35 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 			}
 		}
 
+		checkLegalHold := true
+
 		status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
-		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
-			continue
-		}
 		if err != nil {
-			return err
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
+				checkLegalHold = false
+			} else {
+				return err
+			}
 		}
 
-		if *status && !isAdminOrRoot {
+		if checkLegalHold && *status {
 			return s3err.GetAPIError(s3err.ErrObjectLocked)
 		}
-	}
 
-	if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil && objExists {
-		expirationDate := *bucketLockConfig.CreatedAt
-		if bucketLockConfig.DefaultRetention.Days != nil {
-			expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
-		}
-		if bucketLockConfig.DefaultRetention.Years != nil {
-			expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
-		}
-
-		if expirationDate.After(time.Now()) {
+		if checkDefaultRetention {
 			switch bucketLockConfig.DefaultRetention.Mode {
 			case types.ObjectLockRetentionModeGovernance:
-				if !isAdminOrRoot {
+				if !bypass {
+					return s3err.GetAPIError(s3err.ErrObjectLocked)
+				} else {
 					policy, err := be.GetBucketPolicy(ctx, bucket)
+					if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+						return s3err.GetAPIError(s3err.ErrObjectLocked)
+					}
 					if err != nil {
 						return err
 					}
-					err = verifyBucketPolicy(policy, userAccess, bucket, "", BypassGovernanceRetentionAction)
+					err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
 					if err != nil {
 						return s3err.GetAPIError(s3err.ErrObjectLocked)
 					}

aws/signer/v4/functional_test.go

@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 	}
 }
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 	}
 }

backend/azure/azure.go

@@ -20,10 +20,12 @@ import (
 	"encoding/base64"
 	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -34,6 +36,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/auth"
@@ -48,10 +51,14 @@ import (
 type key string
 
 const (
-	keyAclCapital key = "Acl"
-	keyAclLower   key = "acl"
-	keyTags       key = "Tags"
-	keyPolicy     key = "Policy"
+	keyAclCapital   key = "Acl"
+	keyAclLower     key = "acl"
+	keyOwnership    key = "Ownership"
+	keyTags         key = "Tags"
+	keyPolicy       key = "Policy"
+	keyBucketLock   key = "Bucket-Lock"
+	keyObjRetention key = "Object_retention"
+	keyObjLegalHold key = "Object_legal_hold"
 )
 
 type Azure struct {
@@ -121,13 +128,63 @@ func (az *Azure) String() string {
 func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
 	meta := map[string]*string{
 		string(keyAclCapital): backend.GetStringPtr(string(acl)),
+		string(keyOwnership):  backend.GetStringPtr(string(input.ObjectOwnership)),
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
+	if input.ObjectLockEnabledForBucket != nil && *input.ObjectLockEnabledForBucket {
+		now := time.Now()
+		defaultLock := auth.BucketLockConfig{
+			Enabled:   true,
+			CreatedAt: &now,
+		}
+
+		defaultLockParsed, err := json.Marshal(defaultLock)
+		if err != nil {
+			return fmt.Errorf("parse default bucket lock state: %w", err)
+		}
+
+		meta[string(keyBucketLock)] = backend.GetStringPtr(string(defaultLockParsed))
 	}
 	_, err := az.client.CreateContainer(ctx, *input.Bucket, &container.CreateOptions{Metadata: meta})
+	if errors.Is(s3err.GetAPIError(s3err.ErrBucketAlreadyExists), azureErrToS3Err(err)) {
+		client, err := az.getContainerClient(*input.Bucket)
+		if err != nil {
+			return err
+		}
+
+		props, err := client.GetProperties(ctx, nil)
+		if err != nil {
+			return azureErrToS3Err(err)
+		}
+
+		aclPtr, ok := props.Metadata[string(keyAclCapital)]
+		if !ok {
+			return fmt.Errorf("missing acl in the bucket")
+		}
+
+		var acl auth.ACL
+		if err := json.Unmarshal([]byte(*aclPtr), &acl); err != nil {
+			return fmt.Errorf("unmarshal bucket acl: %w", err)
+		}
+		if acl.Owner == acct.Access {
+			return s3err.GetAPIError(s3err.ErrBucketAlreadyOwnedByYou)
+		}
+	}
 	return azureErrToS3Err(err)
 }
 
 func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
-	pager := az.client.NewListContainersPager(nil)
+	pager := az.client.NewListContainersPager(
+		&service.ListContainersOptions{
+			Include: service.ListContainersInclude{
+				Metadata: true,
+			},
+		})
 
 	var buckets []s3response.ListAllMyBucketsEntry
 	var result s3response.ListAllMyBucketsResult
@@ -138,11 +195,26 @@ func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s
 			return result, azureErrToS3Err(err)
 		}
 		for _, v := range resp.ContainerItems {
-			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-				Name: *v.Name,
-				// TODO: using modification date here instead of creation, is that ok?
-				CreationDate: *v.Properties.LastModified,
-			})
+			if isAdmin {
+				buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+					Name: *v.Name,
+					// TODO: using modification date here instead of creation, is that ok?
+					CreationDate: *v.Properties.LastModified,
+				})
+			} else {
+				acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
+				if err != nil {
+					return result, err
+				}
+
+				if acl.Owner == owner {
+					buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+						Name: *v.Name,
+						// TODO: using modification date here instead of creation, is that ok?
+						CreationDate: *v.Properties.LastModified,
+					})
+				}
+			}
 		}
 	}
 
@@ -167,10 +239,81 @@ func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3
 }
 
 func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
-	_, err := az.client.DeleteContainer(ctx, *input.Bucket, nil)
+	pager := az.client.NewListBlobsFlatPager(*input.Bucket, nil)
+
+	pg, err := pager.NextPage(ctx)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	if len(pg.Segment.BlobItems) > 0 {
+		return s3err.GetAPIError(s3err.ErrBucketNotEmpty)
+	}
+	_, err = az.client.DeleteContainer(ctx, *input.Bucket, nil)
 	return azureErrToS3Err(err)
 }
 
+func (az *Azure) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+	resp.Metadata[string(keyOwnership)] = backend.GetStringPtr(string(ownership))
+
+	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: resp.Metadata})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return ownship, err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return ownship, azureErrToS3Err(err)
+	}
+
+	ownership, ok := resp.Metadata[string(keyOwnership)]
+	if !ok {
+		return ownship, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)
+	}
+
+	return types.ObjectOwnership(*ownership), nil
+}
+
+func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	delete(resp.Metadata, string(keyOwnership))
+
+	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: resp.Metadata})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
 func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
 	tags, err := parseTags(po.Tagging)
 	if err != nil {
@@ -185,6 +328,28 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string,
 		return "", azureErrToS3Err(err)
 	}
 
+	// Set object legal hold
+	if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
+		if err := az.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true); err != nil {
+			return "", err
+		}
+	}
+
+	// Set object retention
+	if po.ObjectLockMode != "" {
+		retention := types.ObjectLockRetention{
+			Mode:            types.ObjectLockRetentionMode(po.ObjectLockMode),
+			RetainUntilDate: po.ObjectLockRetainUntilDate,
+		}
+		retParsed, err := json.Marshal(retention)
+		if err != nil {
+			return "", fmt.Errorf("parse object lock retention: %w", err)
+		}
+		if err := az.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed); err != nil {
+			return "", err
+		}
+	}
+
 	return string(*uploadResp.ETag), nil
 }
 
@@ -231,7 +396,7 @@ func (az *Azure) GetBucketTagging(ctx context.Context, bucket string) (map[strin
 
 	tagsJson, ok := resp.Metadata[string(keyTags)]
 	if !ok {
-		return map[string]string{}, nil
+		return nil, s3err.GetAPIError(s3err.ErrBucketTaggingNotFound)
 	}
 
 	var tags map[string]string
@@ -246,10 +411,10 @@ func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error {
 	return az.PutBucketTagging(ctx, bucket, nil)
 }
 
-func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	var opts *azblob.DownloadStreamOptions
 	if *input.Range != "" {
-		offset, count, err := parseRange(*input.Range)
+		offset, count, err := backend.ParseRange(0, *input.Range)
 		if err != nil {
 			return nil, err
 		}
@@ -264,12 +429,6 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer
 	if err != nil {
 		return nil, azureErrToS3Err(err)
 	}
-	defer blobDownloadResponse.Body.Close()
-
-	_, err = io.Copy(writer, blobDownloadResponse.Body)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
 
 	var tagcount int32
 	if blobDownloadResponse.TagCount != nil {
@@ -286,10 +445,42 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer
 		Metadata:        parseAzMetadata(blobDownloadResponse.Metadata),
 		TagCount:        &tagcount,
 		ContentRange:    blobDownloadResponse.ContentRange,
+		Body:            blobDownloadResponse.Body,
 	}, nil
 }
 
 func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	if input.PartNumber != nil {
+		client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
+		if err != nil {
+			return nil, err
+		}
+
+		res, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+
+		partsCount := int32(len(res.UncommittedBlocks))
+
+		for _, block := range res.UncommittedBlocks {
+			partNumber, err := decodeBlockId(*block.Name)
+			if err != nil {
+				return nil, err
+			}
+
+			if partNumber == int(*input.PartNumber) {
+				return &s3.HeadObjectOutput{
+					ContentLength: block.Size,
+					ETag:          block.Name,
+					PartsCount:    &partsCount,
+				}, nil
+			}
+		}
+
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
 	client, err := az.getBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
 		return nil, err
@@ -300,7 +491,7 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 		return nil, azureErrToS3Err(err)
 	}
 
-	return &s3.HeadObjectOutput{
+	result := &s3.HeadObjectOutput{
 		AcceptRanges:       resp.AcceptRanges,
 		ContentLength:      resp.ContentLength,
 		ContentType:        resp.ContentType,
@@ -311,6 +502,81 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 		LastModified:       resp.LastModified,
 		Metadata:           parseAzMetadata(resp.Metadata),
 		Expires:            resp.ExpiresOn,
+	}
+
+	status, ok := resp.Metadata[string(keyObjLegalHold)]
+	if ok {
+		if *status == "1" {
+			result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
+		} else {
+			result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
+		}
+	}
+
+	retention, ok := resp.Metadata[string(keyObjRetention)]
+	if ok {
+		var config types.ObjectLockRetention
+		if err := json.Unmarshal([]byte(*retention), &config); err == nil {
+			result.ObjectLockMode = types.ObjectLockMode(config.Mode)
+			result.ObjectLockRetainUntilDate = config.RetainUntilDate
+		}
+	}
+
+	return result, nil
+}
+
+func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+	data, err := az.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: input.Bucket,
+		Key:    input.Key,
+	})
+	if err == nil {
+		return s3response.GetObjectAttributesResult{
+			ETag:         data.ETag,
+			LastModified: data.LastModified,
+			ObjectSize:   data.ContentLength,
+			StorageClass: &data.StorageClass,
+			VersionId:    data.VersionId,
+		}, nil
+	}
+	if !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+		return s3response.GetObjectAttributesResult{}, err
+	}
+
+	resp, err := az.ListParts(ctx, &s3.ListPartsInput{
+		Bucket:           input.Bucket,
+		Key:              input.Key,
+		PartNumberMarker: input.PartNumberMarker,
+		MaxParts:         input.MaxParts,
+	})
+	if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)) {
+		return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return s3response.GetObjectAttributesResult{}, err
+	}
+
+	parts := []types.ObjectPart{}
+
+	for _, p := range resp.Parts {
+		partNumber := int32(p.PartNumber)
+		size := p.Size
+
+		parts = append(parts, types.ObjectPart{
+			Size:       &size,
+			PartNumber: &partNumber,
+		})
+	}
+
+	//TODO: handle PartsCount prop
+	return s3response.GetObjectAttributesResult{
+		ObjectParts: &s3response.ObjectParts{
+			IsTruncated:          resp.IsTruncated,
+			MaxParts:             resp.MaxParts,
+			PartNumberMarker:     resp.PartNumberMarker,
+			NextPartNumberMarker: resp.NextPartNumberMarker,
+			Parts:                parts,
+		},
 	}, nil
 }
 
@@ -369,8 +635,14 @@ Pager:
 }
 
 func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	marker := ""
+	if *input.ContinuationToken > *input.StartAfter {
+		marker = *input.ContinuationToken
+	} else {
+		marker = *input.StartAfter
+	}
 	pager := az.client.NewListBlobsFlatPager(*input.Bucket, &azblob.ListBlobsFlatOptions{
-		Marker:     input.ContinuationToken,
+		Marker:     &marker,
 		MaxResults: input.MaxKeys,
 		Prefix:     input.Prefix,
 	})
@@ -419,6 +691,7 @@ Pager:
 		NextContinuationToken: nextMarker,
 		Prefix:                input.Prefix,
 		IsTruncated:           &isTruncated,
+		Delimiter:             input.Delimiter,
 	}, nil
 }
 
@@ -635,20 +908,20 @@ func (az *Azure) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3res
 		if err != nil {
 			return s3response.ListPartsResult{}, err
 		}
-		if partNumberMarker != 0 && partNumberMarker < partNumber {
+		if partNumberMarker != 0 && partNumberMarker >= partNumber {
 			continue
 		}
-		if len(parts) >= int(maxParts) {
-			nextPartNumberMarker = partNumber
-			isTruncated = true
-			break
-		}
 		parts = append(parts, s3response.Part{
 			Size:         *el.Size,
 			ETag:         *el.Name,
 			PartNumber:   partNumber,
 			LastModified: time.Now().Format(backend.RFC3339TimeFormat),
 		})
+		if len(parts) >= int(maxParts) {
+			nextPartNumberMarker = partNumber
+			isTruncated = true
+			break
+		}
 	}
 	return s3response.ListPartsResult{
 		Bucket:               *input.Bucket,
@@ -736,9 +1009,37 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 		return nil, err
 	}
 	blockIds := []string{}
-	for _, el := range input.MultipartUpload.Parts {
-		blockIds = append(blockIds, *el.ETag)
+
+	blockList, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
 	}
+
+	if len(blockList.UncommittedBlocks) != len(input.MultipartUpload.Parts) {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+	}
+
+	slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int {
+		ptNumber, _ := decodeBlockId(*a.Name)
+		nextPtNumber, _ := decodeBlockId(*b.Name)
+		return ptNumber - nextPtNumber
+	})
+
+	for i, block := range blockList.UncommittedBlocks {
+		ptNumber, err := decodeBlockId(*block.Name)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		if *input.MultipartUpload.Parts[i].ETag != *block.Name {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		blockIds = append(blockIds, *block.Name)
+	}
+
 	resp, err := client.CommitBlockList(ctx, blockIds, nil)
 	if err != nil {
 		return nil, parseMpError(err)
@@ -829,7 +1130,7 @@ func (az *Azure) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, er
 
 	policyPtr, ok := props.Metadata[string(keyPolicy)]
 	if !ok {
-		return []byte{}, nil
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)
 	}
 
 	policy, err := base64.StdEncoding.DecodeString(*policyPtr)
@@ -844,6 +1145,229 @@ func (az *Azure) DeleteBucketPolicy(ctx context.Context, bucket string) error {
 	return az.PutBucketPolicy(ctx, bucket, nil)
 }
 
+func (az *Azure) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+
+	props, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	cfg, exists := props.Metadata[string(keyBucketLock)]
+	if !exists {
+		return s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotAllowed)
+	}
+
+	var bucketLockCfg auth.BucketLockConfig
+	if err := json.Unmarshal([]byte(*cfg), &bucketLockCfg); err != nil {
+		return fmt.Errorf("unmarshal object lock config: %w", err)
+	}
+
+	if !bucketLockCfg.Enabled {
+		return s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotAllowed)
+	}
+
+	props.Metadata[string(keyBucketLock)] = backend.GetStringPtr(string(config))
+
+	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{
+		Metadata: props.Metadata,
+	})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return nil, err
+	}
+	props, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	config, ok := props.Metadata[string(keyBucketLock)]
+	if !ok {
+		return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
+	}
+
+	return []byte(*config), nil
+}
+
+func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
+	contClient, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+	contProps, err := contClient.GetProperties(ctx, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	contCfg, ok := contProps.Metadata[string(keyBucketLock)]
+	if !ok {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	var bucketLockConfig auth.BucketLockConfig
+	if err := json.Unmarshal([]byte(*contCfg), &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse bucket lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	blobClient, err := az.getBlobClient(bucket, object)
+	if err != nil {
+		return err
+	}
+
+	blobProps, err := blobClient.GetProperties(ctx, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	meta := blobProps.Metadata
+	if meta == nil {
+		meta = map[string]*string{
+			string(keyObjRetention): backend.GetStringPtr(string(retention)),
+		}
+	} else {
+		objLockCfg, ok := meta[string(keyObjRetention)]
+		if !ok {
+			meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
+		} else {
+			var lockCfg types.ObjectLockRetention
+			if err := json.Unmarshal([]byte(*objLockCfg), &lockCfg); err != nil {
+				return fmt.Errorf("unmarshal object lock config: %w", err)
+			}
+
+			switch lockCfg.Mode {
+			// Compliance mode can't be overridden
+			case types.ObjectLockRetentionModeCompliance:
+				return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+			// To override governance mode user should have "s3:BypassGovernanceRetention" permission
+			case types.ObjectLockRetentionModeGovernance:
+				if !bypass {
+					return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+				}
+			}
+
+			meta[string(keyObjRetention)] = backend.GetStringPtr(string(retention))
+		}
+	}
+
+	_, err = blobClient.SetMetadata(ctx, meta, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
+	client, err := az.getBlobClient(bucket, object)
+	if err != nil {
+		return nil, err
+	}
+	props, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	retentionPtr, ok := props.Metadata[string(keyObjRetention)]
+	if !ok {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
+	}
+
+	return []byte(*retentionPtr), nil
+}
+
+func (az *Azure) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
+	contClient, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+	contProps, err := contClient.GetProperties(ctx, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	contCfg, ok := contProps.Metadata[string(keyBucketLock)]
+	if !ok {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	var bucketLockConfig auth.BucketLockConfig
+	if err := json.Unmarshal([]byte(*contCfg), &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse bucket lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
+	}
+
+	blobClient, err := az.getBlobClient(bucket, object)
+	if err != nil {
+		return err
+	}
+
+	blobProps, err := blobClient.GetProperties(ctx, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	var statusData string
+	if status {
+		statusData = "1"
+	} else {
+		statusData = "0"
+	}
+
+	meta := blobProps.Metadata
+	if meta == nil {
+		meta = map[string]*string{
+			string(keyObjLegalHold): &statusData,
+		}
+	} else {
+		meta[string(keyObjLegalHold)] = &statusData
+	}
+
+	_, err = blobClient.SetMetadata(ctx, meta, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
+	client, err := az.getBlobClient(bucket, object)
+	if err != nil {
+		return nil, err
+	}
+	props, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	retentionPtr, ok := props.Metadata[string(keyObjLegalHold)]
+	if !ok {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
+	}
+
+	status := *retentionPtr == "1"
+
+	return &status, nil
+}
+
 func (az *Azure) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
 	client, err := az.getContainerClient(bucket)
 	if err != nil {
@@ -1029,39 +1553,6 @@ func decodeBlockId(blockID string) (int, error) {
 	return int(binary.LittleEndian.Uint32(slice)), nil
 }
 
-func parseRange(rg string) (offset, count int64, err error) {
-	rangeKv := strings.Split(rg, "=")
-
-	if len(rangeKv) < 2 {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 1 || len(bRange) > 2 {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	offset, err = strconv.ParseInt(bRange[0], 10, 64)
-	if err != nil {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	if len(bRange) == 1 || bRange[1] == "" {
-		return offset, count, nil
-	}
-
-	count, err = strconv.ParseInt(bRange[1], 10, 64)
-	if err != nil {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	if count < offset {
-		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
-	return offset, count - offset + 1, nil
-}
-
 func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 	aclPtr, ok := meta[string(key)]
 	if !ok {

backend/backend.go

@@ -18,9 +18,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 	"github.com/versity/versitygw/s3select"
@@ -43,6 +43,9 @@ type Backend interface {
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
+	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
+	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
+	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
 
 	// multipart operations
 	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
@@ -56,9 +59,9 @@ type Backend interface {
 	// standard object operations
 	PutObject(context.Context, *s3.PutObjectInput) (string, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
 	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
 	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
@@ -84,7 +87,7 @@ type Backend interface {
 	// object lock operations
 	PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
 	GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
-	PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error
+	PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
 	GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
 	PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
 	GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
@@ -138,6 +141,15 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
 func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
 func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -167,14 +179,14 @@ func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+	return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -243,7 +255,7 @@ func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket s
 func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error {
+func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {

backend/common.go

@@ -18,7 +18,9 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"io/fs"
+	"io"
+	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -61,9 +63,9 @@ var (
 
 // ParseRange parses input range header and returns startoffset, length, and
 // error. If no endoffset specified, then length is set to -1.
-func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
+func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, fi.Size(), nil
+		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
@@ -92,13 +94,21 @@ func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 		return 0, 0, errInvalidRange
 	}
 
-	if endOffset < startOffset {
+	if endOffset <= startOffset {
 		return 0, 0, errInvalidRange
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
+func CreateExceedingRangeErr(objSize int64) s3err.APIError {
+	return s3err.APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
@@ -120,3 +130,16 @@ func md5String(data []byte) string {
 	sum := md5.Sum(data)
 	return hex.EncodeToString(sum[:])
 }
+
+type FileSectionReadCloser struct {
+	R io.Reader
+	F *os.File
+}
+
+func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
+	return f.R.Read(p)
+}
+
+func (f *FileSectionReadCloser) Close() error {
+	return f.F.Close()
+}

backend/meta/meta.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package meta
 
 // MetadataStorer defines the interface for managing metadata.

backend/meta/xattr.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package meta
 
 import (

backend/mkdir.go

@@ -1,6 +1,7 @@
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+// Copyright 2024 Versity Software
 
 // MkdirAll borrowed from stdlib to add ability to set ownership
 // as directories are created
@@ -29,7 +30,7 @@ var (
 // Any newly created directory is set to provided uid/gid ownership.
 // If path is already a directory, MkdirAll does nothing
 // and returns nil.
-// Any directoy created will be set to provided uid/gid ownership
+// Any directory created will be set to provided uid/gid ownership
 // if doChown is true.
 func MkdirAll(path string, uid, gid int, doChown bool) error {
 	// Fast path: if we can tell whether path is a directory or file, stop with success or error.

backend/posix/posix.go

@@ -24,12 +24,14 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"math"
 	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -73,11 +75,15 @@ const (
 	contentEncHdr       = "content-encoding"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
 	aclkey              = "acl"
+	ownershipkey        = "ownership"
 	etagkey             = "etag"
 	policykey           = "policy"
 	bucketLockKey       = "bucket-lock"
 	objectRetentionKey  = "object-retention"
 	objectLegalHoldKey  = "object-legal-hold"
+
+	doFalloc   = true
+	skipFalloc = false
 )
 
 type PosixOpts struct {
@@ -212,6 +218,18 @@ func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, a
 
 	err := os.Mkdir(bucket, defaultDirPerm)
 	if err != nil && os.IsExist(err) {
+		aclJSON, err := p.meta.RetrieveAttribute(bucket, "", aclkey)
+		if err != nil {
+			return fmt.Errorf("get bucket acl: %w", err)
+		}
+		var acl auth.ACL
+		if err := json.Unmarshal(aclJSON, &acl); err != nil {
+			return fmt.Errorf("unmarshal acl: %w", err)
+		}
+
+		if acl.Owner == acct.Access {
+			return s3err.GetAPIError(s3err.ErrBucketAlreadyOwnedByYou)
+		}
 		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
 	}
 	if err != nil {
@@ -228,6 +246,26 @@ func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, a
 	if err := p.meta.StoreAttribute(bucket, "", aclkey, acl); err != nil {
 		return fmt.Errorf("set acl: %w", err)
 	}
+	if err := p.meta.StoreAttribute(bucket, "", ownershipkey, []byte(input.ObjectOwnership)); err != nil {
+		return fmt.Errorf("set ownership: %w", err)
+	}
+
+	if input.ObjectLockEnabledForBucket != nil && *input.ObjectLockEnabledForBucket {
+		now := time.Now()
+		defaultLock := auth.BucketLockConfig{
+			Enabled:   true,
+			CreatedAt: &now,
+		}
+
+		defaultLockParsed, err := json.Marshal(defaultLock)
+		if err != nil {
+			return fmt.Errorf("parse default bucket lock state: %w", err)
+		}
+
+		if err := p.meta.StoreAttribute(bucket, "", bucketLockKey, defaultLockParsed); err != nil {
+			return fmt.Errorf("set default bucket lock: %w", err)
+		}
+	}
 
 	return nil
 }
@@ -270,7 +308,62 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
 	return nil
 }
 
-func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+func (p *Posix) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if err := p.meta.StoreAttribute(bucket, "", ownershipkey, []byte(ownership)); err != nil {
+		return fmt.Errorf("set ownership: %w", err)
+	}
+
+	return nil
+}
+func (p *Posix) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return ownship, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return ownship, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	ownership, err := p.meta.RetrieveAttribute(bucket, "", ownershipkey)
+	if errors.Is(err, meta.ErrNoSuchKey) {
+		return ownship, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)
+	}
+	if err != nil {
+		return ownship, fmt.Errorf("get bucket ownership status: %w", err)
+	}
+
+	return types.ObjectOwnership(ownership), nil
+}
+func (p *Posix) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if err := p.meta.DeleteAttribute(bucket, "", ownershipkey); err != nil {
+		if errors.Is(err, meta.ErrNoSuchKey) {
+			return nil
+		}
+
+		return fmt.Errorf("delete ownership: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	if mpu.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -295,6 +388,23 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
 		return nil, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData)
 	}
 
+	// parse object tags
+	tagsStr := getString(mpu.Tagging)
+	tags := make(map[string]string)
+	if tagsStr != "" {
+		tagParts := strings.Split(tagsStr, "&")
+		for _, prt := range tagParts {
+			p := strings.Split(prt, "=")
+			if len(p) != 2 {
+				return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+			}
+			if len(p[0]) > 128 || len(p[1]) > 256 {
+				return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+			}
+			tags[p[0]] = p[1]
+		}
+	}
+
 	// generate random uuid for upload id
 	uploadID := uuid.New().String()
 	// hash object name for multipart container
@@ -322,10 +432,10 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
 		return nil, fmt.Errorf("set name attr for upload: %w", err)
 	}
 
-	// set user attrs
+	// set user metadata
 	for k, v := range mpu.Metadata {
 		err := p.meta.StoreAttribute(bucket, filepath.Join(objdir, uploadID),
-			k, []byte(v))
+			fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
 			// cleanup object if returning error
 			os.RemoveAll(filepath.Join(tmppath, uploadID))
@@ -334,6 +444,60 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
 		}
 	}
 
+	// set object tagging
+	if tagsStr != "" {
+		err := p.PutObjectTagging(ctx, bucket, filepath.Join(objdir, uploadID), tags)
+		if err != nil {
+			// cleanup object if returning error
+			os.RemoveAll(filepath.Join(tmppath, uploadID))
+			os.Remove(tmppath)
+			return nil, err
+		}
+	}
+
+	// set content-type
+	if *mpu.ContentType != "" {
+		err := p.meta.StoreAttribute(bucket, filepath.Join(objdir, uploadID),
+			contentTypeHdr, []byte(*mpu.ContentType))
+		if err != nil {
+			// cleanup object if returning error
+			os.RemoveAll(filepath.Join(tmppath, uploadID))
+			os.Remove(tmppath)
+			return nil, fmt.Errorf("set content-type: %w", err)
+		}
+	}
+
+	// set object legal hold
+	if mpu.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
+		if err := p.PutObjectLegalHold(ctx, bucket, filepath.Join(objdir, uploadID), "", true); err != nil {
+			// cleanup object if returning error
+			os.RemoveAll(filepath.Join(tmppath, uploadID))
+			os.Remove(tmppath)
+			return nil, err
+		}
+	}
+
+	// Set object retention
+	if mpu.ObjectLockMode != "" {
+		retention := types.ObjectLockRetention{
+			Mode:            types.ObjectLockRetentionMode(mpu.ObjectLockMode),
+			RetainUntilDate: mpu.ObjectLockRetainUntilDate,
+		}
+		retParsed, err := json.Marshal(retention)
+		if err != nil {
+			// cleanup object if returning error
+			os.RemoveAll(filepath.Join(tmppath, uploadID))
+			os.Remove(tmppath)
+			return nil, fmt.Errorf("parse object lock retention: %w", err)
+		}
+		if err := p.PutObjectRetention(ctx, bucket, filepath.Join(objdir, uploadID), "", true, retParsed); err != nil {
+			// cleanup object if returning error
+			os.RemoveAll(filepath.Join(tmppath, uploadID))
+			os.Remove(tmppath)
+			return nil, err
+		}
+	}
+
 	return &s3.CreateMultipartUploadOutput{
 		Bucket:   &bucket,
 		Key:      &object,
@@ -404,6 +568,10 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	partsize := int64(0)
 	var totalsize int64
 	for i, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
 		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
 		fullPartPath := filepath.Join(bucket, partObjPath)
 		fi, err := os.Lstat(fullPartPath)
@@ -425,13 +593,13 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 		if err != nil {
 			etag = ""
 		}
-		if etag != *parts[i].ETag {
+		if parts[i].ETag == nil || etag != *parts[i].ETag {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 	}
 
 	f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object,
-		totalsize, acct)
+		totalsize, acct, skipFalloc)
 	if err != nil {
 		if errors.Is(err, syscall.EDQUOT) {
 			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
@@ -441,13 +609,17 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	defer f.cleanup()
 
 	for _, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
 		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
 		fullPartPath := filepath.Join(bucket, partObjPath)
 		pf, err := os.Open(fullPartPath)
 		if err != nil {
 			return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
 		}
-		_, err = io.Copy(f, pf)
+		_, err = io.Copy(f.File(), pf)
 		pf.Close()
 		if err != nil {
 			if errors.Is(err, syscall.EDQUOT) {
@@ -459,7 +631,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 
 	userMetaData := make(map[string]string)
 	upiddir := filepath.Join(objdir, uploadID)
-	p.loadUserMetaData(bucket, objdir, userMetaData)
+	cType, _ := p.loadUserMetaData(bucket, upiddir, userMetaData)
 
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
@@ -476,7 +648,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	}
 
 	for k, v := range userMetaData {
-		err = p.meta.StoreAttribute(bucket, object, k, []byte(v))
+		err = p.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
 			// cleanup object if returning error
 			os.Remove(objname)
@@ -484,6 +656,54 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 		}
 	}
 
+	// load and set tagging
+	tagging, err := p.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
+	if err == nil {
+		if err := p.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object tagging: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object tagging: %w", err)
+	}
+
+	// set content-type
+	if cType != "" {
+		if err := p.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object content type: %w", err)
+		}
+	}
+
+	// load and set legal hold
+	lHold, err := p.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
+	if err == nil {
+		if err := p.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object legal hold: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object legal hold: %w", err)
+	}
+
+	// load and set retention
+	ret, err := p.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
+	if err == nil {
+		if err := p.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object retention: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object retention: %w", err)
+	}
+
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
@@ -495,7 +715,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	}
 
 	// cleanup tmp dirs
-	os.RemoveAll(upiddir)
+	os.RemoveAll(filepath.Join(bucket, objdir, uploadID))
 	// use Remove for objdir in case there are still other uploads
 	// for same object name outstanding, this will fail if there are
 	os.Remove(filepath.Join(bucket, objdir))
@@ -521,6 +741,18 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte,
 	return sum, nil
 }
 
+func (p *Posix) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
+	sum := sha256.Sum256([]byte(object))
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	entries, err := os.ReadDir(objdir)
+	if err != nil || len(entries) == 0 {
+		return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
+	return entries[0].Name(), sum, nil
+}
+
 // fll out the user metadata map with the metadata for the object
 // and return the content type and encoding
 func (p *Posix) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
@@ -828,7 +1060,11 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
 
 	var parts []s3response.Part
 	for _, e := range ents {
-		pn, _ := strconv.Atoi(e.Name())
+		pn, err := strconv.Atoi(e.Name())
+		if err != nil {
+			// file is not a valid part file
+			continue
+		}
 		if pn <= partNumberMarker {
 			continue
 		}
@@ -928,7 +1164,7 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (stri
 	partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part))
 
 	f, err := p.openTmpFile(filepath.Join(bucket, objdir),
-		bucket, partPath, length, acct)
+		bucket, partPath, length, acct, doFalloc)
 	if err != nil {
 		if errors.Is(err, syscall.EDQUOT) {
 			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
@@ -1022,7 +1258,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 		return s3response.CopyObjectResult{}, fmt.Errorf("stat object: %w", err)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi, *upi.CopySourceRange)
+	startOffset, length, err := backend.ParseRange(fi.Size(), *upi.CopySourceRange)
 	if err != nil {
 		return s3response.CopyObjectResult{}, err
 	}
@@ -1032,11 +1268,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 	}
 
 	if startOffset+length > fi.Size()+1 {
-		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRange)
+		return s3response.CopyObjectResult{}, backend.CreateExceedingRangeErr(fi.Size())
 	}
 
 	f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
-		*upi.Bucket, partPath, length, acct)
+		*upi.Bucket, partPath, length, acct, doFalloc)
 	if err != nil {
 		if errors.Is(err, syscall.EDQUOT) {
 			return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded)
@@ -1175,7 +1411,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 	}
 
 	f, err := p.openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
-		*po.Bucket, *po.Key, contentLength, acct)
+		*po.Bucket, *po.Key, contentLength, acct, doFalloc)
 	if err != nil {
 		if errors.Is(err, syscall.EDQUOT) {
 			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
@@ -1214,6 +1450,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 		}
 	}
 
+	// Set object tagging
 	if tagsStr != "" {
 		err := p.PutObjectTagging(ctx, *po.Bucket, *po.Key, tags)
 		if err != nil {
@@ -1221,6 +1458,28 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 		}
 	}
 
+	// Set object legal hold
+	if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
+		if err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true); err != nil {
+			return "", err
+		}
+	}
+
+	// Set object retention
+	if po.ObjectLockMode != "" {
+		retention := types.ObjectLockRetention{
+			Mode:            types.ObjectLockRetentionMode(po.ObjectLockMode),
+			RetainUntilDate: po.ObjectLockRetainUntilDate,
+		}
+		retParsed, err := json.Marshal(retention)
+		if err != nil {
+			return "", fmt.Errorf("parse object lock retention: %w", err)
+		}
+		if err := p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed); err != nil {
+			return "", err
+		}
+	}
+
 	dataSum := hash.Sum(nil)
 	etag := hex.EncodeToString(dataSum[:])
 	err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, []byte(etag))
@@ -1334,7 +1593,7 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 	}, nil
 }
 
-func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1365,7 +1624,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 	}
 
 	acceptRange := *input.Range
-	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
 	if err != nil {
 		return nil, err
 	}
@@ -1378,11 +1637,11 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 	}
 
 	if length == -1 {
-		length = objSize - startOffset + 1
+		length = objSize - startOffset
 	}
 
-	if startOffset+length > objSize+1 {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRange)
+	if startOffset+length > objSize {
+		length = objSize - startOffset
 	}
 
 	var contentRange string
@@ -1402,12 +1661,15 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 			etag = ""
 		}
 
+		var tagCount *int32
 		tags, err := p.getAttrTags(bucket, object)
-		if err != nil {
-			return nil, fmt.Errorf("get object tags: %w", err)
+		if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrBucketTaggingNotFound)) {
+			return nil, err
+		}
+		if tags != nil {
+			tgCount := int32(len(tags))
+			tagCount = &tgCount
 		}
-
-		tagCount := int32(len(tags))
 
 		return &s3.GetObjectOutput{
 			AcceptRanges:    &acceptRange,
@@ -1417,26 +1679,11 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 			ETag:            &etag,
 			LastModified:    backend.GetTimePtr(fi.ModTime()),
 			Metadata:        userMetaData,
-			TagCount:        &tagCount,
+			TagCount:        tagCount,
 			ContentRange:    &contentRange,
 		}, nil
 	}
 
-	f, err := os.Open(objPath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("open object: %w", err)
-	}
-	defer f.Close()
-
-	rdr := io.NewSectionReader(f, startOffset, length)
-	_, err = io.Copy(writer, rdr)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
-
 	userMetaData := make(map[string]string)
 
 	contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
@@ -1447,12 +1694,25 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		etag = ""
 	}
 
+	var tagCount *int32
 	tags, err := p.getAttrTags(bucket, object)
-	if err != nil {
-		return nil, fmt.Errorf("get object tags: %w", err)
+	if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrBucketTaggingNotFound)) {
+		return nil, err
+	}
+	if tags != nil {
+		tgCount := int32(len(tags))
+		tagCount = &tgCount
 	}
 
-	tagCount := int32(len(tags))
+	f, err := os.Open(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("open object: %w", err)
+	}
+
+	rdr := io.NewSectionReader(f, startOffset, length)
 
 	return &s3.GetObjectOutput{
 		AcceptRanges:    &acceptRange,
@@ -1462,12 +1722,13 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		ETag:            &etag,
 		LastModified:    backend.GetTimePtr(fi.ModTime()),
 		Metadata:        userMetaData,
-		TagCount:        &tagCount,
+		TagCount:        tagCount,
 		ContentRange:    &contentRange,
+		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
 	}, nil
 }
 
-func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1477,6 +1738,46 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
 	bucket := *input.Bucket
 	object := *input.Key
 
+	if input.PartNumber != nil {
+		uploadId, sum, err := p.retrieveUploadId(bucket, object)
+		if err != nil {
+			return nil, err
+		}
+
+		ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read parts: %w", err)
+		}
+
+		partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
+
+		part, err := os.Stat(filepath.Join(bucket, partPath))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat part: %w", err)
+		}
+
+		b, err := p.meta.RetrieveAttribute(bucket, partPath, etagkey)
+		etag := string(b)
+		if err != nil {
+			etag = ""
+		}
+		partsCount := int32(len(ents))
+		size := part.Size()
+
+		return &s3.HeadObjectOutput{
+			LastModified:  backend.GetTimePtr(part.ModTime()),
+			ETag:          &etag,
+			PartsCount:    &partsCount,
+			ContentLength: &size,
+		}, nil
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1497,6 +1798,11 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
 	userMetaData := make(map[string]string)
 	contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
 
+	if fi.IsDir() {
+		// this is the media type for directories in AWS and Nextcloud
+		contentType = "application/x-directory"
+	}
+
 	b, err := p.meta.RetrieveAttribute(bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
@@ -1505,13 +1811,103 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
 
 	size := fi.Size()
 
+	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	status, err := p.GetObjectLegalHold(ctx, bucket, object, "")
+	if err == nil {
+		if *status {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
+		} else {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
+		}
+	}
+
+	var objectLockMode types.ObjectLockMode
+	var objectLockRetainUntilDate *time.Time
+	retention, err := p.GetObjectRetention(ctx, bucket, object, "")
+	if err == nil {
+		var config types.ObjectLockRetention
+		if err := json.Unmarshal(retention, &config); err == nil {
+			objectLockMode = types.ObjectLockMode(config.Mode)
+			objectLockRetainUntilDate = config.RetainUntilDate
+		}
+	}
+
+	//TODO: the method must handle multipart upload case
+
 	return &s3.HeadObjectOutput{
-		ContentLength:   &size,
-		ContentType:     &contentType,
-		ContentEncoding: &contentEncoding,
-		ETag:            &etag,
-		LastModified:    backend.GetTimePtr(fi.ModTime()),
-		Metadata:        userMetaData,
+		ContentLength:             &size,
+		ContentType:               &contentType,
+		ContentEncoding:           &contentEncoding,
+		ETag:                      &etag,
+		LastModified:              backend.GetTimePtr(fi.ModTime()),
+		Metadata:                  userMetaData,
+		ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
+		ObjectLockMode:            objectLockMode,
+		ObjectLockRetainUntilDate: objectLockRetainUntilDate,
+	}, nil
+}
+
+func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+	data, err := p.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: input.Bucket,
+		Key:    input.Key,
+	})
+	if err == nil {
+		return s3response.GetObjectAttributesResult{
+			ETag:         data.ETag,
+			LastModified: data.LastModified,
+			ObjectSize:   data.ContentLength,
+			StorageClass: &data.StorageClass,
+			VersionId:    data.VersionId,
+		}, nil
+	}
+	if !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+		return s3response.GetObjectAttributesResult{}, err
+	}
+
+	uploadId, _, err := p.retrieveUploadId(*input.Bucket, *input.Key)
+	if err != nil {
+		return s3response.GetObjectAttributesResult{}, err
+	}
+
+	resp, err := p.ListParts(ctx, &s3.ListPartsInput{
+		Bucket:           input.Bucket,
+		Key:              input.Key,
+		UploadId:         &uploadId,
+		PartNumberMarker: input.PartNumberMarker,
+		MaxParts:         input.MaxParts,
+	})
+	if err != nil {
+		return s3response.GetObjectAttributesResult{}, err
+	}
+
+	parts := []types.ObjectPart{}
+
+	for _, p := range resp.Parts {
+		if !(p.PartNumber > 0 && p.PartNumber <= math.MaxInt32) {
+			return s3response.GetObjectAttributesResult{},
+				s3err.GetAPIError(s3err.ErrInvalidPartNumber)
+		}
+
+		partNumber := int32(p.PartNumber)
+		size := p.Size
+
+		parts = append(parts, types.ObjectPart{
+			Size:       &size,
+			PartNumber: &partNumber,
+		})
+	}
+
+	//TODO: handle PartsCount prop
+	//TODO: Maybe simply calling ListParts isn't a good option
+	return s3response.GetObjectAttributesResult{
+		ObjectParts: &s3response.ObjectParts{
+			IsTruncated:          resp.IsTruncated,
+			MaxParts:             resp.MaxParts,
+			PartNumberMarker:     resp.PartNumberMarker,
+			NextPartNumberMarker: resp.NextPartNumberMarker,
+			Parts:                parts,
+		},
 	}, nil
 }
 
@@ -1900,7 +2296,7 @@ func (p *Posix) getAttrTags(bucket, object string) (map[string]string, error) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
 	if errors.Is(err, meta.ErrNoSuchKey) {
-		return tags, nil
+		return nil, s3err.GetAPIError(s3err.ErrBucketTaggingNotFound)
 	}
 	if err != nil {
 		return nil, fmt.Errorf("get tags: %w", err)
@@ -1998,7 +2394,7 @@ func (p *Posix) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, err
 
 	policy, err := p.meta.RetrieveAttribute(bucket, "", policykey)
 	if errors.Is(err, meta.ErrNoSuchKey) {
-		return []byte{}, nil
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)
 	}
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -2023,6 +2419,23 @@ func (p *Posix) PutObjectLockConfiguration(_ context.Context, bucket string, con
 		return fmt.Errorf("stat bucket: %w", err)
 	}
 
+	cfg, err := p.meta.RetrieveAttribute(bucket, "", bucketLockKey)
+	if errors.Is(err, meta.ErrNoSuchKey) {
+		return s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotAllowed)
+	}
+	if err != nil {
+		return fmt.Errorf("get object lock config: %w", err)
+	}
+
+	var bucketLockCfg auth.BucketLockConfig
+	if err := json.Unmarshal(cfg, &bucketLockCfg); err != nil {
+		return fmt.Errorf("unmarshal object lock config: %w", err)
+	}
+
+	if !bucketLockCfg.Enabled {
+		return s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotAllowed)
+	}
+
 	if err := p.meta.StoreAttribute(bucket, "", bucketLockKey, config); err != nil {
 		return fmt.Errorf("set object lock config: %w", err)
 	}
@@ -2119,7 +2532,7 @@ func (p *Posix) GetObjectLegalHold(_ context.Context, bucket, object, versionId
 	return &result, nil
 }
 
-func (p *Posix) PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error {
+func (p *Posix) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -2145,11 +2558,38 @@ func (p *Posix) PutObjectRetention(_ context.Context, bucket, object, versionId
 		return s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)
 	}
 
-	err = p.meta.StoreAttribute(bucket, object, objectRetentionKey, retention)
+	objectLockCfg, err := p.meta.RetrieveAttribute(bucket, object, objectRetentionKey)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
+	if errors.Is(err, meta.ErrNoSuchKey) {
+		if err := p.meta.StoreAttribute(bucket, object, objectRetentionKey, retention); err != nil {
+			return fmt.Errorf("set object lock config: %w", err)
+		}
+
+		return nil
+	}
 	if err != nil {
+		return fmt.Errorf("get object lock config: %w", err)
+	}
+
+	var lockCfg types.ObjectLockRetention
+	if err := json.Unmarshal(objectLockCfg, &lockCfg); err != nil {
+		return fmt.Errorf("unmarshal object lock config: %w", err)
+	}
+
+	switch lockCfg.Mode {
+	// Compliance mode can't be overridden
+	case types.ObjectLockRetentionModeCompliance:
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	// To override governance mode user should have "s3:BypassGovernanceRetention" permission
+	case types.ObjectLockRetentionModeGovernance:
+		if !bypass {
+			return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+	}
+
+	if err := p.meta.StoreAttribute(bucket, object, objectRetentionKey, retention); err != nil {
 		return fmt.Errorf("set object lock config: %w", err)
 	}
 

backend/posix/with_otmpfile.go

@@ -50,7 +50,7 @@ var (
 	defaultFilePerm uint32 = 0644
 )
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
@@ -81,7 +81,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 			gid:        gid,
 		}
 		// falloc is best effort, its fine if this fails
-		if size > 0 {
+		if size > 0 && dofalloc {
 			tmp.falloc()
 		}
 
@@ -111,7 +111,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	}
 
 	// falloc is best effort, its fine if this fails
-	if size > 0 {
+	if size > 0 && dofalloc {
 		tmp.falloc()
 	}
 
@@ -221,3 +221,7 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
+
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}

backend/posix/without_otmpfile.go

@@ -36,7 +36,7 @@ type tmpfile struct {
 	size    int64
 }
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
 	// Create a temp file for upload while in progress (see link comments below).
@@ -112,3 +112,7 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
+
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}

backend/s3proxy/client.go

@@ -33,6 +33,12 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
 		return nil, err
 	}
 
+	if s.endpoint != "" {
+		return s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &s.endpoint
+		}), nil
+	}
+
 	return s3.NewFromConfig(cfg), nil
 }
 
@@ -50,11 +56,6 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 		config.WithHTTPClient(client),
 	}
 
-	if s.endpoint != "" {
-		opts = append(opts,
-			config.WithEndpointResolverWithOptions(s))
-	}
-
 	if s.disableChecksum {
 		opts = append(opts,
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -67,13 +68,3 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 
 	return config.LoadDefaultConfig(ctx, opts...)
 }
-
-// ResolveEndpoint is used for on prem or non-aws endpoints
-func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
-	return aws.Endpoint{
-		PartitionID:       "aws",
-		URL:               s.endpoint,
-		SigningRegion:     s.awsRegion,
-		HostnameImmutable: true,
-	}, nil
-}

backend/s3proxy/s3.go

@@ -33,6 +33,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
@@ -54,6 +55,8 @@ type S3Proxy struct {
 	debug           bool
 }
 
+var _ backend.Backend = &S3Proxy{}
+
 func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
 	s := &S3Proxy{
 		access:          access,
@@ -127,6 +130,37 @@ func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput)
 	return handleError(err)
 }
 
+func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
+	_, err := s.client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+		Bucket: &bucket,
+		OwnershipControls: &types.OwnershipControls{
+			Rules: []types.OwnershipControlsRule{
+				{
+					ObjectOwnership: ownership,
+				},
+			},
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	resp, err := s.client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return ownship, handleError(err)
+	}
+	return resp.OwnershipControls.Rules[0].ObjectOwnership, nil
+}
+func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	return handleError(err)
+}
+
 func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	out, err := s.client.CreateMultipartUpload(ctx, input)
 	return out, handleError(err)
@@ -280,24 +314,50 @@ func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	return out, handleError(err)
 }
 
-func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	output, err := s.client.GetObject(ctx, input)
 	if err != nil {
 		return nil, handleError(err)
 	}
-	defer output.Body.Close()
-
-	_, err = io.Copy(w, output.Body)
-	if err != nil {
-		return nil, err
-	}
 
 	return output, nil
 }
 
-func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 	out, err := s.client.GetObjectAttributes(ctx, input)
-	return out, handleError(err)
+
+	parts := s3response.ObjectParts{}
+	objParts := out.ObjectParts
+	if objParts != nil {
+		if objParts.PartNumberMarker != nil {
+			partNumberMarker, err := strconv.Atoi(*objParts.PartNumberMarker)
+			if err != nil {
+				parts.PartNumberMarker = partNumberMarker
+			}
+			if objParts.NextPartNumberMarker != nil {
+				nextPartNumberMarker, err := strconv.Atoi(*objParts.NextPartNumberMarker)
+				if err != nil {
+					parts.NextPartNumberMarker = nextPartNumberMarker
+				}
+			}
+			if objParts.IsTruncated != nil {
+				parts.IsTruncated = *objParts.IsTruncated
+			}
+			if objParts.MaxParts != nil {
+				parts.MaxParts = int(*objParts.MaxParts)
+			}
+			parts.Parts = objParts.Parts
+		}
+	}
+
+	return s3response.GetObjectAttributesResult{
+		ETag:         out.ETag,
+		LastModified: out.LastModified,
+		ObjectSize:   out.ObjectSize,
+		StorageClass: &out.StorageClass,
+		VersionId:    out.VersionId,
+		ObjectParts:  &parts,
+	}, handleError(err)
 }
 
 func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
@@ -436,6 +496,129 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
 	return handleError(err)
 }
 
+func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
+	_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+		Bucket: &bucket,
+		Policy: backend.GetStringPtr(string(policy)),
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
+	policy, err := s.client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	result := []byte{}
+	if policy.Policy != nil {
+		result = []byte(*policy.Policy)
+	}
+
+	return result, nil
+}
+
+func (s *S3Proxy) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+		Bucket: &bucket,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
+	cfg, err := auth.ParseBucketLockConfigurationOutput(config)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{
+		Bucket:                  &bucket,
+		ObjectLockConfiguration: cfg,
+	})
+
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
+	resp, err := s.client.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	config := auth.BucketLockConfig{
+		Enabled:          resp.ObjectLockConfiguration.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
+		DefaultRetention: resp.ObjectLockConfiguration.Rule.DefaultRetention,
+	}
+
+	return json.Marshal(config)
+}
+
+func (s *S3Proxy) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
+	ret, err := auth.ParseObjectLockRetentionOutput(retention)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
+		Bucket:                    &bucket,
+		Key:                       &object,
+		VersionId:                 &versionId,
+		Retention:                 ret,
+		BypassGovernanceRetention: &bypass,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
+	resp, err := s.client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{
+		Bucket:    &bucket,
+		Key:       &object,
+		VersionId: &versionId,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	return json.Marshal(resp.Retention)
+}
+
+func (s *S3Proxy) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
+	var st types.ObjectLockLegalHoldStatus
+	if status {
+		st = types.ObjectLockLegalHoldStatusOn
+	} else {
+		st = types.ObjectLockLegalHoldStatusOff
+	}
+
+	_, err := s.client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{
+		Bucket:    &bucket,
+		Key:       &object,
+		VersionId: &versionId,
+		LegalHold: &types.ObjectLockLegalHold{
+			Status: st,
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
+	resp, err := s.client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{
+		Bucket:    &bucket,
+		Key:       &object,
+		VersionId: &versionId,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	status := resp.LegalHold.Status == types.ObjectLockLegalHoldStatusOn
+	return &status, nil
+}
+
 func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
 	if err != nil {

backend/scoutfs/scoutfs.go

@@ -26,12 +26,14 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
 )
@@ -47,6 +49,9 @@ type ScoutFS struct {
 	rootfd  *os.File
 	rootdir string
 
+	// bucket/object metadata storage facility
+	meta meta.MetadataStorer
+
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
 	// HEAD object: if file offline, set obj storage class to GLACIER
@@ -75,8 +80,13 @@ const (
 	metaTmpDir          = ".sgwtmp"
 	metaTmpMultipartDir = metaTmpDir + "/multipart"
 	tagHdr              = "X-Amz-Tagging"
+	metaHdr             = "X-Amz-Meta"
+	contentTypeHdr      = "content-type"
+	contentEncHdr       = "content-encoding"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
-	etagkey             = "user.etag"
+	etagkey             = "etag"
+	objectRetentionKey  = "object-retention"
+	objectLegalHoldKey  = "object-legal-hold"
 )
 
 var (
@@ -87,11 +97,12 @@ var (
 
 const (
 	// ScoutFS special xattr types
-
 	systemPrefix = "scoutfs.hide."
 	onameAttr    = systemPrefix + "objname"
 	flagskey     = systemPrefix + "sam_flags"
 	stagecopykey = systemPrefix + "sam_stagereq"
+
+	fsBlocksize = 4096
 )
 
 const (
@@ -179,18 +190,20 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		return nil, err
 	}
 
-	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+	objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
 
 	// check all parts ok
 	last := len(parts) - 1
 	partsize := int64(0)
 	var totalsize int64
-	for i, p := range parts {
-		if p.PartNumber == nil {
+	for i, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
-		fi, err := os.Lstat(partPath)
+
+		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
+		fullPartPath := filepath.Join(bucket, partObjPath)
+		fi, err := os.Lstat(fullPartPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
@@ -198,23 +211,25 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		if i == 0 {
 			partsize = fi.Size()
 		}
+
+		// partsize must be a multiple of the filesystem blocksize
+		// except for last part
+		if i < last && partsize%fsBlocksize != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
 		totalsize += fi.Size()
 		// all parts except the last need to be the same size
 		if i < last && partsize != fi.Size() {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		// non-last part sizes need to be multiples of 4k for move blocks
-		// TODO: fallback to no move blocks if not 4k aligned?
-		if i == 0 && i < last && fi.Size()%4096 != 0 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
 
-		b, err := xattr.Get(partPath, "user.etag")
+		b, err := s.meta.RetrieveAttribute(bucket, partObjPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
 		}
-		if etag != *parts[i].ETag {
+		if parts[i].ETag == nil || etag != *parts[i].ETag {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 	}
@@ -230,10 +245,16 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}
 	defer f.cleanup()
 
-	for _, p := range parts {
-		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
+	for _, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
+		fullPartPath := filepath.Join(bucket, partObjPath)
+		pf, err := os.Open(fullPartPath)
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
+			return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
@@ -242,13 +263,13 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
+			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
 		}
 	}
 
 	userMetaData := make(map[string]string)
 	upiddir := filepath.Join(objdir, uploadID)
-	loadUserMetaData(upiddir, userMetaData)
+	cType, _ := s.loadUserMetaData(bucket, upiddir, userMetaData)
 
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
@@ -265,7 +286,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}
 
 	for k, v := range userMetaData {
-		err = xattr.Set(objname, "user."+k, []byte(v))
+		err = s.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
 			// cleanup object if returning error
 			os.Remove(objname)
@@ -273,10 +294,58 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		}
 	}
 
+	// load and set tagging
+	tagging, err := s.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
+	if err == nil {
+		if err := s.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object tagging: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object tagging: %w", err)
+	}
+
+	// set content-type
+	if cType != "" {
+		if err := s.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object content type: %w", err)
+		}
+	}
+
+	// load and set legal hold
+	lHold, err := s.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
+	if err == nil {
+		if err := s.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object legal hold: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object legal hold: %w", err)
+	}
+
+	// load and set retention
+	ret, err := s.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
+	if err == nil {
+		if err := s.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object retention: %w", err)
+		}
+	}
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object retention: %w", err)
+	}
+
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
-	err = xattr.Set(objname, "user.etag", []byte(s3MD5))
+	err = s.meta.StoreAttribute(bucket, object, etagkey, []byte(s3MD5))
 	if err != nil {
 		// cleanup object if returning error
 		os.Remove(objname)
@@ -310,61 +379,104 @@ func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte
 	return sum, nil
 }
 
-func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
-	ents, err := xattr.List(path)
+// fll out the user metadata map with the metadata for the object
+// and return the content type and encoding
+func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
+	ents, err := s.meta.ListAttributes(bucket, object)
 	if err != nil || len(ents) == 0 {
-		return
+		return "", ""
 	}
 	for _, e := range ents {
 		if !isValidMeta(e) {
 			continue
 		}
-		b, err := xattr.Get(path, e)
-		if err == errNoData {
-			m[strings.TrimPrefix(e, "user.")] = ""
-			continue
-		}
+		b, err := s.meta.RetrieveAttribute(bucket, object, e)
 		if err != nil {
 			continue
 		}
-		m[strings.TrimPrefix(e, "user.")] = string(b)
+		if b == nil {
+			m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = ""
+			continue
+		}
+		m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
 	}
 
-	b, err := xattr.Get(path, "user.content-type")
+	var contentType, contentEncoding string
+	b, _ := s.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
 	contentType = string(b)
-	if err != nil {
-		contentType = ""
-	}
 	if contentType != "" {
-		m["content-type"] = contentType
+		m[contentTypeHdr] = contentType
 	}
 
-	b, err = xattr.Get(path, "user.content-encoding")
+	b, _ = s.meta.RetrieveAttribute(bucket, object, contentEncHdr)
 	contentEncoding = string(b)
-	if err != nil {
-		contentEncoding = ""
-	}
 	if contentEncoding != "" {
-		m["content-encoding"] = contentEncoding
+		m[contentEncHdr] = contentEncoding
 	}
 
-	return
+	return contentType, contentEncoding
 }
 
 func isValidMeta(val string) bool {
-	if strings.HasPrefix(val, "user.X-Amz-Meta") {
+	if strings.HasPrefix(val, metaHdr) {
 		return true
 	}
-	if strings.EqualFold(val, "user.Expires") {
+	if strings.EqualFold(val, "Expires") {
 		return true
 	}
 	return false
 }
 
-func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	if input.Key == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
 	bucket := *input.Bucket
 	object := *input.Key
 
+	if input.PartNumber != nil {
+		uploadId, sum, err := s.retrieveUploadId(bucket, object)
+		if err != nil {
+			return nil, err
+		}
+
+		ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read parts: %w", err)
+		}
+
+		partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
+
+		part, err := os.Stat(filepath.Join(bucket, partPath))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat part: %w", err)
+		}
+
+		b, err := s.meta.RetrieveAttribute(bucket, partPath, etagkey)
+		etag := string(b)
+		if err != nil {
+			etag = ""
+		}
+		partsCount := int32(len(ents))
+		size := part.Size()
+
+		return &s3.HeadObjectOutput{
+			LastModified:  backend.GetTimePtr(part.ModTime()),
+			ETag:          &etag,
+			PartsCount:    &partsCount,
+			ContentLength: &size,
+		}, nil
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -383,9 +495,14 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 	}
 
 	userMetaData := make(map[string]string)
-	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := xattr.Get(objPath, etagkey)
+	if fi.IsDir() {
+		// this is the media type for directories in AWS and Nextcloud
+		contentType = "application/x-directory"
+	}
+
+	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -424,19 +541,55 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 
 	contentLength := fi.Size()
 
+	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, "")
+	if err == nil {
+		if *status {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
+		} else {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
+		}
+	}
+
+	var objectLockMode types.ObjectLockMode
+	var objectLockRetainUntilDate *time.Time
+	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, "")
+	if err == nil {
+		var config types.ObjectLockRetention
+		if err := json.Unmarshal(retention, &config); err == nil {
+			objectLockMode = types.ObjectLockMode(config.Mode)
+			objectLockRetainUntilDate = config.RetainUntilDate
+		}
+	}
+
 	return &s3.HeadObjectOutput{
-		ContentLength:   &contentLength,
-		ContentType:     &contentType,
-		ContentEncoding: &contentEncoding,
-		ETag:            &etag,
-		LastModified:    backend.GetTimePtr(fi.ModTime()),
-		Metadata:        userMetaData,
-		StorageClass:    stclass,
-		Restore:         &requestOngoing,
+		ContentLength:             &contentLength,
+		ContentType:               &contentType,
+		ContentEncoding:           &contentEncoding,
+		ETag:                      &etag,
+		LastModified:              backend.GetTimePtr(fi.ModTime()),
+		Metadata:                  userMetaData,
+		StorageClass:              stclass,
+		Restore:                   &requestOngoing,
+		ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
+		ObjectLockMode:            objectLockMode,
+		ObjectLockRetainUntilDate: objectLockRetainUntilDate,
 	}, nil
 }
 
-func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
+	sum := sha256.Sum256([]byte(object))
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	entries, err := os.ReadDir(objdir)
+	if err != nil || len(entries) == 0 {
+		return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
+	return entries[0].Name(), sum, nil
+}
+
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	acceptRange := *input.Range
@@ -458,7 +611,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
 	if err != nil {
 		return nil, err
 	}
@@ -505,19 +658,14 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 	if err != nil {
 		return nil, fmt.Errorf("open object: %w", err)
 	}
-	defer f.Close()
 
 	rdr := io.NewSectionReader(f, startOffset, length)
-	_, err = io.Copy(writer, rdr)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
 
 	userMetaData := make(map[string]string)
 
-	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := xattr.Get(objPath, etagkey)
+	b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -541,6 +689,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
 		ContentRange:    &contentRange,
+		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
 	}, nil
 }
 
@@ -671,14 +820,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			etagBytes, err := xattr.Get(objPath, etagkey)
-			if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
-				return types.Object{}, backend.ErrSkipObj
-			}
+			b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
 			if err != nil {
 				return types.Object{}, fmt.Errorf("get etag: %w", err)
 			}
-			etag := string(etagBytes)
+			etag := string(b)
 
 			fi, err := d.Info()
 			if errors.Is(err, fs.ErrNotExist) {
@@ -698,14 +844,14 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		}
 
 		// file object, get object info and fill out object data
-		etagBytes, err := xattr.Get(objPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
 			return types.Object{}, backend.ErrSkipObj
 		}
-		if err != nil && !isNoAttr(err) {
+		if err != nil {
 			return types.Object{}, fmt.Errorf("get etag: %w", err)
 		}
-		etag := string(etagBytes)
+		etag := string(b)
 
 		fi, err := d.Info()
 		if errors.Is(err, fs.ErrNotExist) {
@@ -819,15 +965,9 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 }
 
 func isNoAttr(err error) bool {
-	if err == nil {
-		return false
-	}
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == errNoData {
-		return true
-	}
 	return false
 }

backend/scoutfs/scoutfs_compat.go

@@ -23,7 +23,6 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
-	"syscall"
 
 	"golang.org/x/sys/unix"
 
@@ -35,7 +34,9 @@ import (
 )
 
 func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
-	p, err := posix.New(rootdir, meta.XattrMeta{}, posix.PosixOpts{
+	metastore := meta.XattrMeta{}
+
+	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
 		ChownUID: opts.ChownUID,
 		ChownGID: opts.ChownGID,
 	})
@@ -49,11 +50,13 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:    p,
-		rootfd:   f,
-		rootdir:  rootdir,
-		chownuid: opts.ChownUID,
-		chowngid: opts.ChownGID,
+		Posix:       p,
+		rootfd:      f,
+		rootdir:     rootdir,
+		meta:        metastore,
+		chownuid:    opts.ChownUID,
+		chowngid:    opts.ChownGID,
+		glaciermode: opts.GlacierMode,
 	}, nil
 }
 
@@ -100,11 +103,6 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 		gid:        gid,
 	}
 
-	// falloc is best effort, its fine if this fails
-	if size > 0 {
-		tmp.falloc()
-	}
-
 	if doChown {
 		err := f.Chown(uid, gid)
 		if err != nil {
@@ -115,14 +113,6 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 	return tmp, nil
 }
 
-func (tmp *tmpfile) falloc() error {
-	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
-	if err != nil {
-		return fmt.Errorf("fallocate: %v", err)
-	}
-	return nil
-}
-
 func (tmp *tmpfile) link() error {
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict

backend/scoutfs/with_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build !freebsd && !openbsd && !netbsd
-// +build !freebsd,!openbsd,!netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENODATA
-)

backend/scoutfs/without_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build freebsd || openbsd || netbsd
-// +build freebsd openbsd netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENOATTR
-)

cmd/versitygw/admin.go

@@ -80,10 +80,33 @@ func adminCommand() *cli.Command {
 						Usage:   "groupID for the new user",
 						Aliases: []string{"gi"},
 					},
+				},
+			},
+			{
+				Name:   "update-user",
+				Usage:  "Updates a user account",
+				Action: updateUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "user access key id to be updated",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:    "secret",
+						Usage:   "secret access key for the new user",
+						Aliases: []string{"s"},
+					},
 					&cli.IntFlag{
-						Name:    "project-id",
-						Usage:   "projectID for the new user",
-						Aliases: []string{"pi"},
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
 					},
 				},
 			},
@@ -176,21 +199,20 @@ func initHTTPClient() *http.Client {
 
 func createUser(ctx *cli.Context) error {
 	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
-	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
+	userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
 	if access == "" || secret == "" {
-		return fmt.Errorf("invalid input parameters for the new user")
+		return fmt.Errorf("invalid input parameters for the new user access/secret keys")
 	}
 	if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
 		return fmt.Errorf("invalid input parameter for role: %v", role)
 	}
 
 	acc := auth.Account{
-		Access:    access,
-		Secret:    secret,
-		Role:      auth.Role(role),
-		UserID:    userID,
-		GroupID:   groupID,
-		ProjectID: projectID,
+		Access:  access,
+		Secret:  secret,
+		Role:    auth.Role(role),
+		UserID:  userID,
+		GroupID: groupID,
 	}
 
 	accJson, err := json.Marshal(acc)
@@ -240,7 +262,7 @@ func createUser(ctx *cli.Context) error {
 func deleteUser(ctx *cli.Context) error {
 	access := ctx.String("access")
 	if access == "" {
-		return fmt.Errorf("invalid input parameter for the new user")
+		return fmt.Errorf("invalid input parameter for the user access key")
 	}
 
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
@@ -282,6 +304,63 @@ func deleteUser(ctx *cli.Context) error {
 	return nil
 }
 
+func updateUser(ctx *cli.Context) error {
+	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
+	props := auth.MutableProps{}
+	if ctx.IsSet("secret") {
+		props.Secret = &secret
+	}
+	if ctx.IsSet("user-id") {
+		props.UserID = &userId
+	}
+	if ctx.IsSet("group-id") {
+		props.GroupID = &groupId
+	}
+
+	propsJSON, err := json.Marshal(props)
+	if err != nil {
+		return fmt.Errorf("failed to parse user attributes: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(propsJSON)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
 func listUsers(ctx *cli.Context) error {
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
 	if err != nil {
@@ -339,10 +418,10 @@ const (
 func printAcctTable(accs []auth.Account) {
 	w := new(tabwriter.Writer)
 	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
-	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
-	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------")
 	for _, acc := range accs {
-		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
 	}
 	fmt.Fprintln(w)
 	w.Flush()

cmd/versitygw/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api"
 	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
@@ -34,33 +35,43 @@ import (
 )
 
 var (
-	port, admPort                          string
-	rootUserAccess                         string
-	rootUserSecret                         string
-	region                                 string
-	admCertFile, admKeyFile                string
-	certFile, keyFile                      string
-	kafkaURL, kafkaTopic, kafkaKey         string
-	natsURL, natsTopic                     string
-	eventWebhookURL                        string
-	eventConfigFilePath                    string
-	logWebhookURL                          string
-	accessLog                              string
-	healthPath                             string
-	debug                                  bool
-	pprof                                  string
-	quiet                                  bool
-	iamDir                                 string
-	ldapURL, ldapBindDN, ldapPassword      string
-	ldapQueryBase, ldapObjClasses          string
-	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
-	s3IamAccess, s3IamSecret               string
-	s3IamRegion, s3IamBucket               string
-	s3IamEndpoint                          string
-	s3IamSslNoVerify, s3IamDebug           bool
-	iamCacheDisable                        bool
-	iamCacheTTL                            int
-	iamCachePrune                          int
+	port, admPort                            string
+	rootUserAccess                           string
+	rootUserSecret                           string
+	region                                   string
+	admCertFile, admKeyFile                  string
+	certFile, keyFile                        string
+	kafkaURL, kafkaTopic, kafkaKey           string
+	natsURL, natsTopic                       string
+	eventWebhookURL                          string
+	eventConfigFilePath                      string
+	logWebhookURL                            string
+	accessLog                                string
+	healthPath                               string
+	debug                                    bool
+	pprof                                    string
+	quiet                                    bool
+	readonly                                 bool
+	iamDir                                   string
+	ldapURL, ldapBindDN, ldapPassword        string
+	ldapQueryBase, ldapObjClasses            string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr   string
+	ldapUserIdAtr, ldapGroupIdAtr            string
+	vaultEndpointURL, vaultSecretStoragePath string
+	vaultMountPath, vaultRootToken           string
+	vaultRoleId, vaultRoleSecret             string
+	vaultServerCert, vaultClientCert         string
+	vaultClientCertKey                       string
+	s3IamAccess, s3IamSecret                 string
+	s3IamRegion, s3IamBucket                 string
+	s3IamEndpoint                            string
+	s3IamSslNoVerify, s3IamDebug             bool
+	iamCacheDisable                          bool
+	iamCacheTTL                              int
+	iamCachePrune                            int
+	metricsService                           string
+	statsdServers                            string
+	dogstatsServers                          string
 )
 
 var (
@@ -321,6 +332,72 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-user-id-atr",
+			Usage:       "ldap server user id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_USER_ID_ATR"},
+			Destination: &ldapUserIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-group-id-atr",
+			Usage:       "ldap server user group id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
+			Destination: &ldapGroupIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-endpoint-url",
+			Usage:       "vault server url",
+			EnvVars:     []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
+			Destination: &vaultEndpointURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-secret-storage-path",
+			Usage:       "vault server secret storage path",
+			EnvVars:     []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
+			Destination: &vaultSecretStoragePath,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-mount-path",
+			Usage:       "vault server mount path",
+			EnvVars:     []string{"VGW_IAM_VAULT_MOUNT_PATH"},
+			Destination: &vaultMountPath,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-root-token",
+			Usage:       "vault server root token",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
+			Destination: &vaultRootToken,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-role-id",
+			Usage:       "vault server user role id",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_ID"},
+			Destination: &vaultRoleId,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-role-secret",
+			Usage:       "vault server user role secret",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_SECRET"},
+			Destination: &vaultRoleSecret,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-server_cert",
+			Usage:       "vault server TLS certificate",
+			EnvVars:     []string{"VGW_IAM_VAULT_SERVER_CERT"},
+			Destination: &vaultServerCert,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-client_cert",
+			Usage:       "vault client TLS certificate",
+			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT"},
+			Destination: &vaultClientCert,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-client_cert_key",
+			Usage:       "vault client TLS certificate key",
+			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
+			Destination: &vaultClientCertKey,
+		},
 		&cli.StringFlag{
 			Name:        "s3-iam-access",
 			Usage:       "s3 IAM access key",
@@ -391,6 +468,33 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_HEALTH"},
 			Destination: &healthPath,
 		},
+		&cli.BoolFlag{
+			Name:        "readonly",
+			Usage:       "allow only read operations across all the gateway",
+			EnvVars:     []string{"VGW_READ_ONLY"},
+			Destination: &readonly,
+		},
+		&cli.StringFlag{
+			Name:        "metrics-service-name",
+			Usage:       "service name tag for metrics, hostname if blank",
+			EnvVars:     []string{"VGW_METRICS_SERVICE_NAME"},
+			Aliases:     []string{"msn"},
+			Destination: &metricsService,
+		},
+		&cli.StringFlag{
+			Name:        "metrics-statsd-servers",
+			Usage:       "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
+			EnvVars:     []string{"VGW_METRICS_STATSD_SERVERS"},
+			Aliases:     []string{"mss"},
+			Destination: &statsdServers,
+		},
+		&cli.StringFlag{
+			Name:        "metrics-dogstatsd-servers",
+			Usage:       "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
+			EnvVars:     []string{"VGW_METRICS_DOGSTATS_SERVERS"},
+			Aliases:     []string{"mds"},
+			Destination: &dogstatsServers,
+		},
 	}
 }
 
@@ -442,6 +546,9 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if healthPath != "" {
 		opts = append(opts, s3api.WithHealth(healthPath))
 	}
+	if readonly {
+		opts = append(opts, s3api.WithReadOnly())
+	}
 
 	admApp := fiber.New(fiber.Config{
 		AppName:      "versitygw",
@@ -466,25 +573,36 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
-		Dir:                iamDir,
-		LDAPServerURL:      ldapURL,
-		LDAPBindDN:         ldapBindDN,
-		LDAPPassword:       ldapPassword,
-		LDAPQueryBase:      ldapQueryBase,
-		LDAPObjClasses:     ldapObjClasses,
-		LDAPAccessAtr:      ldapAccessAtr,
-		LDAPSecretAtr:      ldapSecAtr,
-		LDAPRoleAtr:        ldapRoleAtr,
-		S3Access:           s3IamAccess,
-		S3Secret:           s3IamSecret,
-		S3Region:           s3IamRegion,
-		S3Bucket:           s3IamBucket,
-		S3Endpoint:         s3IamEndpoint,
-		S3DisableSSlVerfiy: s3IamSslNoVerify,
-		S3Debug:            s3IamDebug,
-		CacheDisable:       iamCacheDisable,
-		CacheTTL:           iamCacheTTL,
-		CachePrune:         iamCachePrune,
+		Dir:                    iamDir,
+		LDAPServerURL:          ldapURL,
+		LDAPBindDN:             ldapBindDN,
+		LDAPPassword:           ldapPassword,
+		LDAPQueryBase:          ldapQueryBase,
+		LDAPObjClasses:         ldapObjClasses,
+		LDAPAccessAtr:          ldapAccessAtr,
+		LDAPSecretAtr:          ldapSecAtr,
+		LDAPRoleAtr:            ldapRoleAtr,
+		LDAPUserIdAtr:          ldapUserIdAtr,
+		LDAPGroupIdAtr:         ldapGroupIdAtr,
+		VaultEndpointURL:       vaultEndpointURL,
+		VaultSecretStoragePath: vaultSecretStoragePath,
+		VaultMountPath:         vaultMountPath,
+		VaultRootToken:         vaultRootToken,
+		VaultRoleId:            vaultRoleId,
+		VaultRoleSecret:        vaultRoleSecret,
+		VaultServerCert:        vaultServerCert,
+		VaultClientCert:        vaultClientCert,
+		VaultClientCertKey:     vaultClientCertKey,
+		S3Access:               s3IamAccess,
+		S3Secret:               s3IamSecret,
+		S3Region:               s3IamRegion,
+		S3Bucket:               s3IamBucket,
+		S3Endpoint:             s3IamEndpoint,
+		S3DisableSSlVerfiy:     s3IamSslNoVerify,
+		S3Debug:                s3IamDebug,
+		CacheDisable:           iamCacheDisable,
+		CacheTTL:               iamCacheTTL,
+		CachePrune:             iamCachePrune,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)
@@ -498,6 +616,15 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		return fmt.Errorf("setup logger: %w", err)
 	}
 
+	metricsManager, err := metrics.NewManager(ctx, metrics.Config{
+		ServiceName:      metricsService,
+		StatsdServers:    statsdServers,
+		DogStatsdServers: dogstatsServers,
+	})
+	if err != nil {
+		return fmt.Errorf("init metrics manager: %w", err)
+	}
+
 	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
 		KafkaURL:             kafkaURL,
 		KafkaTopic:           kafkaTopic,
@@ -514,7 +641,7 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
 		Access: rootUserAccess,
 		Secret: rootUserSecret,
-	}, port, region, iam, logger, evSender, opts...)
+	}, port, region, iam, logger, evSender, metricsManager, opts...)
 	if err != nil {
 		return fmt.Errorf("init gateway: %v", err)
 	}
@@ -577,5 +704,9 @@ Loop:
 		}
 	}
 
+	if metricsManager != nil {
+		metricsManager.Close()
+	}
+
 	return saveErr
 }

cmd/versitygw/utils.go

@@ -54,22 +54,24 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
 	}
 
 	config := s3event.EventFilter{
-		s3event.EventObjectCreated:           true,
-		s3event.EventObjectCreatedPut:        true,
-		s3event.EventObjectCreatedPost:       true,
-		s3event.EventObjectCreatedCopy:       true,
-		s3event.EventCompleteMultipartUpload: true,
-		s3event.EventObjectDeleted:           true,
-		s3event.EventObjectTagging:           true,
-		s3event.EventObjectTaggingPut:        true,
-		s3event.EventObjectTaggingDelete:     true,
-		s3event.EventObjectAclPut:            true,
-		s3event.EventObjectRestore:           true,
-		s3event.EventObjectRestorePost:       true,
-		s3event.EventObjectRestoreCompleted:  true,
+		s3event.EventObjectCreated:              true,
+		s3event.EventObjectCreatedPut:           true,
+		s3event.EventObjectCreatedPost:          true,
+		s3event.EventObjectCreatedCopy:          true,
+		s3event.EventCompleteMultipartUpload:    true,
+		s3event.EventObjectRemoved:              true,
+		s3event.EventObjectRemovedDelete:        true,
+		s3event.EventObjectRemovedDeleteObjects: true,
+		s3event.EventObjectTagging:              true,
+		s3event.EventObjectTaggingPut:           true,
+		s3event.EventObjectTaggingDelete:        true,
+		s3event.EventObjectAclPut:               true,
+		s3event.EventObjectRestore:              true,
+		s3event.EventObjectRestorePost:          true,
+		s3event.EventObjectRestoreCompleted:     true,
 	}
 
-	configBytes, err := json.Marshal(config)
+	configBytes, err := json.MarshalIndent(config, "", "  ")
 	if err != nil {
 		return fmt.Errorf("parse event config: %w", err)
 	}

extra/example.conf

@@ -153,6 +153,14 @@ ROOT_SECRET_ACCESS_KEY=
 # specified, all configured bucket events will be sent to the webhook.
 #VGW_EVENT_WEBHOOK_URL=
 
+# Bucket events can be filtered for any of the above event types. The
+# VGW_EVENT_FILTER option specifies a config file that contains the event
+# filter rules. The event filter rules are used to determine which events are
+# sent to the configured event services. Run:
+# versitygw utils gen-event-filter-config --path .
+# to generate a default rules file "event_config.json" in the current directory.
+#VGW_EVENT_FILTER=
+
 #######################
 # Debug / Diagnostics #
 #######################
@@ -182,20 +190,25 @@ ROOT_SECRET_ACCESS_KEY=
 # as a dedicated IAM service.
 #VGW_IAM_DIR=
 
-# The ldap options will enable the LDAP IAM service with accounts stored in an
-# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
-# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
-# secret credentials and role respectively. The other options are used to
-# connect to the LDAP service.
-#VGW_IAM_LDAP_URL=
-#VGW_IAM_LDAP_BASE_DN=
-#VGW_IAM_LDAP_BIND_DN=
-#VGW_IAM_LDAP_BIND_PASS=
-#VGW_IAM_LDAP_QUERY_BASE=
-#VGW_IAM_LDAP_OBJECT_CLASSES=
-#VGW_IAM_LDAP_ACCESS_ATR=
-#VGW_IAM_LDAP_SECRET_ATR=
-#VGW_IAM_LDAP_ROLE_ATR=
+# The Vault options will enable the Vault IAM service with accounts stored in
+# the HashiCorp Vault service. The Vault URL is the address and port of the
+# Vault server with the format <IP/host>:<port>. A root taken can be used for
+# testing, but it is recommended to use the role based authentication in
+# production. The Vault server certificate, client certificate, and client
+# certificate key are optional, and will default to not verifying the server
+# certificate and not using client certificates. The Vault server certificate
+# is used to verify the Vault server, and the client certificate and key are
+# used to authenticate the gateway to the Vault server. See wiki documentation
+# for an example of using Vault in dev mode with the gateway.
+#VGW_IAM_VAULT_ENDPOINT_URL=
+#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
+#VGW_IAM_VAULT_MOUNT_PATH=
+#VGW_IAM_VAULT_ROOT_TOKEN=
+#VGW_IAM_VAULT_ROLE_ID=
+#VGW_IAM_VAULT_ROLE_SECRET=
+#VGW_IAM_VAULT_SERVER_CERT=
+#VGW_IAM_VAULT_CLIENT_CERT=
+#VGW_IAM_VAULT_CLIENT_CERT_KEY=
 
 # The VGW_S3 IAM service is similar to the internal IAM service, but instead
 # stores the account information JSON encoded in an S3 object. This should use
@@ -210,6 +223,21 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_S3_IAM_BUCKET=
 #VGW_S3_IAM_NO_VERIFY=
 
+# The LDAP options will enable the LDAP IAM service with accounts stored in an
+# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
+# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
+# secret credentials and role respectively. The other options are used to
+# connect to the LDAP service.
+#VGW_IAM_LDAP_URL=
+#VGW_IAM_LDAP_BASE_DN=
+#VGW_IAM_LDAP_BIND_DN=
+#VGW_IAM_LDAP_BIND_PASS=
+#VGW_IAM_LDAP_QUERY_BASE=
+#VGW_IAM_LDAP_OBJECT_CLASSES=
+#VGW_IAM_LDAP_ACCESS_ATR=
+#VGW_IAM_LDAP_SECRET_ATR=
+#VGW_IAM_LDAP_ROLE_ATR=
+
 ###############
 # IAM caching #
 ###############
@@ -228,6 +256,29 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_CACHE_TTL=120
 #VGW_IAM_CACHE_PRUNE=3600
 
+###########
+# Metrics #
+###########
+
+# The metrics service name is a tag that is added to all metrics to help
+# identify the source of the metrics. This is especially useful when multiple
+# gateways are running. The default is the hostname of the system.
+#VGW_METRICS_SERVICE_NAME=$HOSTNAME
+
+# The metrics service will send metrics to the configured statsd servers. The
+# servers are specified as a comma separated list of host:port pairs. The
+# default is to not send metrics to any statsd servers. The gateway uses
+# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
+#VGW_METRICS_STATSD_SERVERS=
+
+# The metrics service will send metrics to the configured dogstatsd servers.
+# The servers are specified as a comma separated list of host:port pairs. The
+# default is to not send metrics to any dogstatsd servers. Generally
+# DataDog recommends installing a local agent to collect metrics and forward
+# them to the DataDog service. In this case the option value would be the
+# local agent address: 127.0.0.1:8125.
+#VGW_METRICS_DOGSTATS_SERVERS=
+
 ######################################
 # VersityGW Backend Specific Options #
 ######################################

go.mod

@@ -3,62 +3,73 @@ module github.com/versity/versitygw
 go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
-	github.com/aws/aws-sdk-go-v2 v1.26.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
-	github.com/aws/smithy-go v1.20.2
+	github.com/DataDog/datadog-go/v5 v5.5.0
+	github.com/aws/aws-sdk-go-v2 v1.30.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0
+	github.com/aws/smithy-go v1.20.3
 	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/gofiber/fiber/v2 v2.52.4
+	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.34.1
+	github.com/hashicorp/vault-client-go v0.4.3
+	github.com/nats-io/nats.go v1.36.0
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
-	github.com/urfave/cli/v2 v2.27.1
-	github.com/valyala/fasthttp v1.52.0
+	github.com/smira/go-statsd v1.3.3
+	github.com/urfave/cli/v2 v2.27.2
+	github.com/valyala/fasthttp v1.55.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sys v0.19.0
+	golang.org/x/sys v0.22.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.11
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.24
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.24
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.5
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect

go.sum

@@ -1,9 +1,9 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 h1:sUFnFjzDUie80h24I7mrKtwCKgLY9L8h5Tp2x9+TWqk=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0/go.mod h1:52JbnQTp15qg5mRkMBHwp0j0ZFwHJ42Sx3zVV5RE9p0=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1 h1:Xy/qV1DyOhhqsU/z0PyFMJfYCxnzna+vBEUtFW0ksQo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1/go.mod h1:oib6iWdC+sILvNUoJbbBn3xv7TXow7mEp/WRcsYvmow=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
@@ -12,73 +12,91 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
+github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
-github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
-github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
-github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
+github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o=
+github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
+github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo=
+github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.5 h1:qkipTyOc+ElVS+TgGJCf/6gqu0CL5+ii19W/eMQfY94=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.5/go.mod h1:UjB35RXl+ESpnVtyaKqdw11NhMxm90lF9o2zqJNbi14=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
-github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
-github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 h1:THZJJ6TU/FOiM7DZFnisYV9d49oxXWUzsVIMTuf3VNU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13/go.mod h1:VISUTg6n+uBaYIWPBaIG0jk7mbBxm7DUqBtU2cUDDWI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 h1:2jyRZ9rVIMisyQRnhSS/SqlckveoxXneIumECVFP91Y=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15/go.mod h1:bDRG3m382v1KJBk1cKz7wIajg87/61EiiymEyfLvAe0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 h1:Eq2THzHt6P41mpjS2sUzz/3dJYFRqdWZ+vQaEMm98EM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13/go.mod h1:FgwTca6puegxgCInYwGjmd4tB9195Dd6LCuA+8MjpWw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0 h1:4rhV0Hn+bf8IAIUphRX1moBcEvKJipCPmswMCl6Q5mw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0/go.mod h1:hdV0NTYd0RwV4FvNKhKUNbPLZoq9CTr/lke+3I7aCAI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-asn1-ber/asn1-ber v1.5.6 h1:CYsqysemXfEaQbyrLJmdsCRuufHoLa3P/gGWGl5TDrM=
-github.com/go-asn1-ber/asn1-ber v1.5.6/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
+github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
 github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
-github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
+github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
+github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@@ -96,8 +114,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -107,8 +125,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
-github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU=
+github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -118,6 +138,7 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
 github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -127,23 +148,30 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
+github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
+github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
-github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
+github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
-github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
+github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
+github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -156,20 +184,25 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
 github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@@ -177,13 +210,19 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -196,8 +235,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -212,13 +251,19 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

metrics/actions.go

@@ -0,0 +1,261 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+type Action struct {
+	Name    string
+	Service string
+}
+
+var (
+	ActionMap map[string]Action
+)
+
+var (
+	ActionUndetected                    = "ActionUnDetected"
+	ActionAbortMultipartUpload          = "s3_AbortMultipartUpload"
+	ActionCompleteMultipartUpload       = "s3_CompleteMultipartUpload"
+	ActionCopyObject                    = "s3_CopyObject"
+	ActionCreateBucket                  = "s3_CreateBucket"
+	ActionCreateMultipartUpload         = "s3_CreateMultipartUpload"
+	ActionDeleteBucket                  = "s3_DeleteBucket"
+	ActionDeleteBucketPolicy            = "s3_DeleteBucketPolicy"
+	ActionDeleteBucketTagging           = "s3_DeleteBucketTagging"
+	ActionDeleteObject                  = "s3_DeleteObject"
+	ActionDeleteObjectTagging           = "s3_DeleteObjectTagging"
+	ActionDeleteObjects                 = "s3_DeleteObjects"
+	ActionGetBucketAcl                  = "s3_GetBucketAcl"
+	ActionGetBucketPolicy               = "s3_GetBucketPolicy"
+	ActionGetBucketTagging              = "s3_GetBucketTagging"
+	ActionGetBucketVersioning           = "s3_GetBucketVersioning"
+	ActionGetObject                     = "s3_GetObject"
+	ActionGetObjectAcl                  = "s3_GetObjectAcl"
+	ActionGetObjectAttributes           = "s3_GetObjectAttributes"
+	ActionGetObjectLegalHold            = "s3_GetObjectLegalHold"
+	ActionGetObjectLockConfiguration    = "s3_GetObjectLockConfiguration"
+	ActionGetObjectRetention            = "s3_GetObjectRetention"
+	ActionGetObjectTagging              = "s3_GetObjectTagging"
+	ActionHeadBucket                    = "s3_HeadBucket"
+	ActionHeadObject                    = "s3_HeadObject"
+	ActionListAllMyBuckets              = "s3_ListAllMyBuckets"
+	ActionListMultipartUploads          = "s3_ListMultipartUploads"
+	ActionListObjectVersions            = "s3_ListObjectVersions"
+	ActionListObjects                   = "s3_ListObjects"
+	ActionListObjectsV2                 = "s3_ListObjectsV2"
+	ActionListParts                     = "s3_ListParts"
+	ActionPutBucketAcl                  = "s3_PutBucketAcl"
+	ActionPutBucketPolicy               = "s3_PutBucketPolicy"
+	ActionPutBucketTagging              = "s3_PutBucketTagging"
+	ActionPutBucketVersioning           = "s3_PutBucketVersioning"
+	ActionPutObject                     = "s3_PutObject"
+	ActionPutObjectAcl                  = "s3_PutObjectAcl"
+	ActionPutObjectLegalHold            = "s3_PutObjectLegalHold"
+	ActionPutObjectLockConfiguration    = "s3_PutObjectLockConfiguration"
+	ActionPutObjectRetention            = "s3_PutObjectRetention"
+	ActionPutObjectTagging              = "s3_PutObjectTagging"
+	ActionRestoreObject                 = "s3_RestoreObject"
+	ActionSelectObjectContent           = "s3_SelectObjectContent"
+	ActionUploadPart                    = "s3_UploadPart"
+	ActionUploadPartCopy                = "s3_UploadPartCopy"
+	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
+	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
+	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
+)
+
+func init() {
+	ActionMap = make(map[string]Action)
+
+	ActionMap[ActionUndetected] = Action{
+		Name:    "ActionUnDetected",
+		Service: "unknown",
+	}
+
+	ActionMap[ActionAbortMultipartUpload] = Action{
+		Name:    "AbortMultipartUpload",
+		Service: "s3",
+	}
+	ActionMap[ActionCompleteMultipartUpload] = Action{
+		Name:    "CompleteMultipartUpload",
+		Service: "s3",
+	}
+	ActionMap[ActionCopyObject] = Action{
+		Name:    "CopyObject",
+		Service: "s3",
+	}
+	ActionMap[ActionCreateBucket] = Action{
+		Name:    "CreateBucket",
+		Service: "s3",
+	}
+	ActionMap[ActionCreateMultipartUpload] = Action{
+		Name:    "CreateMultipartUpload",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucket] = Action{
+		Name:    "DeleteBucket",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucketPolicy] = Action{
+		Name:    "DeleteBucketPolicy",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucketTagging] = Action{
+		Name:    "DeleteBucketTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteObject] = Action{
+		Name:    "DeleteObject",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteObjectTagging] = Action{
+		Name:    "DeleteObjectTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteObjects] = Action{
+		Name:    "DeleteObjects",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketAcl] = Action{
+		Name:    "GetBucketAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketPolicy] = Action{
+		Name:    "GetBucketPolicy",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketTagging] = Action{
+		Name:    "GetBucketTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketVersioning] = Action{
+		Name:    "GetBucketVersioning",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObject] = Action{
+		Name:    "GetObject",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectAcl] = Action{
+		Name:    "GetObjectAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectAttributes] = Action{
+		Name:    "GetObjectAttributes",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectLegalHold] = Action{
+		Name:    "GetObjectLegalHold",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectLockConfiguration] = Action{
+		Name:    "GetObjectLockConfiguration",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectRetention] = Action{
+		Name:    "GetObjectRetention",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectTagging] = Action{
+		Name:    "GetObjectTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionHeadBucket] = Action{
+		Name:    "HeadBucket",
+		Service: "s3",
+	}
+	ActionMap[ActionHeadObject] = Action{
+		Name:    "HeadObject",
+		Service: "s3",
+	}
+	ActionMap[ActionListAllMyBuckets] = Action{
+		Name:    "ListAllMyBuckets",
+		Service: "s3",
+	}
+	ActionMap[ActionListMultipartUploads] = Action{
+		Name:    "ListMultipartUploads",
+		Service: "s3",
+	}
+	ActionMap[ActionListObjectVersions] = Action{
+		Name:    "ListObjectVersions",
+		Service: "s3",
+	}
+	ActionMap[ActionListObjects] = Action{
+		Name:    "ListObjects",
+		Service: "s3",
+	}
+	ActionMap[ActionListObjectsV2] = Action{
+		Name:    "ListObjectsV2",
+		Service: "s3",
+	}
+	ActionMap[ActionListParts] = Action{
+		Name:    "ListParts",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketAcl] = Action{
+		Name:    "PutBucketAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketPolicy] = Action{
+		Name:    "PutBucketPolicy",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketTagging] = Action{
+		Name:    "PutBucketTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketVersioning] = Action{
+		Name:    "PutBucketVersioning",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObject] = Action{
+		Name:    "PutObject",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectAcl] = Action{
+		Name:    "PutObjectAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectLegalHold] = Action{
+		Name:    "PutObjectLegalHold",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectLockConfiguration] = Action{
+		Name:    "PutObjectLockConfiguration",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectRetention] = Action{
+		Name:    "PutObjectRetention",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectTagging] = Action{
+		Name:    "PutObjectTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionRestoreObject] = Action{
+		Name:    "RestoreObject",
+		Service: "s3",
+	}
+	ActionMap[ActionSelectObjectContent] = Action{
+		Name:    "SelectObjectContent",
+		Service: "s3",
+	}
+	ActionMap[ActionUploadPart] = Action{
+		Name:    "UploadPart",
+		Service: "s3",
+	}
+	ActionMap[ActionUploadPartCopy] = Action{
+		Name:    "UploadPartCopy",
+		Service: "s3",
+	}
+}

metrics/dogstats.go

@@ -0,0 +1,65 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+import (
+	"fmt"
+
+	dogstats "github.com/DataDog/datadog-go/v5/statsd"
+)
+
+// vgwDogStatsd metrics type
+type vgwDogStatsd struct {
+	c *dogstats.Client
+}
+
+var (
+	rateSampleAlways = 1.0
+)
+
+// newDogStatsd takes a server address and returns a statsd merics
+func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
+	c, err := dogstats.New(server,
+		dogstats.WithMaxMessagesPerPayload(1000),
+		dogstats.WithNamespace("versitygw"),
+		dogstats.WithTags([]string{
+			"service:" + service,
+		}))
+	if err != nil {
+		return nil, err
+	}
+	return &vgwDogStatsd{c: c}, nil
+}
+
+// Close closes statsd connections
+func (s *vgwDogStatsd) Close() {
+	s.c.Close()
+}
+
+func (t Tag) ddString() string {
+	if t.Value == "" {
+		return t.Key
+	}
+	return fmt.Sprintf("%v:%v", t.Key, t.Value)
+}
+
+// Add adds value to key
+func (s *vgwDogStatsd) Add(key string, value int64, tags ...Tag) {
+	stags := make([]string, len(tags))
+	for i, t := range tags {
+		stags[i] = t.ddString()
+	}
+	s.c.Count(key, value, stags, rateSampleAlways)
+}

metrics/metrics.go

@@ -0,0 +1,225 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	// max size of data items to buffer before dropping
+	// new incoming data items
+	dataItemCount = 100000
+)
+
+// Tag is added metadata for metrics
+type Tag struct {
+	// Key is tag name
+	Key string
+	// Value is tag data
+	Value string
+}
+
+// Manager is a manager of metrics plugins
+type Manager struct {
+	wg  sync.WaitGroup
+	ctx context.Context
+
+	config Config
+
+	publishers  []publisher
+	addDataChan chan datapoint
+}
+
+type Config struct {
+	ServiceName      string
+	StatsdServers    string
+	DogStatsdServers string
+}
+
+// NewManager initializes metrics plugins and returns a new metrics manager
+func NewManager(ctx context.Context, conf Config) (*Manager, error) {
+	if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
+		return nil, nil
+	}
+
+	if conf.ServiceName == "" {
+		hostname, err := os.Hostname()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get hostname: %w", err)
+		}
+		conf.ServiceName = hostname
+	}
+
+	addDataChan := make(chan datapoint, dataItemCount)
+
+	mgr := &Manager{
+		addDataChan: addDataChan,
+		ctx:         ctx,
+		config:      conf,
+	}
+
+	// setup statsd endpoints
+	if len(conf.StatsdServers) > 0 {
+		statsdServers := strings.Split(conf.StatsdServers, ",")
+
+		for _, server := range statsdServers {
+			statsd, err := newStatsd(server, conf.ServiceName)
+			if err != nil {
+				return nil, err
+			}
+			mgr.publishers = append(mgr.publishers, statsd)
+		}
+	}
+
+	// setup dogstatsd endpoints
+	if len(conf.DogStatsdServers) > 0 {
+		dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
+
+		for _, server := range dogStatsdServers {
+			dogStatsd, err := newDogStatsd(server, conf.ServiceName)
+			if err != nil {
+				return nil, err
+			}
+			mgr.publishers = append(mgr.publishers, dogStatsd)
+		}
+	}
+
+	mgr.wg.Add(1)
+	go mgr.addForwarder(addDataChan)
+
+	return mgr, nil
+}
+
+func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) {
+	// In case of Authentication failures, url parsing ...
+	if action == "" {
+		action = ActionUndetected
+	}
+
+	a := ActionMap[action]
+	reqTags := []Tag{
+		{Key: "method", Value: ctx.Method()},
+		{Key: "api", Value: a.Service},
+		{Key: "action", Value: a.Name},
+	}
+
+	reqStatus := status
+
+	if err != nil {
+		var apierr s3err.APIError
+		if errors.As(err, &apierr) {
+			reqStatus = apierr.HTTPStatusCode
+		} else {
+			reqStatus = http.StatusInternalServerError
+		}
+	}
+	if reqStatus == 0 {
+		reqStatus = http.StatusOK
+	}
+
+	reqTags = append(reqTags, Tag{
+		Key:   "status",
+		Value: fmt.Sprintf("%v", reqStatus),
+	})
+
+	if err != nil {
+		m.increment("failed_count", reqTags...)
+	} else {
+		m.increment("success_count", reqTags...)
+	}
+
+	switch action {
+	case ActionPutObject:
+		m.add("bytes_written", count, reqTags...)
+		m.increment("object_created_count", reqTags...)
+	case ActionCompleteMultipartUpload:
+		m.increment("object_created_count", reqTags...)
+	case ActionUploadPart:
+		m.add("bytes_written", count, reqTags...)
+	case ActionGetObject:
+		m.add("bytes_read", count, reqTags...)
+	case ActionDeleteObject:
+		m.increment("object_removed_count", reqTags...)
+	case ActionDeleteObjects:
+		m.add("object_removed_count", count, reqTags...)
+	}
+}
+
+// increment increments the key by one
+func (m *Manager) increment(key string, tags ...Tag) {
+	m.add(key, 1, tags...)
+}
+
+// add adds value to key
+func (m *Manager) add(key string, value int64, tags ...Tag) {
+	if m.ctx.Err() != nil {
+		return
+	}
+
+	d := datapoint{
+		key:   key,
+		value: value,
+		tags:  tags,
+	}
+
+	select {
+	case m.addDataChan <- d:
+	default:
+		// channel full, drop the updates
+	}
+}
+
+// Close closes metrics channels, waits for data to complete, closes all plugins
+func (m *Manager) Close() {
+	// drain the datapoint channels
+	close(m.addDataChan)
+	m.wg.Wait()
+
+	// close all publishers
+	for _, p := range m.publishers {
+		p.Close()
+	}
+}
+
+// publisher is the interface for interacting with the metrics plugins
+type publisher interface {
+	Add(key string, value int64, tags ...Tag)
+	Close()
+}
+
+func (m *Manager) addForwarder(addChan <-chan datapoint) {
+	for data := range addChan {
+		for _, s := range m.publishers {
+			s.Add(data.key, data.value, data.tags...)
+		}
+	}
+	m.wg.Done()
+}
+
+type datapoint struct {
+	key   string
+	value int64
+	tags  []Tag
+}

metrics/statsd.go

@@ -0,0 +1,51 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+import (
+	"github.com/smira/go-statsd"
+)
+
+// vgwStatsd metrics type
+type vgwStatsd struct {
+	c *statsd.Client
+}
+
+// newStatsd takes a server address and returns a statsd merics
+// Supply service name to be used as a tag to identify the spcific
+// gateway instance, this may typically be the gateway hostname
+func newStatsd(server string, service string) (*vgwStatsd, error) {
+	c := statsd.NewClient(
+		server,
+		statsd.MetricPrefix("versitygw."),
+		statsd.TagStyle(statsd.TagFormatInfluxDB),
+		statsd.DefaultTags(statsd.StringTag("service", service)),
+	)
+	return &vgwStatsd{c: c}, nil
+}
+
+// Close closes statsd connections
+func (s *vgwStatsd) Close() {
+	s.c.Close()
+}
+
+// Add adds value to key
+func (s *vgwStatsd) Add(key string, value int64, tags ...Tag) {
+	stags := make([]statsd.Tag, len(tags))
+	for i, t := range tags {
+		stags[i] = statsd.StringTag(t.Key, t.Value)
+	}
+	s.c.Incr(key, value, stags...)
+}

s3api/admin-router.go

@@ -32,6 +32,9 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
 	// DeleteUsers admin api
 	app.Patch("/delete-user", controller.DeleteUser)
 
+	// UpdateUser admin api
+	app.Patch("/update-user", controller.UpdateUser)
+
 	// ListUsers admin api
 	app.Patch("/list-users", controller.ListUsers)
 

s3api/admin-server.go

@@ -46,12 +46,11 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 
 	// Logging middlewares
 	app.Use(logger.New())
-	app.Use(middlewares.DecodeURL(nil))
+	app.Use(middlewares.DecodeURL(nil, nil))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
+	app.Use(middlewares.VerifyV4Signature(root, iam, nil, nil, region, false))
 	app.Use(middlewares.VerifyMD5Body(nil))
-	app.Use(middlewares.AclParser(be, nil))
 
 	server.router.Init(app, be, iam)
 

s3api/controllers/admin.go

@@ -17,6 +17,7 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
@@ -35,31 +36,69 @@ func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	var usr auth.Account
 	err := json.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return fmt.Errorf("failed to parse request body: %w", err)
+		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("failed to parse request body: %w", err).Error())
 	}
 
 	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
+		return ctx.Status(fiber.StatusBadRequest).SendString("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		return fmt.Errorf("failed to create user: %w", err)
+		status := fiber.StatusInternalServerError
+		msg := fmt.Errorf("failed to create user: %w", err).Error()
+
+		if strings.Contains(msg, "user already exists") {
+			status = fiber.StatusConflict
+		}
+
+		return ctx.Status(status).SendString(msg)
 	}
 
-	return ctx.SendString("The user has been created successfully")
+	return ctx.Status(fiber.StatusCreated).SendString("The user has been created successfully")
+}
+
+func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
+	}
+
+	access := ctx.Query("access")
+	if access == "" {
+		return ctx.Status(fiber.StatusBadRequest).SendString("missing user access parameter")
+	}
+
+	var props auth.MutableProps
+	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
+		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("invalid request body %w", err).Error())
+	}
+
+	err := c.iam.UpdateUserAccount(access, props)
+	if err != nil {
+		status := fiber.StatusInternalServerError
+		msg := fmt.Errorf("failed to update user account: %w", err).Error()
+
+		if strings.Contains(msg, "user not found") {
+			status = fiber.StatusNotFound
+		}
+
+		return ctx.Status(status).SendString(msg)
+	}
+
+	return ctx.SendString("the user has been updated successfully")
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 
 	err := c.iam.DeleteUserAccount(access)
@@ -73,7 +112,7 @@ func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	accs, err := c.iam.ListUserAccounts()
 	if err != nil {
@@ -86,7 +125,7 @@ func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
@@ -96,7 +135,7 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user specified as the new bucket owner does not exist")
+		return ctx.Status(fiber.StatusNotFound).SendString("user specified as the new bucket owner does not exist")
 	}
 
 	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
@@ -104,13 +143,13 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		return err
 	}
 
-	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+	return ctx.SendString("Bucket owner has been updated successfully")
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())

s3api/controllers/admin_test.go

@@ -85,7 +85,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 201,
 		},
 		{
 			name: "Admin-create-user-invalid-user-role",
@@ -94,7 +94,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
@@ -103,7 +103,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -119,6 +119,122 @@ func TestAdminController_CreateUser(t *testing.T) {
 	}
 }
 
+func TestAdminController_UpdateUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/update-user", adminController.UpdateUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/update-user", adminController.UpdateUser)
+
+	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	appNotFound := fiber.New()
+
+	appNotFound.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-update-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-update-user-missing-access",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-request-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
+		{
+			name: "Admin-update-user-not-found",
+			app:  appNotFound,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 404,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
 func TestAdminController_DeleteUser(t *testing.T) {
 	type args struct {
 		req *http.Request
@@ -173,7 +289,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -251,7 +367,7 @@ func TestAdminController_ListUsers(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Admin-list-users-iam-error",
@@ -368,7 +484,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -386,7 +502,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 404,
 		},
 		{
 			name: "Change-bucket-owner-success",
@@ -395,7 +511,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 201,
+			statusCode: 200,
 		},
 	}
 	for _, tt := range tests {
@@ -455,7 +571,7 @@ func TestAdminController_ListBuckets(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "List-buckets-success",

s3api/controllers/backend_moq_test.go

@@ -7,9 +7,9 @@ import (
 	"bufio"
 	"context"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3response"
-	"io"
 	"sync"
 )
 
@@ -44,6 +44,9 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketOwnershipControls method")
+//			},
 //			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketPolicy method")
 //			},
@@ -62,6 +65,9 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+//				panic("mock out the GetBucketOwnershipControls method")
+//			},
 //			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 //				panic("mock out the GetBucketPolicy method")
 //			},
@@ -71,13 +77,13 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
 //				panic("mock out the GetBucketVersioning method")
 //			},
-//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 //				panic("mock out the GetObject method")
 //			},
 //			GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 //				panic("mock out the GetObjectAcl method")
 //			},
-//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 //				panic("mock out the GetObjectAttributes method")
 //			},
 //			GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error) {
@@ -122,6 +128,9 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+//				panic("mock out the PutBucketOwnershipControls method")
+//			},
 //			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
 //				panic("mock out the PutBucketPolicy method")
 //			},
@@ -143,7 +152,7 @@ var _ backend.Backend = &BackendMock{}
 //			PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
 //				panic("mock out the PutObjectLockConfiguration method")
 //			},
-//			PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string, retention []byte) error {
+//			PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string, bypass bool, retention []byte) error {
 //				panic("mock out the PutObjectRetention method")
 //			},
 //			PutObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error {
@@ -195,6 +204,9 @@ type BackendMock struct {
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
+	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
+	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
 	DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -213,6 +225,9 @@ type BackendMock struct {
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketOwnershipControlsFunc mocks the GetBucketOwnershipControls method.
+	GetBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error)
+
 	// GetBucketPolicyFunc mocks the GetBucketPolicy method.
 	GetBucketPolicyFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
 
@@ -223,13 +238,13 @@ type BackendMock struct {
 	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 
 	// GetObjectFunc mocks the GetObject method.
-	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 
 	// GetObjectAclFunc mocks the GetObjectAcl method.
 	GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 
 	// GetObjectAttributesFunc mocks the GetObjectAttributes method.
-	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
 
 	// GetObjectLegalHoldFunc mocks the GetObjectLegalHold method.
 	GetObjectLegalHoldFunc func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error)
@@ -273,6 +288,9 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketOwnershipControlsFunc mocks the PutBucketOwnershipControls method.
+	PutBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error
+
 	// PutBucketPolicyFunc mocks the PutBucketPolicy method.
 	PutBucketPolicyFunc func(contextMoqParam context.Context, bucket string, policy []byte) error
 
@@ -295,7 +313,7 @@ type BackendMock struct {
 	PutObjectLockConfigurationFunc func(contextMoqParam context.Context, bucket string, config []byte) error
 
 	// PutObjectRetentionFunc mocks the PutObjectRetention method.
-	PutObjectRetentionFunc func(contextMoqParam context.Context, bucket string, object string, versionId string, retention []byte) error
+	PutObjectRetentionFunc func(contextMoqParam context.Context, bucket string, object string, versionId string, bypass bool, retention []byte) error
 
 	// PutObjectTaggingFunc mocks the PutObjectTagging method.
 	PutObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error
@@ -373,6 +391,13 @@ type BackendMock struct {
 			// DeleteBucketInput is the deleteBucketInput argument value.
 			DeleteBucketInput *s3.DeleteBucketInput
 		}
+		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
+		DeleteBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
 		DeleteBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -417,6 +442,13 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketOwnershipControls holds details about calls to the GetBucketOwnershipControls method.
+		GetBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetBucketPolicy holds details about calls to the GetBucketPolicy method.
 		GetBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -444,8 +476,6 @@ type BackendMock struct {
 			ContextMoqParam context.Context
 			// GetObjectInput is the getObjectInput argument value.
 			GetObjectInput *s3.GetObjectInput
-			// Writer is the writer argument value.
-			Writer io.Writer
 		}
 		// GetObjectAcl holds details about calls to the GetObjectAcl method.
 		GetObjectAcl []struct {
@@ -571,6 +601,15 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketOwnershipControls holds details about calls to the PutBucketOwnershipControls method.
+		PutBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Ownership is the ownership argument value.
+			Ownership types.ObjectOwnership
+		}
 		// PutBucketPolicy holds details about calls to the PutBucketPolicy method.
 		PutBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -642,6 +681,8 @@ type BackendMock struct {
 			Object string
 			// VersionId is the versionId argument value.
 			VersionId string
+			// Bypass is the bypass argument value.
+			Bypass bool
 			// Retention is the retention argument value.
 			Retention []byte
 		}
@@ -691,54 +732,57 @@ type BackendMock struct {
 			UploadPartCopyInput *s3.UploadPartCopyInput
 		}
 	}
-	lockAbortMultipartUpload       sync.RWMutex
-	lockChangeBucketOwner          sync.RWMutex
-	lockCompleteMultipartUpload    sync.RWMutex
-	lockCopyObject                 sync.RWMutex
-	lockCreateBucket               sync.RWMutex
-	lockCreateMultipartUpload      sync.RWMutex
-	lockDeleteBucket               sync.RWMutex
-	lockDeleteBucketPolicy         sync.RWMutex
-	lockDeleteBucketTagging        sync.RWMutex
-	lockDeleteObject               sync.RWMutex
-	lockDeleteObjectTagging        sync.RWMutex
-	lockDeleteObjects              sync.RWMutex
-	lockGetBucketAcl               sync.RWMutex
-	lockGetBucketPolicy            sync.RWMutex
-	lockGetBucketTagging           sync.RWMutex
-	lockGetBucketVersioning        sync.RWMutex
-	lockGetObject                  sync.RWMutex
-	lockGetObjectAcl               sync.RWMutex
-	lockGetObjectAttributes        sync.RWMutex
-	lockGetObjectLegalHold         sync.RWMutex
-	lockGetObjectLockConfiguration sync.RWMutex
-	lockGetObjectRetention         sync.RWMutex
-	lockGetObjectTagging           sync.RWMutex
-	lockHeadBucket                 sync.RWMutex
-	lockHeadObject                 sync.RWMutex
-	lockListBuckets                sync.RWMutex
-	lockListBucketsAndOwners       sync.RWMutex
-	lockListMultipartUploads       sync.RWMutex
-	lockListObjectVersions         sync.RWMutex
-	lockListObjects                sync.RWMutex
-	lockListObjectsV2              sync.RWMutex
-	lockListParts                  sync.RWMutex
-	lockPutBucketAcl               sync.RWMutex
-	lockPutBucketPolicy            sync.RWMutex
-	lockPutBucketTagging           sync.RWMutex
-	lockPutBucketVersioning        sync.RWMutex
-	lockPutObject                  sync.RWMutex
-	lockPutObjectAcl               sync.RWMutex
-	lockPutObjectLegalHold         sync.RWMutex
-	lockPutObjectLockConfiguration sync.RWMutex
-	lockPutObjectRetention         sync.RWMutex
-	lockPutObjectTagging           sync.RWMutex
-	lockRestoreObject              sync.RWMutex
-	lockSelectObjectContent        sync.RWMutex
-	lockShutdown                   sync.RWMutex
-	lockString                     sync.RWMutex
-	lockUploadPart                 sync.RWMutex
-	lockUploadPartCopy             sync.RWMutex
+	lockAbortMultipartUpload          sync.RWMutex
+	lockChangeBucketOwner             sync.RWMutex
+	lockCompleteMultipartUpload       sync.RWMutex
+	lockCopyObject                    sync.RWMutex
+	lockCreateBucket                  sync.RWMutex
+	lockCreateMultipartUpload         sync.RWMutex
+	lockDeleteBucket                  sync.RWMutex
+	lockDeleteBucketOwnershipControls sync.RWMutex
+	lockDeleteBucketPolicy            sync.RWMutex
+	lockDeleteBucketTagging           sync.RWMutex
+	lockDeleteObject                  sync.RWMutex
+	lockDeleteObjectTagging           sync.RWMutex
+	lockDeleteObjects                 sync.RWMutex
+	lockGetBucketAcl                  sync.RWMutex
+	lockGetBucketOwnershipControls    sync.RWMutex
+	lockGetBucketPolicy               sync.RWMutex
+	lockGetBucketTagging              sync.RWMutex
+	lockGetBucketVersioning           sync.RWMutex
+	lockGetObject                     sync.RWMutex
+	lockGetObjectAcl                  sync.RWMutex
+	lockGetObjectAttributes           sync.RWMutex
+	lockGetObjectLegalHold            sync.RWMutex
+	lockGetObjectLockConfiguration    sync.RWMutex
+	lockGetObjectRetention            sync.RWMutex
+	lockGetObjectTagging              sync.RWMutex
+	lockHeadBucket                    sync.RWMutex
+	lockHeadObject                    sync.RWMutex
+	lockListBuckets                   sync.RWMutex
+	lockListBucketsAndOwners          sync.RWMutex
+	lockListMultipartUploads          sync.RWMutex
+	lockListObjectVersions            sync.RWMutex
+	lockListObjects                   sync.RWMutex
+	lockListObjectsV2                 sync.RWMutex
+	lockListParts                     sync.RWMutex
+	lockPutBucketAcl                  sync.RWMutex
+	lockPutBucketOwnershipControls    sync.RWMutex
+	lockPutBucketPolicy               sync.RWMutex
+	lockPutBucketTagging              sync.RWMutex
+	lockPutBucketVersioning           sync.RWMutex
+	lockPutObject                     sync.RWMutex
+	lockPutObjectAcl                  sync.RWMutex
+	lockPutObjectLegalHold            sync.RWMutex
+	lockPutObjectLockConfiguration    sync.RWMutex
+	lockPutObjectRetention            sync.RWMutex
+	lockPutObjectTagging              sync.RWMutex
+	lockRestoreObject                 sync.RWMutex
+	lockSelectObjectContent           sync.RWMutex
+	lockShutdown                      sync.RWMutex
+	lockString                        sync.RWMutex
+	lockUploadPart                    sync.RWMutex
+	lockUploadPartCopy                sync.RWMutex
 }
 
 // AbortMultipartUpload calls AbortMultipartUploadFunc.
@@ -1001,6 +1045,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketOwnershipControls calls DeleteBucketOwnershipControlsFunc.
+func (mock *BackendMock) DeleteBucketOwnershipControls(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.DeleteBucketOwnershipControlsFunc: method is nil but Backend.DeleteBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketOwnershipControls.Lock()
+	mock.calls.DeleteBucketOwnershipControls = append(mock.calls.DeleteBucketOwnershipControls, callInfo)
+	mock.lockDeleteBucketOwnershipControls.Unlock()
+	return mock.DeleteBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketOwnershipControlsCalls gets all the calls that were made to DeleteBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketOwnershipControlsCalls())
+func (mock *BackendMock) DeleteBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketOwnershipControls.RLock()
+	calls = mock.calls.DeleteBucketOwnershipControls
+	mock.lockDeleteBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // DeleteBucketPolicy calls DeleteBucketPolicyFunc.
 func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketPolicyFunc == nil {
@@ -1221,6 +1301,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketOwnershipControls calls GetBucketOwnershipControlsFunc.
+func (mock *BackendMock) GetBucketOwnershipControls(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+	if mock.GetBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.GetBucketOwnershipControlsFunc: method is nil but Backend.GetBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketOwnershipControls.Lock()
+	mock.calls.GetBucketOwnershipControls = append(mock.calls.GetBucketOwnershipControls, callInfo)
+	mock.lockGetBucketOwnershipControls.Unlock()
+	return mock.GetBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// GetBucketOwnershipControlsCalls gets all the calls that were made to GetBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketOwnershipControlsCalls())
+func (mock *BackendMock) GetBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketOwnershipControls.RLock()
+	calls = mock.calls.GetBucketOwnershipControls
+	mock.lockGetBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // GetBucketPolicy calls GetBucketPolicyFunc.
 func (mock *BackendMock) GetBucketPolicy(contextMoqParam context.Context, bucket string) ([]byte, error) {
 	if mock.GetBucketPolicyFunc == nil {
@@ -1330,23 +1446,21 @@ func (mock *BackendMock) GetBucketVersioningCalls() []struct {
 }
 
 // GetObject calls GetObjectFunc.
-func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	if mock.GetObjectFunc == nil {
 		panic("BackendMock.GetObjectFunc: method is nil but Backend.GetObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}{
 		ContextMoqParam: contextMoqParam,
 		GetObjectInput:  getObjectInput,
-		Writer:          writer,
 	}
 	mock.lockGetObject.Lock()
 	mock.calls.GetObject = append(mock.calls.GetObject, callInfo)
 	mock.lockGetObject.Unlock()
-	return mock.GetObjectFunc(contextMoqParam, getObjectInput, writer)
+	return mock.GetObjectFunc(contextMoqParam, getObjectInput)
 }
 
 // GetObjectCalls gets all the calls that were made to GetObject.
@@ -1356,12 +1470,10 @@ func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInp
 func (mock *BackendMock) GetObjectCalls() []struct {
 	ContextMoqParam context.Context
 	GetObjectInput  *s3.GetObjectInput
-	Writer          io.Writer
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}
 	mock.lockGetObject.RLock()
 	calls = mock.calls.GetObject
@@ -1406,7 +1518,7 @@ func (mock *BackendMock) GetObjectAclCalls() []struct {
 }
 
 // GetObjectAttributes calls GetObjectAttributesFunc.
-func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 	if mock.GetObjectAttributesFunc == nil {
 		panic("BackendMock.GetObjectAttributesFunc: method is nil but Backend.GetObjectAttributes was just called")
 	}
@@ -1969,6 +2081,46 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketOwnershipControls calls PutBucketOwnershipControlsFunc.
+func (mock *BackendMock) PutBucketOwnershipControls(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+	if mock.PutBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.PutBucketOwnershipControlsFunc: method is nil but Backend.PutBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Ownership:       ownership,
+	}
+	mock.lockPutBucketOwnershipControls.Lock()
+	mock.calls.PutBucketOwnershipControls = append(mock.calls.PutBucketOwnershipControls, callInfo)
+	mock.lockPutBucketOwnershipControls.Unlock()
+	return mock.PutBucketOwnershipControlsFunc(contextMoqParam, bucket, ownership)
+}
+
+// PutBucketOwnershipControlsCalls gets all the calls that were made to PutBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketOwnershipControlsCalls())
+func (mock *BackendMock) PutBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Ownership       types.ObjectOwnership
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}
+	mock.lockPutBucketOwnershipControls.RLock()
+	calls = mock.calls.PutBucketOwnershipControls
+	mock.lockPutBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // PutBucketPolicy calls PutBucketPolicyFunc.
 func (mock *BackendMock) PutBucketPolicy(contextMoqParam context.Context, bucket string, policy []byte) error {
 	if mock.PutBucketPolicyFunc == nil {
@@ -2246,7 +2398,7 @@ func (mock *BackendMock) PutObjectLockConfigurationCalls() []struct {
 }
 
 // PutObjectRetention calls PutObjectRetentionFunc.
-func (mock *BackendMock) PutObjectRetention(contextMoqParam context.Context, bucket string, object string, versionId string, retention []byte) error {
+func (mock *BackendMock) PutObjectRetention(contextMoqParam context.Context, bucket string, object string, versionId string, bypass bool, retention []byte) error {
 	if mock.PutObjectRetentionFunc == nil {
 		panic("BackendMock.PutObjectRetentionFunc: method is nil but Backend.PutObjectRetention was just called")
 	}
@@ -2255,18 +2407,20 @@ func (mock *BackendMock) PutObjectRetention(contextMoqParam context.Context, buc
 		Bucket          string
 		Object          string
 		VersionId       string
+		Bypass          bool
 		Retention       []byte
 	}{
 		ContextMoqParam: contextMoqParam,
 		Bucket:          bucket,
 		Object:          object,
 		VersionId:       versionId,
+		Bypass:          bypass,
 		Retention:       retention,
 	}
 	mock.lockPutObjectRetention.Lock()
 	mock.calls.PutObjectRetention = append(mock.calls.PutObjectRetention, callInfo)
 	mock.lockPutObjectRetention.Unlock()
-	return mock.PutObjectRetentionFunc(contextMoqParam, bucket, object, versionId, retention)
+	return mock.PutObjectRetentionFunc(contextMoqParam, bucket, object, versionId, bypass, retention)
 }
 
 // PutObjectRetentionCalls gets all the calls that were made to PutObjectRetention.
@@ -2278,6 +2432,7 @@ func (mock *BackendMock) PutObjectRetentionCalls() []struct {
 	Bucket          string
 	Object          string
 	VersionId       string
+	Bypass          bool
 	Retention       []byte
 } {
 	var calls []struct {
@@ -2285,6 +2440,7 @@ func (mock *BackendMock) PutObjectRetentionCalls() []struct {
 		Bucket          string
 		Object          string
 		VersionId       string
+		Bypass          bool
 		Retention       []byte
 	}
 	mock.lockPutObjectRetention.RLock()

s3api/controllers/base_test.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -77,7 +76,7 @@ func TestNew(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := New(tt.args.be, tt.args.iam, nil, nil, false)
+			got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false)
 			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("New() = %v, want %v", got, tt.want)
 			}
@@ -188,10 +187,10 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 				return &s3.GetObjectAclOutput{}, nil
 			},
-			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
-				return &s3.GetObjectAttributesOutput{}, nil
+			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+				return s3response.GetObjectAttributesResult{}, nil
 			},
-			GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+			GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
 					Metadata:        map[string]string{"hello": "world"},
 					ContentType:     getPtr("application/xml"),
@@ -395,6 +394,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 				return objectLockResult, nil
 			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerEnforced, nil
+			},
 		},
 	}
 
@@ -448,6 +450,15 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 404,
 		},
+		{
+			name: "Get-bucket-ownership-control-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-bucket-tagging-success",
 			app:  app,
@@ -562,7 +573,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	app := fiber.New()
 
 	// Mock valid acl
-	acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
+	acl := auth.ACL{Owner: "valid access"}
 	acldata, err := json.Marshal(acl)
 	if err != nil {
 		t.Errorf("Failed to parse the params: %v", err.Error())
@@ -593,14 +604,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</AccessControlPolicy>
 	`
 
-	succBody := `
-	<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-		<Owner>
-			<ID>valid access</ID>
-		</Owner>
-	</AccessControlPolicy>
-	`
-
 	tagBody := `
 	<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
 		<TagSet>
@@ -644,6 +647,22 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</ObjectLockConfiguration>
 	`
 
+	ownershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
+	invalidOwnershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>invalid_value</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -667,6 +686,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
 				return nil
 			},
+			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+				return nil
+			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerPreferred, nil
+			},
 		},
 	}
 	// Mock ctx.Locals
@@ -690,16 +715,18 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 
 	// PutBucketAcl incorrect bucket owner case
 	incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
-	incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
 
 	// PutBucketAcl acl success
-	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
+	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
 	aclSuccReq.Header.Set("X-Amz-Acl", "private")
 
 	// Invalid acl body case
 	errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
 	errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
+	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -725,6 +752,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Put-bucket-ownership-controls-invalid-ownership",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Put-object-lock-configuration-invalid-body",
 			app:  app,
@@ -825,7 +870,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 200,
 		},
 		{
-			name: "Put-bucket-invalid-bucket-name",
+			name: "Create-bucket-invalid-acl-ownership-combination",
+			app:  app,
+			args: args{
+				req: invAclOwnershipReq,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Create-bucket-invalid-bucket-name",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
@@ -834,7 +888,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 400,
 		},
 		{
-			name: "Put-bucket-success",
+			name: "Create-bucket-success",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
@@ -896,7 +950,7 @@ func TestS3ApiController_PutActions(t *testing.T) {
 
 	legalHoldBody := `
 	<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-		<Status>string</Status>
+		<Status>ON</Status>
 	</LegalHold>
 	`
 
@@ -929,7 +983,7 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
 				return nil
 			},
-			PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, retention []byte) error {
+			PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
 				return nil
 			},
 			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
@@ -1169,6 +1223,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 				return nil
 			},
+			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
+			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
 		},
 	}
 
@@ -1207,6 +1267,23 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			wantErr:    false,
 			statusCode: 204,
 		},
+		{
+			name: "Delete-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		}, {
+			name: "Delete-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -1655,20 +1732,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 		{
 			name: "Restore-object-success",
 			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
-		{
-			name: "Restore-object-error",
-			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 200,
 		},
 		{
 			name: "Select-object-content-invalid-body",

s3api/controllers/iam_moq_test.go

@@ -33,6 +33,9 @@ var _ auth.IAMService = &IAMServiceMock{}
 //			ShutdownFunc: func() error {
 //				panic("mock out the Shutdown method")
 //			},
+//			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+//				panic("mock out the UpdateUserAccount method")
+//			},
 //		}
 //
 //		// use mockedIAMService in code that requires auth.IAMService
@@ -55,6 +58,9 @@ type IAMServiceMock struct {
 	// ShutdownFunc mocks the Shutdown method.
 	ShutdownFunc func() error
 
+	// UpdateUserAccountFunc mocks the UpdateUserAccount method.
+	UpdateUserAccountFunc func(access string, props auth.MutableProps) error
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// CreateAccount holds details about calls to the CreateAccount method.
@@ -78,12 +84,20 @@ type IAMServiceMock struct {
 		// Shutdown holds details about calls to the Shutdown method.
 		Shutdown []struct {
 		}
+		// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
+		UpdateUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+			// Props is the props argument value.
+			Props auth.MutableProps
+		}
 	}
 	lockCreateAccount     sync.RWMutex
 	lockDeleteUserAccount sync.RWMutex
 	lockGetUserAccount    sync.RWMutex
 	lockListUserAccounts  sync.RWMutex
 	lockShutdown          sync.RWMutex
+	lockUpdateUserAccount sync.RWMutex
 }
 
 // CreateAccount calls CreateAccountFunc.
@@ -235,3 +249,39 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
 	mock.lockShutdown.RUnlock()
 	return calls
 }
+
+// UpdateUserAccount calls UpdateUserAccountFunc.
+func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
+	if mock.UpdateUserAccountFunc == nil {
+		panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+		Props  auth.MutableProps
+	}{
+		Access: access,
+		Props:  props,
+	}
+	mock.lockUpdateUserAccount.Lock()
+	mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
+	mock.lockUpdateUserAccount.Unlock()
+	return mock.UpdateUserAccountFunc(access, props)
+}
+
+// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.UpdateUserAccountCalls())
+func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
+	Access string
+	Props  auth.MutableProps
+} {
+	var calls []struct {
+		Access string
+		Props  auth.MutableProps
+	}
+	mock.lockUpdateUserAccount.RLock()
+	calls = mock.calls.UpdateUserAccount
+	mock.lockUpdateUserAccount.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -24,6 +24,7 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
@@ -31,7 +32,7 @@ var (
 	singlePath = regexp.MustCompile(`^/[^/]+/?$`)
 )
 
-func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
+func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
 		path := ctx.Path()
@@ -49,10 +50,18 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 			!ctx.Request().URI().QueryArgs().Has("tagging") &&
 			!ctx.Request().URI().QueryArgs().Has("versioning") &&
 			!ctx.Request().URI().QueryArgs().Has("policy") &&
-			!ctx.Request().URI().QueryArgs().Has("object-lock") {
+			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
+			!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
+			if readonly {
+				return controllers.SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrAccessDenied),
+					&controllers.MetaOpts{
+						Logger: logger,
+						Action: "CreateBucket",
+					})
+			}
 			return ctx.Next()
 		}
 		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})

s3api/middlewares/authentication.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
@@ -40,7 +41,7 @@ type RootUserConfig struct {
 	Secret string
 }
 
-func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
+func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
@@ -54,16 +55,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
 		if authorization == "" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
 		}
 
 		authData, err := utils.ParseAuthorization(authorization)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		if authData.Algorithm != "AWS4-HMAC-SHA256" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger, mm)
 		}
 
 		if authData.Region != region {
@@ -71,40 +72,40 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 				Code:           "SignatureDoesNotMatch",
 				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
 				HTTPStatusCode: http.StatusForbidden,
-			}, logger)
+			}, logger, mm)
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
 		}
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 		ctx.Locals("account", account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
 		if date == "" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm)
 		}
 
 		// Parse the date and check the date validity
 		tdate, err := time.Parse(iso8601Format, date)
 		if err != nil {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm)
 		}
 
 		if date[:8] != authData.Date {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm)
 		}
 
 		// Validate the dates difference
 		err = utils.ValidateDate(tdate)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		if utils.IsBigDataAction(ctx) {
@@ -125,7 +126,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 
 			// Compare the calculated hash with the hash provided
 			if hashPayload != hexPayload {
-				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm)
 			}
 		}
 
@@ -134,13 +135,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if contentLengthStr != "" {
 			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
 			if err != nil {
-				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
 			}
 		}
 
 		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		return ctx.Next()
@@ -164,6 +165,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
 	return a.iam.GetUserAccount(access)
 }
 
-func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
-	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error {
+	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 }

s3api/middlewares/chunk.go

@@ -20,13 +20,14 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3log"
 )
 
 // ProcessChunkedBody initializes the chunked upload stream if the
 // request appears to be a chunked upload
-func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
+func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
 		if decodedLength == "" {
@@ -36,7 +37,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
 
 		authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		acct := ctx.Locals("account").(auth.Account)
@@ -51,7 +52,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
 				return cr
 			})
 			if err != nil {
-				return sendResponse(ctx, err, logger)
+				return sendResponse(ctx, err, logger, mm)
 			}
 			return ctx.Next()
 		}

s3api/middlewares/presign-auth.go

@@ -20,12 +20,13 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
-func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
+func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
@@ -38,16 +39,16 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 
 		authData, err := utils.ParsePresignedURIParts(ctx)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
 		}
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 		ctx.Locals("account", account)
 
@@ -61,7 +62,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 
 		err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		return ctx.Next()

s3api/middlewares/url-decoder.go

@@ -18,17 +18,18 @@ import (
 	"net/url"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
-func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
+func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		reqURL := ctx.Request().URI().String()
 		decoded, err := url.Parse(reqURL)
 		if err != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}
 		ctx.Path(decoded.Path)
 		return ctx.Next()

s3api/router.go

@@ -18,6 +18,7 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
@@ -27,8 +28,8 @@ type S3ApiRouter struct {
 	WithAdmSrv bool
 }
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
-	s3ApiController := controllers.New(be, iam, logger, evs, debug)
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
+	s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
 
 	if sa.WithAdmSrv {
 		adminController := controllers.NewAdminController(iam, be)
@@ -39,6 +40,9 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		// DeleteUsers admin api
 		app.Patch("/delete-user", adminController.DeleteUser)
 
+		// UpdateUser admin api
+		app.Patch("update-user", adminController.UpdateUser)
+
 		// ListUsers admin api
 		app.Patch("/list-users", adminController.ListUsers)
 

s3api/router_test.go

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, false, false)
 		})
 	}
 }

s3api/server.go

@@ -22,23 +22,35 @@ import (
 	"github.com/gofiber/fiber/v2/middleware/logger"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
 )
 
 type S3ApiServer struct {
-	app     *fiber.App
-	backend backend.Backend
-	router  *S3ApiRouter
-	port    string
-	cert    *tls.Certificate
-	quiet   bool
-	debug   bool
-	health  string
+	app      *fiber.App
+	backend  backend.Backend
+	router   *S3ApiRouter
+	port     string
+	cert     *tls.Certificate
+	quiet    bool
+	debug    bool
+	readonly bool
+	health   string
 }
 
-func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
+func New(
+	app *fiber.App,
+	be backend.Backend,
+	root middlewares.RootUserConfig,
+	port, region string,
+	iam auth.IAMService,
+	l s3log.AuditLogger,
+	evs s3event.S3EventSender,
+	mm *metrics.Manager,
+	opts ...Option,
+) (*S3ApiServer, error) {
 	server := &S3ApiServer{
 		app:     app,
 		backend: be,
@@ -60,17 +72,17 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 			return ctx.SendStatus(http.StatusOK)
 		})
 	}
-	app.Use(middlewares.DecodeURL(l))
+	app.Use(middlewares.DecodeURL(l, mm))
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
-	app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
-	app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
+	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
+	app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
+	app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
 	app.Use(middlewares.VerifyMD5Body(l))
-	app.Use(middlewares.AclParser(be, l))
+	app.Use(middlewares.AclParser(be, l, server.readonly))
 
-	server.router.Init(app, be, iam, l, evs, server.debug)
+	server.router.Init(app, be, iam, l, evs, mm, server.debug, server.readonly)
 
 	return server, nil
 }
@@ -103,6 +115,10 @@ func WithHealth(health string) Option {
 	return func(s *S3ApiServer) { s.health = health }
 }
 
+func WithReadOnly() Option {
+	return func(s *S3ApiServer) { s.readonly = true }
+}
+
 func (sa *S3ApiServer) Serve() (err error) {
 	if sa.cert != nil {
 		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

s3api/server_test.go

@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
-				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
+				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
 				return

s3api/utils/auth_test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package utils
 
 import (

s3api/utils/chunk-reader.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"hash"
 	"io"
+	"math"
 	"strconv"
 	"time"
 
@@ -192,6 +193,9 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		cr.chunkDataLeft = 0
 		cr.chunkHash.Write(p[:chunkSize])
 		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		if (chunkSize + int64(n)) > math.MaxInt {
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
 		return n + int(chunkSize), err
 	}
 

s3api/utils/utils.go

@@ -31,6 +31,7 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
 var (
@@ -223,24 +224,17 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
 	return false
 }
 
+// expiration time window
+// https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationTimeStamp
+const timeExpirationSec = 15 * 60
+
 func ValidateDate(date time.Time) error {
 	now := time.Now().UTC()
 	diff := date.Unix() - now.Unix()
 
-	// Checks the dates difference to be less than a minute
-	if diff > 60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
-	}
-	if diff < -60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
+	// Checks the dates difference to be within allotted window
+	if diff > timeExpirationSec || diff < -timeExpirationSec {
+		return s3err.GetAPIError(s3err.ErrRequestTimeTooSkewed)
 	}
 
 	return nil
@@ -253,3 +247,95 @@ func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
 
 	return
 }
+
+func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResult) s3response.GetObjectAttributesResult {
+	if _, ok := attrs[types.ObjectAttributesEtag]; !ok {
+		output.ETag = nil
+	}
+	if _, ok := attrs[types.ObjectAttributesObjectParts]; !ok {
+		output.ObjectParts = nil
+	}
+	if _, ok := attrs[types.ObjectAttributesObjectSize]; !ok {
+		output.ObjectSize = nil
+	}
+	if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
+		output.StorageClass = nil
+	}
+
+	return output
+}
+
+func ParseObjectAttributes(ctx *fiber.Ctx) map[types.ObjectAttributes]struct{} {
+	attrs := map[types.ObjectAttributes]struct{}{}
+	ctx.Request().Header.VisitAll(func(key, value []byte) {
+		if string(key) == "X-Amz-Object-Attributes" {
+			oattrs := strings.Split(string(value), ",")
+			for _, a := range oattrs {
+				attrs[types.ObjectAttributes(a)] = struct{}{}
+			}
+		}
+	})
+
+	return attrs
+}
+
+type objLockCfg struct {
+	RetainUntilDate time.Time
+	ObjectLockMode  types.ObjectLockMode
+	LegalHoldStatus types.ObjectLockLegalHoldStatus
+}
+
+func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
+	legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold")
+	objLockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode")
+	objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")
+
+	if (objLockDate != "" && objLockModeHdr == "") || (objLockDate == "" && objLockModeHdr != "") {
+		return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)
+	}
+
+	var retainUntilDate time.Time
+	if objLockDate != "" {
+		rDate, err := time.Parse(time.RFC3339, objLockDate)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		}
+		if rDate.Before(time.Now()) {
+			return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
+		}
+		retainUntilDate = rDate
+	}
+
+	objLockMode := types.ObjectLockMode(objLockModeHdr)
+
+	if objLockMode != "" &&
+		objLockMode != types.ObjectLockModeCompliance &&
+		objLockMode != types.ObjectLockModeGovernance {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)
+
+	if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	return &objLockCfg{
+		RetainUntilDate: retainUntilDate,
+		ObjectLockMode:  objLockMode,
+		LegalHoldStatus: legalHold,
+	}, nil
+}
+
+func IsValidOwnership(val types.ObjectOwnership) bool {
+	switch val {
+	case types.ObjectOwnershipBucketOwnerEnforced:
+		return true
+	case types.ObjectOwnershipBucketOwnerPreferred:
+		return true
+	case types.ObjectOwnershipObjectWriter:
+		return true
+	default:
+		return false
+	}
+}

s3api/utils/utils_test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package utils
 
 import (
@@ -6,8 +20,10 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3response"
 )
 
 func TestCreateHttpRequestFromCtx(t *testing.T) {
@@ -264,3 +280,105 @@ func TestParseUint(t *testing.T) {
 		})
 	}
 }
+
+func TestFilterObjectAttributes(t *testing.T) {
+	type args struct {
+		attrs  map[types.ObjectAttributes]struct{}
+		output s3response.GetObjectAttributesResult
+	}
+	etag, objSize := "etag", int64(3222)
+	tests := []struct {
+		name string
+		args args
+		want s3response.GetObjectAttributesResult
+	}{
+		{
+			name: "keep only ETag",
+			args: args{
+				attrs: map[types.ObjectAttributes]struct{}{
+					types.ObjectAttributesEtag: {},
+				},
+				output: s3response.GetObjectAttributesResult{
+					ObjectSize: &objSize,
+					ETag:       &etag,
+				},
+			},
+			want: s3response.GetObjectAttributesResult{ETag: &etag},
+		},
+		{
+			name: "keep multiple props",
+			args: args{
+				attrs: map[types.ObjectAttributes]struct{}{
+					types.ObjectAttributesEtag:         {},
+					types.ObjectAttributesObjectSize:   {},
+					types.ObjectAttributesStorageClass: {},
+				},
+				output: s3response.GetObjectAttributesResult{
+					ObjectSize:  &objSize,
+					ETag:        &etag,
+					ObjectParts: &s3response.ObjectParts{},
+					VersionId:   &etag,
+				},
+			},
+			want: s3response.GetObjectAttributesResult{
+				ETag:       &etag,
+				ObjectSize: &objSize,
+				VersionId:  &etag,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := FilterObjectAttributes(tt.args.attrs, tt.args.output); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("FilterObjectAttributes() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsValidOwnership(t *testing.T) {
+	type args struct {
+		val types.ObjectOwnership
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "valid-BucketOwnerEnforced",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerEnforced,
+			},
+			want: true,
+		},
+		{
+			name: "valid-BucketOwnerPreferred",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerPreferred,
+			},
+			want: true,
+		},
+		{
+			name: "valid-ObjectWriter",
+			args: args{
+				val: types.ObjectOwnershipObjectWriter,
+			},
+			want: true,
+		},
+		{
+			name: "invalid_value",
+			args: args{
+				val: types.ObjectOwnership("invalid_value"),
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidOwnership(tt.args.val); got != tt.want {
+				t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -69,6 +69,7 @@ const (
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
 	ErrInvalidPart
+	ErrInvalidPartNumber
 	ErrInternalError
 	ErrInvalidCopyDest
 	ErrInvalidCopySource
@@ -114,8 +115,18 @@ const (
 	ErrObjectLockConfigurationNotFound
 	ErrNoSuchObjectLockConfiguration
 	ErrInvalidBucketObjectLockConfiguration
+	ErrObjectLockConfigurationNotAllowed
 	ErrObjectLocked
 	ErrPastObjectLockRetainDate
+	ErrObjectLockInvalidRetentionPeriod
+	ErrNoSuchBucketPolicy
+	ErrBucketTaggingNotFound
+	ErrObjectLockInvalidHeaders
+	ErrRequestTimeTooSkewed
+	ErrInvalidBucketAclWithObjectOwnership
+	ErrBothCannedAndHeaderGrants
+	ErrOwnershipControlsNotFound
+	ErrAclNotSupported
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
@@ -205,6 +216,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidPartNumber: {
+		Code:           "InvalidArgument",
+		Description:    "Part number must be an integer between 1 and 10000, inclusive",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -420,6 +436,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Bucket is missing ObjectLockConfiguration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectLockConfigurationNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrObjectLocked: {
 		Code:           "InvalidRequest",
 		Description:    "Object is WORM protected and cannot be overwritten",
@@ -430,6 +451,53 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "the retain until date must be in the future",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectLockInvalidRetentionPeriod: {
+		Code:           "InvalidRetentionPeriod",
+		Description:    "the retention days/years must be positive integer",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNoSuchBucketPolicy: {
+		Code:           "NoSuchBucketPolicy",
+		Description:    "The bucket policy does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrBucketTaggingNotFound: {
+		Code:           "NoSuchTagSet",
+		Description:    "The TagSet does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrObjectLockInvalidHeaders: {
+		Code:           "InvalidRequest",
+		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrRequestTimeTooSkewed: {
+		Code:           "RequestTimeTooSkewed",
+		Description:    "The difference between the request time and the server's time is too large.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidBucketAclWithObjectOwnership: {
+		Code:           "ErrInvalidBucketAclWithObjectOwnership",
+		Description:    "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBothCannedAndHeaderGrants: {
+		Code:           "InvalidRequest",
+		Description:    "Specifying both Canned ACLs and Header Grants is not allowed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOwnershipControlsNotFound: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAclNotSupported: {
+		Code:           "AccessControlListNotSupported",
+		Description:    "The bucket does not allow ACLs",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	// non aws errors
 	ErrExistingObjectIsDirectory: {
 		Code:           "ExistingObjectIsDirectory",
 		Description:    "Existing Object is a directory.",

s3event/event.go

@@ -37,11 +37,11 @@ type EventMeta struct {
 	VersionId   *string
 }
 
-type EventFields struct {
-	Records []EventSchema
+type EventSchema struct {
+	Records []EventRecord
 }
 
-type EventSchema struct {
+type EventRecord struct {
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
 	AwsRegion         string                `json:"awsRegion"`
@@ -117,7 +117,7 @@ type EventConfig struct {
 }
 
 func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
-	filter, err := parseEventFilters(cfg.FilterConfigFilePath)
+	filter, err := parseEventFiltersFile(cfg.FilterConfigFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("parse event filter config file %w", err)
 	}
@@ -139,54 +139,54 @@ func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
 	return evSender, err
 }
 
-func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) ([]byte, error) {
+func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) EventSchema {
 	path := strings.Split(ctx.Path(), "/")
 	bucket, object := path[1], strings.Join(path[2:], "/")
 	acc := ctx.Locals("account").(auth.Account)
 
-	event := []EventSchema{
-		{
-			EventVersion: "2.2",
-			EventSource:  "aws:s3",
-			AwsRegion:    ctx.Locals("region").(string),
-			EventTime:    time.Now().Format(time.RFC3339),
-			EventName:    meta.EventName,
-			UserIdentity: EventUserIdentity{
-				PrincipalId: acc.Access,
-			},
-			RequestParameters: EventRequestParams{
-				SourceIPAddress: ctx.IP(),
-			},
-			ResponseElements: EventResponseElements{
-				RequestId: ctx.Get("X-Amz-Request-Id"),
-				HostId:    ctx.Get("X-Amz-Id-2"),
-			},
-			S3: EventS3Data{
-				S3SchemaVersion: "1.0",
-				ConfigurationId: configId,
-				Bucket: EventS3BucketData{
-					Name: bucket,
-					OwnerIdentity: EventUserIdentity{
-						PrincipalId: meta.BucketOwner,
+	return EventSchema{
+		Records: []EventRecord{
+			{
+				EventVersion: "2.2",
+				EventSource:  "aws:s3",
+				AwsRegion:    ctx.Locals("region").(string),
+				EventTime:    time.Now().Format(time.RFC3339),
+				EventName:    meta.EventName,
+				UserIdentity: EventUserIdentity{
+					PrincipalId: acc.Access,
+				},
+				RequestParameters: EventRequestParams{
+					SourceIPAddress: ctx.IP(),
+				},
+				ResponseElements: EventResponseElements{
+					RequestId: ctx.Get("X-Amz-Request-Id"),
+					HostId:    ctx.Get("X-Amz-Id-2"),
+				},
+				S3: EventS3Data{
+					S3SchemaVersion: "1.0",
+					ConfigurationId: configId,
+					Bucket: EventS3BucketData{
+						Name: bucket,
+						OwnerIdentity: EventUserIdentity{
+							PrincipalId: meta.BucketOwner,
+						},
+						Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+					},
+					Object: EventObjectData{
+						Key:       object,
+						Size:      meta.ObjectSize,
+						ETag:      meta.ObjectETag,
+						VersionId: meta.VersionId,
+						Sequencer: genSequencer(),
 					},
-					Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
 				},
-				Object: EventObjectData{
-					Key:       object,
-					Size:      meta.ObjectSize,
-					ETag:      meta.ObjectETag,
-					VersionId: meta.VersionId,
-					Sequencer: genSequencer(),
+				GlacierEventData: EventGlacierData{
+					// Not supported
+					RestoreEventData: EventRestoreData{},
 				},
 			},
-			GlacierEventData: EventGlacierData{
-				// Not supported
-				RestoreEventData: EventRestoreData{},
-			},
 		},
 	}
-
-	return json.Marshal(event)
 }
 
 func generateTestEvent() ([]byte, error) {

s3event/filter.go

@@ -17,6 +17,7 @@ package s3event
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,19 +26,21 @@ import (
 type EventType string
 
 const (
-	EventObjectCreated           EventType = "s3:ObjectCreated:*" // ObjectCreated
-	EventObjectCreatedPut        EventType = "s3:ObjectCreated:Put"
-	EventObjectCreatedPost       EventType = "s3:ObjectCreated:Post"
-	EventObjectCreatedCopy       EventType = "s3:ObjectCreated:Copy"
-	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
-	EventObjectDeleted           EventType = "s3:ObjectRemoved:Delete" // ObjectRemoved
-	EventObjectTagging           EventType = "s3:ObjectTagging:*"      // ObjectTagging
-	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
-	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
-	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
-	EventObjectRestore           EventType = "s3:ObjectRestore:*" // ObjectRestore
-	EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
-	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
+	EventObjectCreated              EventType = "s3:ObjectCreated:*" // ObjectCreated
+	EventObjectCreatedPut           EventType = "s3:ObjectCreated:Put"
+	EventObjectCreatedPost          EventType = "s3:ObjectCreated:Post"
+	EventObjectCreatedCopy          EventType = "s3:ObjectCreated:Copy"
+	EventCompleteMultipartUpload    EventType = "s3:ObjectCreated:CompleteMultipartUpload"
+	EventObjectRemoved              EventType = "s3:ObjectRemoved:*"
+	EventObjectRemovedDelete        EventType = "s3:ObjectRemoved:Delete"
+	EventObjectRemovedDeleteObjects EventType = "s3:ObjectRemoved:DeleteObjects" // non AWS custom type for DeleteObjects
+	EventObjectTagging              EventType = "s3:ObjectTagging:*"             // ObjectTagging
+	EventObjectTaggingPut           EventType = "s3:ObjectTagging:Put"
+	EventObjectTaggingDelete        EventType = "s3:ObjectTagging:Delete"
+	EventObjectAclPut               EventType = "s3:ObjectAcl:Put"
+	EventObjectRestore              EventType = "s3:ObjectRestore:*" // ObjectRestore
+	EventObjectRestorePost          EventType = "s3:ObjectRestore:Post"
+	EventObjectRestoreCompleted     EventType = "s3:ObjectRestore:Completed"
 	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
 	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
 )
@@ -48,24 +51,26 @@ func (event EventType) IsValid() bool {
 }
 
 var supportedEventFilters = map[EventType]struct{}{
-	EventObjectCreated:           {},
-	EventObjectCreatedPut:        {},
-	EventObjectCreatedPost:       {},
-	EventObjectCreatedCopy:       {},
-	EventCompleteMultipartUpload: {},
-	EventObjectDeleted:           {},
-	EventObjectTagging:           {},
-	EventObjectTaggingPut:        {},
-	EventObjectTaggingDelete:     {},
-	EventObjectAclPut:            {},
-	EventObjectRestore:           {},
-	EventObjectRestorePost:       {},
-	EventObjectRestoreCompleted:  {},
+	EventObjectCreated:              {},
+	EventObjectCreatedPut:           {},
+	EventObjectCreatedPost:          {},
+	EventObjectCreatedCopy:          {},
+	EventCompleteMultipartUpload:    {},
+	EventObjectRemoved:              {},
+	EventObjectRemovedDelete:        {},
+	EventObjectRemovedDeleteObjects: {},
+	EventObjectTagging:              {},
+	EventObjectTaggingPut:           {},
+	EventObjectTaggingDelete:        {},
+	EventObjectAclPut:               {},
+	EventObjectRestore:              {},
+	EventObjectRestorePost:          {},
+	EventObjectRestoreCompleted:     {},
 }
 
 type EventFilter map[EventType]bool
 
-func parseEventFilters(path string) (EventFilter, error) {
+func parseEventFiltersFile(path string) (EventFilter, error) {
 	// if no filter config file path is specified return nil map
 	if path == "" {
 		return nil, nil
@@ -83,8 +88,12 @@ func parseEventFilters(path string) (EventFilter, error) {
 	}
 	defer file.Close()
 
+	return parseEventFilters(file)
+}
+
+func parseEventFilters(r io.Reader) (EventFilter, error) {
 	var filter EventFilter
-	if err := json.NewDecoder(file).Decode(&filter); err != nil {
+	if err := json.NewDecoder(r).Decode(&filter); err != nil {
 		return nil, err
 	}
 
@@ -112,7 +121,7 @@ func (ef EventFilter) Filter(event EventType) bool {
 	}
 
 	// check wildcard match
-	wildCardEv := EventType(string(event[strings.LastIndex(string(event), ":")+1]) + "*")
+	wildCardEv := EventType(string(event[:strings.LastIndex(string(event), ":")+1]) + "*")
 	wildcard, found := ef[wildCardEv]
 	if found {
 		return wildcard

s3event/filter_test.go

@@ -0,0 +1,52 @@
+package s3event
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFilterWildcardCreated(t *testing.T) {
+	filterString := `{"s3:ObjectCreated:*": true}`
+	strReader := strings.NewReader(filterString)
+
+	ef, err := parseEventFilters(strReader)
+	if err != nil {
+		t.Fatalf("failed to parse event filter: %v", err)
+	}
+
+	created := []string{
+		"s3:ObjectCreated:Put",
+		"s3:ObjectCreated:Post",
+		"s3:ObjectCreated:Copy",
+		"s3:ObjectCreated:CompleteMultipartUpload",
+	}
+
+	for _, event := range created {
+		allowed := ef.Filter(EventType(event))
+		if !allowed {
+			t.Errorf("expected event to be allowed: %s", event)
+		}
+	}
+}
+
+func TestFilterWildcardRemoved(t *testing.T) {
+	filterString := `{"s3:ObjectRemoved:*": true}`
+	strReader := strings.NewReader(filterString)
+
+	ef, err := parseEventFilters(strReader)
+	if err != nil {
+		t.Fatalf("failed to parse event filter: %v", err)
+	}
+
+	removed := []string{
+		"s3:ObjectRemoved:Delete",
+		"s3:ObjectRemoved:DeleteObjects",
+	}
+
+	for _, event := range removed {
+		allowed := ef.Filter(EventType(event))
+		if !allowed {
+			t.Errorf("expected event to be allowed: %s", event)
+		}
+	}
+}

s3event/kafka.go

@@ -16,6 +16,8 @@ package s3event
 
 import (
 	"context"
+	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"os"
 	"sync"
@@ -23,6 +25,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/segmentio/kafka-go"
+	"github.com/versity/versitygw/s3response"
 )
 
 var sequencer = 0
@@ -78,12 +81,29 @@ func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 		return
 	}
 
-	schema, err := createEventSchema(ctx, meta, ConfigurationIdKafka)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to create kafka event: %v\n", err.Error())
+	if meta.EventName == EventObjectRemovedDeleteObjects {
+		var dObj s3response.DeleteObjects
+
+		if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
+			fmt.Fprintf(os.Stderr, "failed to parse delete objects input payload: %v\n", err.Error())
+			return
+		}
+
+		// Events aren't send in correct order
+		for _, obj := range dObj.Objects {
+			key := *obj.Key
+			schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+			schema.Records[0].S3.Object.Key = key
+			schema.Records[0].S3.Object.VersionId = obj.VersionId
+
+			go ks.send(schema)
+		}
+
 		return
 	}
 
+	schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+
 	go ks.send(schema)
 }
 
@@ -91,14 +111,20 @@ func (ks *Kafka) Close() error {
 	return ks.writer.Close()
 }
 
-func (ks *Kafka) send(event []byte) {
+func (ks *Kafka) send(event EventSchema) {
+	eventBytes, err := json.Marshal(event)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse event data: %v\n", err.Error())
+		return
+	}
+
 	message := kafka.Message{
 		Key:   []byte(ks.key),
-		Value: event,
+		Value: eventBytes,
 	}
 
 	ctx := context.Background()
-	err := ks.writer.WriteMessages(ctx, message)
+	err = ks.writer.WriteMessages(ctx, message)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
 	}

s3event/nats.go

@@ -15,12 +15,15 @@
 package s3event
 
 import (
+	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"os"
 	"sync"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/nats-io/nats.go"
+	"github.com/versity/versitygw/s3response"
 )
 
 type NatsEventSender struct {
@@ -65,12 +68,29 @@ func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 		return
 	}
 
-	schema, err := createEventSchema(ctx, meta, ConfigurationIdNats)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to create nats event: %v\n", err.Error())
+	if meta.EventName == EventObjectRemovedDeleteObjects {
+		var dObj s3response.DeleteObjects
+
+		if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
+			fmt.Fprintf(os.Stderr, "failed to parse delete objects input payload: %v\n", err.Error())
+			return
+		}
+
+		// Events aren't send in correct order
+		for _, obj := range dObj.Objects {
+			key := *obj.Key
+			schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+			schema.Records[0].S3.Object.Key = key
+			schema.Records[0].S3.Object.VersionId = obj.VersionId
+
+			go ns.send(schema)
+		}
+
 		return
 	}
 
+	schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+
 	go ns.send(schema)
 }
 
@@ -79,8 +99,13 @@ func (ns *NatsEventSender) Close() error {
 	return nil
 }
 
-func (ns *NatsEventSender) send(event []byte) {
-	err := ns.client.Publish(ns.topic, event)
+func (ns *NatsEventSender) send(event EventSchema) {
+	eventBytes, err := json.Marshal(event)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse event data: %v\n", err.Error())
+		return
+	}
+	err = ns.client.Publish(ns.topic, eventBytes)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
 	}

s3event/webhook.go

@@ -16,6 +16,8 @@ package s3event
 
 import (
 	"bytes"
+	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"net"
 	"net/http"
@@ -24,6 +26,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3response"
 )
 
 type Webhook struct {
@@ -77,12 +80,29 @@ func (w *Webhook) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 		return
 	}
 
-	schema, err := createEventSchema(ctx, meta, ConfigurationIdWebhook)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to create webhook event: %v\n", err.Error())
+	if meta.EventName == EventObjectRemovedDeleteObjects {
+		var dObj s3response.DeleteObjects
+
+		if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
+			fmt.Fprintf(os.Stderr, "failed to parse delete objects input payload: %v\n", err.Error())
+			return
+		}
+
+		// Events aren't send in correct order
+		for _, obj := range dObj.Objects {
+			key := *obj.Key
+			schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+			schema.Records[0].S3.Object.Key = key
+			schema.Records[0].S3.Object.VersionId = obj.VersionId
+
+			go w.send(schema)
+		}
+
 		return
 	}
 
+	schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+
 	go w.send(schema)
 }
 
@@ -90,8 +110,14 @@ func (w *Webhook) Close() error {
 	return nil
 }
 
-func (w *Webhook) send(event []byte) {
-	req, err := http.NewRequest(http.MethodPost, w.url, bytes.NewReader(event))
+func (w *Webhook) send(event EventSchema) {
+	eventBytes, err := json.Marshal(event)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse event data: %v\n", err.Error())
+		return
+	}
+
+	req, err := http.NewRequest(http.MethodPost, w.url, bytes.NewReader(eventBytes))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to create webhook event request: %v\n", err.Error())
 		return

s3response/s3response.go

@@ -52,6 +52,23 @@ type ListPartsResult struct {
 	Parts []Part `xml:"Part"`
 }
 
+type GetObjectAttributesResult struct {
+	ETag         *string
+	LastModified *time.Time
+	ObjectSize   *int64
+	StorageClass *types.StorageClass
+	VersionId    *string
+	ObjectParts  *ObjectParts
+}
+
+type ObjectParts struct {
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+	Parts                []types.ObjectPart `xml:"Part"`
+}
+
 // ListMultipartUploadsResponse - s3 api list multipart uploads response.
 type ListMultipartUploadsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
@@ -200,3 +217,7 @@ type Grantee struct {
 	ID          string
 	DisplayName string
 }
+
+type OwnershipControls struct {
+	Rules []types.OwnershipControlsRule `xml:"Rule"`
+}

tests/.env.default

@@ -14,4 +14,7 @@ SECRETS_FILE=./tests/.secrets
 MC_ALIAS=versity
 LOG_LEVEL=2
 GOCOVERDIR=$PWD/cover
-USERS_FOLDER=$PWD/iam
+USERS_FOLDER=$PWD/iam
+#TEST_LOG_FILE=test.log
+#VERSITY_LOG_FILE=versity.log
+IAM_TYPE=folder

tests/README.md

@@ -10,6 +10,7 @@
    * **s3cmd**:  Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
    * **mc**:  Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
 3. Install BATS.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
+4. If running on Mac OS, install **jq** with the command `brew install jq`.
 4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
 5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
 ```
@@ -30,6 +31,10 @@
 8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
 9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
+### Static Bucket Mode
+
+To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`.  Two utility functions are included, if needed, to create, and delete buckets for this:  `tests/setup_static.sh` and `tests/remove_static.sh`.
+
 ### S3 Backend
 
 Instructions are mostly the same; however, testing with the S3 backend requires two S3 accounts.  Ideally, these are two real accounts, but one can also be a dummy account that versity uses internally.

tests/commands/abort_multipart_upload.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+abort_multipart_upload() {
+  if [ $# -ne 3 ]; then
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID"
+    return 1
+  fi
+  if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $error"
+    return 1
+  fi
+  return 0
+}
+
+abort_multipart_upload_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID, username, password"
+    return 1
+  fi
+  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $abort_multipart_upload_error"
+    export abort_multipart_upload_error
+    return 1
+  fi
+  return 0
+}

tests/commands/complete_multipart_upload.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+complete_multipart_upload() {
+  if [[ $# -ne 4 ]]; then
+    log 2 "'complete multipart upload' command requires bucket, key, upload ID, parts list"
+    return 1
+  fi
+  log 5 "complete multipart upload id: $3, parts: $4"
+  error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
+  if [[ $completed -ne 0 ]]; then
+    log 2 "error completing multipart upload: $error"
+    return 1
+  fi
+  log 5 "complete multipart upload error: $error"
+  return 0
+}

tests/commands/copy_object.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+copy_object() {
+  if [ $# -ne 4 ]; then
+    echo "copy object command requires command type, source, bucket, key"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == 's3' ]]; then
+    error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4"
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+  else
+    echo "'copy-object' not implemented for '$1'"
+    return 1
+  fi
+  log 5 "copy object exit code: $exit_code"
+  if [ $exit_code -ne 0 ]; then
+    echo "error copying object to bucket: $error"
+    return 1
+  fi
+  return 0
+}
+
+copy_object_empty() {
+  error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
+  if [[ $result -eq 0 ]]; then
+    log 2 "copy object with empty parameters returned no error"
+    return 1
+  fi
+  if [[ $error != *"the following arguments are required: --bucket, --copy-source, --key" ]]; then
+    log 2 "copy object with no params returned mismatching error: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/create_bucket.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# create an AWS bucket
+# param:  bucket name
+# return 0 for success, 1 for failure
+create_bucket() {
+  if [ $# -ne 2 ]; then
+    log 2 "create bucket missing command type, bucket name"
+    return 1
+  fi
+
+  local exit_code=0
+  local error
+  log 6 "create bucket"
+  if [[ $1 == 's3' ]]; then
+    error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "mc" ]]; then
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    log 2 "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error creating bucket: $error"
+    return 1
+  fi
+  return 0
+}
+
+create_bucket_object_lock_enabled() {
+  if [ $# -ne 1 ]; then
+    log 2 "create bucket missing bucket name"
+    return 1
+  fi
+
+  local exit_code=0
+  error=$(aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error creating bucket: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/create_multipart_upload.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# initialize a multipart upload
+# params:  bucket, key
+# return 0 for success, 1 for failure
+create_multipart_upload() {
+  if [ $# -ne 2 ]; then
+    log 2 "create multipart upload function must have bucket, key"
+    return 1
+  fi
+
+  if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+    log 2 "Error creating multipart upload: $multipart_data"
+    return 1
+  fi
+
+  if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
+    log 2 "error parsing upload ID: $upload_id"
+    return 1
+  fi
+  upload_id="${upload_id//\"/}"
+  export upload_id
+  return 0
+}
+
+create_multipart_upload_with_user() {
+  if [ $# -ne 4 ]; then
+    log 2 "create multipart upload function must have bucket, key, username, password"
+    return 1
+  fi
+
+  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+    log 2 "Error creating multipart upload: $multipart_data"
+    return 1
+  fi
+
+  if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
+    log 2 "error parsing upload ID: $upload_id"
+    return 1
+  fi
+  upload_id="${upload_id//\"/}"
+  export upload_id
+  return 0
+}
+
+create_multipart_upload_params() {
+  if [ $# -ne 8 ]; then
+    log 2 "create multipart upload function with params must have bucket, key, content type, metadata, object lock legal hold status, " \
+      "object lock mode, object lock retain until date, and tagging"
+    return 1
+  fi
+  local multipart_data
+  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload \
+    --bucket "$1" \
+    --key "$2" \
+    --content-type "$3" \
+    --metadata "$4" \
+    --object-lock-legal-hold-status "$5" \
+    --object-lock-mode "$6" \
+    --object-lock-retain-until-date "$7" \
+    --tagging "$8" 2>&1) || local create_result=$?
+  if [[ $create_result -ne 0 ]]; then
+    log 2 "error creating multipart upload with params: $multipart_data"
+    return 1
+  fi
+  export multipart_data
+  upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId')
+  upload_id="${upload_id//\"/}"
+  export upload_id
+  return 0
+}
+
+create_multipart_upload_custom() {
+  if [ $# -lt 2 ]; then
+    log 2 "create multipart upload custom function must have at least bucket and key"
+    return 1
+  fi
+  local multipart_data
+  log 5 "additional create multipart params"
+  for i in "$@"; do
+    log 5 "$i"
+  done
+  log 5 "${*:3}"
+  log 5 "aws --no-verify-ssl s3api create-multipart-upload --bucket $1 --key $2 ${*:3}"
+  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$?
+  if [[ $result -ne 0 ]]; then
+    log 2 "error creating custom multipart data command: $multipart_data"
+    return 1
+  fi
+  export multipart_data
+  log 5 "multipart data: $multipart_data"
+  upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId')
+  upload_id="${upload_id//\"/}"
+  log 5 "upload id: $upload_id"
+  export upload_id
+  return 0
+}

tests/commands/delete_bucket.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# delete an AWS bucket
+# param:  bucket name
+# return 0 for success, 1 for failure
+delete_bucket() {
+  if [ $# -ne 2 ]; then
+    log 2 "delete bucket missing command type, bucket name"
+    return 1
+  fi
+
+  local exit_code=0
+  local error
+  if [[ $1 == 's3' ]]; then
+    error=$(aws --no-verify-ssl s3 rb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
+  else
+    log 2 "Invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    if [[ "$error" == *"The specified bucket does not exist"* ]]; then
+      return 0
+    else
+      log 2 "error deleting bucket: $error"
+      return 1
+    fi
+  fi
+  return 0
+}

tests/commands/delete_bucket_policy.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+delete_bucket_policy() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "delete bucket policy command requires command type, bucket"
+    return 1
+  fi
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
+  else
+    log 2 "command 'delete bucket policy' not implemented for '$1'"
+    return 1
+  fi
+  if [[ $delete_result -ne 0 ]]; then
+    log 2 "error deleting bucket policy: $error"
+    return 1
+  fi
+  return 0
+}
+
+delete_bucket_policy_with_user() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete bucket policy with user' command requires bucket, username, password"
+    return 1
+  fi
+  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
+    log 2 "error deleting bucket policy: $delete_bucket_policy_error"
+    export delete_bucket_policy_error
+    return 1
+  fi
+  return 0
+}

tests/commands/delete_object.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+delete_object() {
+  if [ $# -ne 3 ]; then
+    log 2 "delete object command requires command type, bucket, key"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 's3' ]]; then
+    delete_object_error=$(aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    delete_object_error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+  else
+    log 2 "invalid command type $1"
+    return 1
+  fi
+  log 5 "delete object exit code: $exit_code"
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error deleting object: $delete_object_error"
+    export delete_object_error
+    return 1
+  fi
+  return 0
+}
+
+delete_object_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "delete object with user command requires command type, bucket, key, access ID, secret key"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 's3' ]]; then
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
+  else
+    log 2 "command 'delete object with user' not implemented for '$1'"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error deleting object: $delete_object_error"
+    export delete_object_error
+    return 1
+  fi
+  return 0
+}

tests/commands/delete_object_tagging.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+delete_object_tagging() {
+  if [[ $# -ne 3 ]]; then
+    echo "delete object tagging command missing command type, bucket, key"
+    return 1
+  fi
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
+  else
+    echo "delete-object-tagging command not implemented for '$1'"
+    return 1
+  fi
+  if [[ $delete_result -ne 0 ]]; then
+    echo "error deleting object tagging: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/get_bucket_acl.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+get_bucket_acl() {
+  if [ $# -ne 2 ]; then
+    log 2 "bucket ACL command missing command type, bucket name"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    acl=$(aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
+  elif [[ $1 == 's3cmd' ]]; then
+    acl=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
+  else
+    log 2 "command 'get bucket acl' not implemented for $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    log 2 "Error getting bucket ACLs: $acl"
+    return 1
+  fi
+  export acl
+}
+
+get_bucket_acl_with_user() {
+  if [ $# -ne 3 ]; then
+    log 2 "'get bucket ACL with user' command requires bucket name, username, password"
+    return 1
+  fi
+  if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
+    log 2 "error getting bucket ACLs: $bucket_acl"
+    return 1
+  fi
+  export bucket_acl
+  return 0
+}

tests/commands/get_bucket_location.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+get_bucket_location() {
+  if [[ $# -ne 2 ]]; then
+    echo "get bucket location command requires command type, bucket name"
+    return 1
+  fi
+  if [[ $1 == 'aws' ]]; then
+    get_bucket_location_aws "$2" || get_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    get_bucket_location_s3cmd "$2" || get_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    get_bucket_location_mc "$2" || get_result=$?
+  else
+    echo "command type '$1' not implemented for get_bucket_location"
+    return 1
+  fi
+  if [[ $get_result -ne 0 ]]; then
+    return 1
+  fi
+  location=$(echo "$location_json" | jq -r '.LocationConstraint')
+  export location
+}
+
+get_bucket_location_aws() {
+  if [[ $# -ne 1 ]]; then
+    echo "get bucket location (aws) requires bucket name"
+    return 1
+  fi
+  location_json=$(aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$?
+  if [[ $location_result -ne 0 ]]; then
+    echo "error getting bucket location: $location"
+    return 1
+  fi
+  bucket_location=$(echo "$location_json" | jq -r '.LocationConstraint')
+  export bucket_location
+  return 0
+}
+
+get_bucket_location_s3cmd() {
+  if [[ $# -ne 1 ]]; then
+    echo "get bucket location (s3cmd) requires bucket name"
+    return 1
+  fi
+  info=$(s3cmd --no-check-certificate info "s3://$1") || results=$?
+  if [[ $results -ne 0 ]]; then
+    echo "error getting s3cmd info: $info"
+    return 1
+  fi
+  bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}')
+  export bucket_location
+  return 0
+}
+
+get_bucket_location_mc() {
+  if [[ $# -ne 1 ]]; then
+    echo "get bucket location (mc) requires bucket name"
+    return 1
+  fi
+  info=$(mc --insecure stat "$MC_ALIAS/$1") || results=$?
+  if [[ $results -ne 0 ]]; then
+    echo "error getting s3cmd info: $info"
+    return 1
+  fi
+  bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}')
+  export bucket_location
+  return 0
+}

tests/commands/get_bucket_ownership_controls.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+get_bucket_ownership_controls() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get bucket ownership controls' command requires bucket name"
+    return 1
+  fi
+
+  if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
+    log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
+    return 1
+  fi
+
+  log 5 "Raw bucket Ownership Controls:  $raw_bucket_ownership_controls"
+  bucket_ownership_controls=$(echo "$raw_bucket_ownership_controls" | grep -v "InsecureRequestWarning")
+  export bucket_ownership_controls
+  return 0
+}
+
+get_object_ownership_rule() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get object ownership rule' command requires bucket name"
+    return 1
+  fi
+  if ! get_bucket_ownership_controls "$1"; then
+    log 2 "error getting bucket ownership controls"
+    return 1
+  fi
+  if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then
+    log 2 "error getting object ownership rule: $object_ownership_rule"
+    return 1
+  fi
+  log 5 "object ownership rule: $object_ownership_rule"
+  export object_ownership_rule
+  return 0
+}

tests/commands/get_bucket_policy.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+
+get_bucket_policy() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "get bucket policy command requires command type, bucket"
+    return 1
+  fi
+  local get_bucket_policy_result=0
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    get_bucket_policy_aws "$2" || get_bucket_policy_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    get_bucket_policy_mc "$2" || get_bucket_policy_result=$?
+  else
+    log 2 "command 'get bucket policy' not implemented for '$1'"
+    return 1
+  fi
+  if [[ $get_bucket_policy_result -ne 0 ]]; then
+    log 2 "error getting policy: $bucket_policy"
+    return 1
+  fi
+  export bucket_policy
+  return 0
+}
+
+get_bucket_policy_aws() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "aws 'get bucket policy' command requires bucket"
+    return 1
+  fi
+  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
+  policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
+  log 5 "$policy_json"
+  if [[ $get_result -ne 0 ]]; then
+    if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
+      bucket_policy=
+    else
+      log 2 "error getting policy: $policy_json"
+      return 1
+    fi
+  else
+    bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
+  fi
+  export bucket_policy
+  return 0
+}
+
+get_bucket_policy_with_user() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'get bucket policy with user' command requires bucket, username, password"
+    return 1
+  fi
+  if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
+    policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
+    bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
+  else
+    if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
+      bucket_policy=
+    else
+      log 2 "error getting policy for user $2: $policy_json"
+      return 1
+    fi
+  fi
+  export bucket_policy
+  return 0
+}
+
+get_bucket_policy_s3cmd() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "s3cmd 'get bucket policy' command requires bucket"
+    return 1
+  fi
+
+  if ! info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
+    log 2 "error getting bucket policy: $info"
+    return 1
+  fi
+
+  log 5 "policy info: $info"
+  bucket_policy=""
+  policy_brackets=false
+  # NOTE:  versitygw sends policies back in multiple lines here, direct in single line
+  while IFS= read -r line; do
+    if [[ $policy_brackets == false ]]; then
+      policy_line=$(echo "$line" | grep 'Policy: ')
+      if [[ $policy_line != "" ]]; then
+        if [[ $policy_line != *'{'* ]]; then
+          break
+        fi
+        if [[ $policy_line == *'}'* ]]; then
+          log 5 "policy on single line"
+          bucket_policy=${policy_line//Policy:/}
+          break
+        else
+          policy_brackets=true
+          bucket_policy+="{"
+        fi
+      fi
+    else
+      bucket_policy+=$line
+      if [[ $line == "" ]]; then
+        break
+      fi
+    fi
+  done <<< "$info"
+  log 5 "bucket policy: $bucket_policy"
+  export bucket_policy
+  return 0
+}
+
+get_bucket_policy_mc() {
+  if [[ $# -ne 1 ]]; then
+    echo "aws 'get bucket policy' command requires bucket"
+    return 1
+  fi
+  bucket_policy=$(mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
+  if [[ $get_result -ne 0 ]]; then
+    echo "error getting policy: $bucket_policy"
+    return 1
+  fi
+  export bucket_policy
+  return 0
+}

tests/commands/get_bucket_tagging.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# get bucket tags
+# params:  bucket
+# export 'tags' on success, return 1 for error
+get_bucket_tagging() {
+  if [ $# -ne 2 ]; then
+    echo "get bucket tag command missing command type, bucket name"
+    return 1
+  fi
+  local result
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  log 5 "Tags: $tags"
+  tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
+  if [[ $result -ne 0 ]]; then
+    if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
+      export tags=
+      return 0
+    fi
+    echo "error getting bucket tags: $tags"
+    return 1
+  fi
+  export tags
+}

tests/commands/get_bucket_versioning.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+get_bucket_versioning() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "put bucket versioning command requires command type, bucket name"
+    return 1
+  fi
+  local get_result=0
+  if [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
+  fi
+  if [[ $get_result -ne 0 ]]; then
+    log 2 "error getting bucket versioning: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/get_object.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+get_object() {
+  if [ $# -ne 4 ]; then
+    log 2 "get object command requires command type, bucket, key, destination"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == 's3' ]]; then
+    error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+  else
+    log 2 "'get object' command not implemented for '$1'"
+    return 1
+  fi
+  log 5 "get object exit code: $exit_code"
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error getting object: $error"
+    return 1
+  fi
+  return 0
+}
+
+get_object_with_range() {
+  if [[ $# -ne 4 ]]; then
+    log 2 "'get object with range' requires bucket, key, range, outfile"
+    return 1
+  fi
+  error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1) || local exit_code=$?
+  if [[ $exit_code -ne 0 ]]; then
+    log 2 "error getting object with range: $error"
+    return 1
+  fi
+  return 0
+}
+
+get_object_with_user() {
+  if [ $# -ne 6 ]; then
+    log 2 "'get object with user' command requires command type, bucket, key, save location, aws ID, aws secret key"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+  else
+    log 2 "'get object with user' command not implemented for '$1'"
+    return 1
+  fi
+  log 5 "put object exit code: $exit_code"
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error getting object: $get_object_error"
+    export get_object_error
+    return 1
+  fi
+  return 0
+}

tests/commands/get_object_attributes.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+get_object_attributes() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "'get object attributes' command requires bucket, key"
+    return 1
+  fi
+  attributes=$(aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$?
+  if [[ $get_result -ne 0 ]]; then
+    log 2 "error getting object attributes: $attributes"
+    return 1
+  fi
+  attributes=$(echo "$attributes" | grep -v "InsecureRequestWarning")
+  log 5 "$attributes"
+  export attributes
+  return 0
+}

tests/commands/get_object_legal_hold.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+get_object_legal_hold() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "'get object legal hold' command requires bucket, key"
+    return 1
+  fi
+  legal_hold=$(aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
+  if [[ $get_result -ne 0 ]]; then
+    log 2 "error getting object legal hold: $legal_hold"
+    return 1
+  fi
+  export legal_hold
+  return 0
+}

tests/commands/get_object_lock_configuration.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+get_object_lock_configuration() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get object lock configuration' command missing bucket name"
+    return 1
+  fi
+  lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1") || local get_result=$?
+  if [[ $get_result -ne 0 ]]; then
+    log 2 "error obtaining lock config: $lock_config"
+    return 1
+  fi
+  export lock_config
+  return 0
+}

tests/commands/get_object_retention.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+get_object_retention() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "'get object retention' command requires bucket, key"
+    return 1
+  fi
+  retention=$(aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1) || local get_result=$?
+  if [[ $get_result -ne 0 ]]; then
+    log 2 "error getting object retention: $retention"
+    return 1
+  fi
+  export retention
+  return 0
+}

tests/commands/get_object_tagging.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+get_object_tagging() {
+  if [ $# -ne 3 ]; then
+    log 2 "get object tag command missing command type, bucket, and/or key"
+    return 1
+  fi
+  local result
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
+  else
+    log 2 "invalid command type $1"
+    return 1
+  fi
+  if [[ $result -ne 0 ]]; then
+    if [[ "$tags" == *"NoSuchTagSet"* ]] || [[ "$tags" == *"No tags found"* ]]; then
+      tags=
+    else
+      log 2 "error getting object tags: $tags"
+      return 1
+    fi
+  else
+    log 5 "$tags"
+    tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
+  fi
+  export tags
+}

tests/commands/head_bucket.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+head_bucket() {
+  if [ $# -ne 2 ]; then
+    echo "head bucket command missing command type, bucket name"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+    bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    bucket_info=$(s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error getting bucket info: $bucket_info"
+    return 1
+  fi
+  export bucket_info
+  return 0
+}

tests/commands/head_object.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+head_object() {
+  if [ $# -ne 3 ]; then
+    log 2 "head-object missing command, bucket name, object name"
+    return 2
+  fi
+  local exit_code=0
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
+    metadata=$(aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?"
+  elif [[ $1 == 's3cmd' ]]; then
+    metadata=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?"
+  elif [[ $1 == 'mc' ]]; then
+    metadata=$(mc --insecure stat "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+  else
+    log 2 "invalid command type $1"
+    return 2
+  fi
+  if [ $exit_code -ne 0 ]; then
+    if [[ "$metadata" == *"404"* ]] || [[ "$metadata" == *"does not exist"* ]]; then
+      log 5 "file doesn't exist ($metadata)"
+      return 1
+    else
+      log 2 "error checking if object exists: $metadata"
+      return 2
+    fi
+  fi
+  export metadata
+  return 0
+}

tests/commands/list_buckets.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+list_buckets() {
+  if [ $# -ne 1 ]; then
+    echo "list buckets command missing command type"
+    return 1
+  fi
+
+  local exit_code=0
+  if [[ $1 == 's3' ]]; then
+    buckets=$(aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$?
+  elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    list_buckets_s3api || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    buckets=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    buckets=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
+  else
+    echo "list buckets command not implemented for '$1'"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error listing buckets: $buckets"
+    return 1
+  fi
+
+  if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    return 0
+  fi
+
+  bucket_array=()
+  while IFS= read -r line; do
+    bucket_name=$(echo "$line" | awk '{print $NF}')
+    bucket_array+=("${bucket_name%/}")
+  done <<< "$buckets"
+  export bucket_array
+  return 0
+}
+
+list_buckets_s3api() {
+  output=$(aws --no-verify-ssl s3api list-buckets 2>&1) || exit_code=$?
+  if [[ $exit_code -ne 0 ]]; then
+    echo "error listing buckets: $output"
+    return 1
+  fi
+
+  modified_output=""
+  while IFS= read -r line; do
+    if [[ $line != *InsecureRequestWarning* ]]; then
+      modified_output+="$line"
+    fi
+  done <<< "$output"
+
+  bucket_array=()
+  names=$(jq -r '.Buckets[].Name' <<<"$modified_output")
+  IFS=$'\n' read -rd '' -a bucket_array <<<"$names"
+
+  export bucket_array
+  return 0
+}

tests/commands/list_multipart_uploads.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+list_multipart_uploads() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'list multipart uploads' command requires bucket name"
+    return 1
+  fi
+  if ! uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+    log 2 "error listing uploads: $uploads"
+    return 1
+  fi
+  export uploads
+}
+
+list_multipart_uploads_with_user() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'list multipart uploads' command requires bucket name, username, password"
+    return 1
+  fi
+  if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
+    log 2 "error listing uploads: $uploads"
+    list_multipart_uploads_error=$uploads
+    export list_multipart_uploads_error
+    return 1
+  fi
+  export uploads
+}

tests/commands/list_object_versions.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+list_object_versions() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'list object versions' command requires bucket name"
+    return 1
+  fi
+  versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1") || local list_result=$?
+  if [[ $list_result -ne 0 ]]; then
+    log 2 "error listing object versions: $versions"
+    return 1
+  fi
+  export versions
+  return 0
+}