Merge pull request #806 from versity/ben/common_prefixes

fix: list objects trim common prefixes that match marker prefix

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,27 +1,23 @@
 ---
-name: Bug report
+name: Bug Report
 about: Create a report to help us improve
-title: ''
+title: '[Bug] - <Short Description>'
 labels: bug
 assignees: ''
 
 ---
 
 **Describe the bug**
-A clear and concise description of what the bug is.
+<!-- A clear and concise description of what the bug is. -->
 
 **To Reproduce**
-Steps to reproduce the behavior.
+<!-- Steps to reproduce the behavior. -->
 
 **Expected behavior**
-A clear and concise description of what you expected to happen.
+<!-- A clear and concise description of what you expected to happen. -->
 
 **Server Version**
- output of
-```
-./versitygw -version
-uname -a
-```
+<!-- output of: './versitygw -version && uname -a'  -->
 
 **Additional context**
-Describe s3 client and version if applicable.
+<!-- Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,14 +1,14 @@
 ---
-name: Feature request
+name: Feature Request
 about: Suggest an idea for this project
-title: ''
+title: '[Feature] - <Short Description>'
 labels: enhancement
 assignees: ''
 
 ---
 
 **Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+<!-- A clear and concise description of what you want to happen. -->
 
 **Additional context**
-Add any other context or screenshots about the feature request here.
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/test_case.md

@@ -0,0 +1,33 @@
+---
+name: Test Case Request
+about: Request new test cases or additional test coverage
+title: '[Test Case] - <Short Description>'
+labels: 'testcase'
+assignees: ''
+
+---
+
+## Description
+<!-- Please provide a detailed description of the test case or test coverage request. -->
+
+## Purpose
+<!-- Explain why this test case is important and what it aims to achieve. -->
+
+## Scope
+<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
+
+## Acceptance Criteria
+<!-- List the criteria that must be met for the test case to be considered complete. -->
+
+1. 
+2. 
+3. 
+
+## Additional Context
+<!-- Add any other context or screenshots about the feature request here. -->
+
+## Resources
+<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
+
+
+**Thank you for contributing to our project!**

.github/workflows/docker-bats.yaml

@@ -0,0 +1,28 @@
+name: docker bats tests
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker Image
+        run: |
+          mv tests/.env.docker.default tests/.env.docker
+          mv tests/.secrets.default tests/.secrets
+          docker build --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain -f tests/Dockerfile_test_bats -t bats_test .
+
+      - name: Set up Docker Compose
+        run: sudo apt-get install -y docker-compose
+
+      - name: Run Docker Container
+        run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from posix_backend posix_backend

.github/workflows/docker.yaml

@@ -15,6 +15,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -43,6 +50,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
           build-args: |
             VERSION=${{ github.event.release.tag_name }}
             TIME=${{ github.event.release.published_at }}

.github/workflows/system.yml

@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - set: 1
+          - set: "s3cmd, posix"
             LOCAL_FOLDER: /tmp/gw1
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
             BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
@@ -16,8 +16,10 @@ jobs:
             USERS_FOLDER: /tmp/iam1
             AWS_ENDPOINT_URL: https://127.0.0.1:7070
             RUN_SET: "s3cmd"
+            RECREATE_BUCKETS: "true"
             PORT: 7070
-          - set: 2
+            BACKEND: "posix"
+          - set: "s3, posix"
             LOCAL_FOLDER: /tmp/gw2
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
             BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
@@ -25,8 +27,10 @@ jobs:
             USERS_FOLDER: /tmp/iam2
             AWS_ENDPOINT_URL: https://127.0.0.1:7071
             RUN_SET: "s3"
+            RECREATE_BUCKETS: "true"
             PORT: 7071
-          - set: 3
+            BACKEND: "posix"
+          - set: "s3api, posix"
             LOCAL_FOLDER: /tmp/gw3
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
             BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
@@ -34,8 +38,10 @@ jobs:
             USERS_FOLDER: /tmp/iam3
             AWS_ENDPOINT_URL: https://127.0.0.1:7072
             RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
             PORT: 7072
-          - set: 4
+            BACKEND: "posix"
+          - set: "mc, posix"
             LOCAL_FOLDER: /tmp/gw4
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
             BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
@@ -43,17 +49,75 @@ jobs:
             USERS_FOLDER: /tmp/iam4
             AWS_ENDPOINT_URL: https://127.0.0.1:7073
             RUN_SET: "mc"
+            RECREATE_BUCKETS: "true"
             PORT: 7073
-          - set: 5
-            LOCAL_FOLDER: /tmp/gw4
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+            BACKEND: "posix"
+          - set: "s3api-user, posix, s3 IAM"
+            LOCAL_FOLDER: /tmp/gw5
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
             IAM_TYPE: s3
             USERS_BUCKET: versity-gwtest-iam
             AWS_ENDPOINT_URL: https://127.0.0.1:7074
-            RUN_SET: "aws-user"
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "true"
             PORT: 7074
-
+            BACKEND: "posix"
+          - set: "s3api non-policy, static buckets"
+            LOCAL_FOLDER: /tmp/gw6
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam6
+            AWS_ENDPOINT_URL: https://127.0.0.1:7075
+            RUN_SET: "s3api-non-policy"
+            RECREATE_BUCKETS: "false"
+            PORT: 7075
+            BACKEND: "posix"
+          - set: "s3api, s3 backend"
+            LOCAL_FOLDER: /tmp/gw7
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam7
+            AWS_ENDPOINT_URL: https://127.0.0.1:7076
+            RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
+            PORT: 7076
+            BACKEND: "s3"
+          - set: "REST, posix"
+            LOCAL_FOLDER: /tmp/gw8
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam8
+            AWS_ENDPOINT_URL: https://127.0.0.1:7077
+            RUN_SET: "rest"
+            RECREATE_BUCKETS: "true"
+            PORT: 7077
+            BACKEND: "posix"
+          - set: "s3api policy, static buckets"
+            LOCAL_FOLDER: /tmp/gw9
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-8
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-8
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam9
+            AWS_ENDPOINT_URL: https://127.0.0.1:7078
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "false"
+            PORT: 7078
+            BACKEND: "posix"
+          - set: "s3api user, static buckets"
+            LOCAL_FOLDER: /tmp/gw10
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-9
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-9
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam10
+            AWS_ENDPOINT_URL: https://127.0.0.1:7079
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "false"
+            PORT: 7079
+            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -72,6 +136,8 @@ jobs:
         run: |
           git clone https://github.com/bats-core/bats-core.git
           cd bats-core && ./install.sh $HOME
+          git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
+          git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
 
       - name: Install s3cmd
         run: |
@@ -82,6 +148,10 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
+      - name: Install xmllint (for rest)
+        run: |
+          sudo apt-get install libxml2-utils
+
       - name: Build and run, posix backend
         env:
           LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
@@ -96,19 +166,26 @@ jobs:
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
-          BACKEND: posix
-          RECREATE_BUCKETS: true
+          BACKEND: ${{ matrix.BACKEND }}
+          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
           S3CMD_CONFIG: tests/s3cfg.local.default
           MC_ALIAS: versity
           LOG_LEVEL: 4
           GOCOVERDIR: ${{ github.workspace }}/cover
+          USERNAME_ONE: ABCDEFG
+          PASSWORD_ONE: 1234567
+          USERNAME_TWO: HIJKLMN
+          PASSWORD_TWO: 8901234
+          TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
+          export AWS_ACCESS_KEY_ID_TWO=user
+          export AWS_SECRET_ACCESS_KEY_TWO=pass
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity
@@ -117,22 +194,11 @@ jobs:
           openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
           mkdir $GOCOVERDIR $USERS_FOLDER
+          if [[ $RECREATE_BUCKETS == "false" ]]; then
+            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
+          fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
-      #- name: Build and run, s3 backend
-      #  run: |
-      #    make testbin
-      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export AWS_REGION=us-east-1
-      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
-      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
-      #    aws configure set aws_region $AWS_REGION --profile versity_s3
-      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export WORKSPACE=$GITHUB_WORKSPACE
-      #    VERSITYGW_TEST_ENV=./tests/.env.s3 GOCOVERDIR=/tmp/cover ./tests/run_all.sh
-
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

.gitignore

@@ -45,6 +45,7 @@ tests/.secrets*
 
 # IAM users files often created in testing
 users.json
+users.json.backup
 
 # env files for testing
 **/.env*
@@ -59,3 +60,10 @@ tests/!s3cfg.local.default
 
 # patches
 *.patch
+
+# grafana's local database (kept on filesystem for survival between instantiations)
+metrics-exploration/grafana_data/**
+
+# bats tools
+/tests/bats-assert
+/tests/bats-support

Dockerfile

@@ -32,4 +32,4 @@ RUN mkdir -p $SETUP_DIR
 
 COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
 
-ENTRYPOINT [ "/app/versitygw" ]
+ENTRYPOINT [ "/app/versitygw" ]

auth/acl.go

@@ -28,7 +28,6 @@ import (
 )
 
 type ACL struct {
-	ACL      types.BucketCannedACL
 	Owner    string
 	Grantees []Grantee
 }
@@ -36,6 +35,7 @@ type ACL struct {
 type Grantee struct {
 	Permission types.Permission
 	Access     string
+	Type       types.Type
 }
 
 type GetBucketAclOutput struct {
@@ -43,12 +43,36 @@ type GetBucketAclOutput struct {
 	AccessControlList AccessControlList
 }
 
-type AccessControlList struct {
-	Grants []types.Grant `xml:"Grant"`
+type PutBucketAclInput struct {
+	Bucket              *string
+	ACL                 types.BucketCannedACL
+	AccessControlPolicy *AccessControlPolicy
+	GrantFullControl    *string
+	GrantRead           *string
+	GrantReadACP        *string
+	GrantWrite          *string
+	GrantWriteACP       *string
 }
+
 type AccessControlPolicy struct {
 	AccessControlList AccessControlList `xml:"AccessControlList"`
-	Owner             types.Owner
+	Owner             *types.Owner
+}
+
+type AccessControlList struct {
+	Grants []Grant `xml:"Grant"`
+}
+
+type Grant struct {
+	Grantee    *Grt
+	Permission types.Permission
+}
+
+type Grt struct {
+	XMLNS  string     `xml:"xmlns:xsi,attr"`
+	XMLXSI types.Type `xml:"xsi:type,attr"`
+	Type   types.Type `xml:"Type"`
+	ID     string     `xml:"ID"`
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -69,11 +93,19 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
-	grants := []types.Grant{}
+	grants := []Grant{}
 
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
-		grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
+		grants = append(grants, Grant{
+			Grantee: &Grt{
+				XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+				XMLXSI: elem.Type,
+				ID:     acs,
+				Type:   elem.Type,
+			},
+			Permission: elem.Permission,
+		})
 	}
 
 	return GetBucketAclOutput{
@@ -86,20 +118,43 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 	}, nil
 }
 
-func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
+func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
 	if input == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
-	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
-		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+
+	defaultGrantees := []Grantee{
+		{
+			Permission: types.PermissionFullControl,
+			Access:     acl.Owner,
+			Type:       types.TypeCanonicalUser,
+		},
 	}
 
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
-		acl.ACL = input.ACL
-		acl.Grantees = []Grantee{}
+		switch input.ACL {
+		case types.BucketCannedACLPublicRead:
+			defaultGrantees = append(defaultGrantees, Grantee{
+				Permission: types.PermissionRead,
+				Access:     "all-users",
+				Type:       types.TypeGroup,
+			})
+		case types.BucketCannedACLPublicReadWrite:
+			defaultGrantees = append(defaultGrantees, []Grantee{
+				{
+					Permission: types.PermissionRead,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+				{
+					Permission: types.PermissionWrite,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+			}...)
+		}
 	} else {
-		grantees := []Grantee{}
 		accs := []string{}
 
 		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -108,45 +163,71 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionFullControl,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantRead != nil && *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionRead,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionReadAcp,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantWrite != nil && *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionWrite,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionWriteAcp,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 
 			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
 		} else {
 			cache := make(map[string]bool)
-			for _, grt := range input.AccessControlPolicy.Grants {
-				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
+			for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
+				if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
 					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 				}
-				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
-				if _, ok := cache[*grt.Grantee.ID]; !ok {
-					cache[*grt.Grantee.ID] = true
-					accs = append(accs, *grt.Grantee.ID)
+
+				access := grt.Grantee.ID
+				defaultGrantees = append(defaultGrantees, Grantee{
+					Access:     access,
+					Permission: grt.Permission,
+					Type:       types.TypeCanonicalUser,
+				})
+				if _, ok := cache[access]; !ok {
+					cache[access] = true
+					accs = append(accs, access)
 				}
 			}
 		}
@@ -159,11 +240,10 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		if len(accList) > 0 {
 			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
 		}
-
-		acl.Grantees = grantees
-		acl.ACL = ""
 	}
 
+	acl.Grantees = defaultGrantees
+
 	result, err := json.Marshal(acl)
 	if err != nil {
 		return nil, err
@@ -207,34 +287,33 @@ func splitUnique(s, divider string) []string {
 }
 
 func verifyACL(acl ACL, access string, permission types.Permission) error {
-	if acl.ACL != "" {
-		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
+	grantee := Grantee{
+		Access:     access,
+		Permission: permission,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeFullCtrl := Grantee{
+		Access:     access,
+		Permission: types.PermissionFullControl,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeAllUsers := Grantee{
+		Access:     "all-users",
+		Permission: permission,
+		Type:       types.TypeGroup,
+	}
 
+	isFound := false
+
+	for _, grt := range acl.Grantees {
+		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
+			isFound = true
+			break
+		}
+	}
+
+	if isFound {
 		return nil
-	} else {
-		if len(acl.Grantees) == 0 {
-			return nil
-		}
-		grantee := Grantee{Access: access, Permission: permission}
-		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
-
-		isFound := false
-
-		for _, grt := range acl.Grantees {
-			if grt == grantee || grt == granteeFullCtrl {
-				isFound = true
-				break
-			}
-		}
-
-		if isFound {
-			return nil
-		}
 	}
 
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -295,23 +374,16 @@ func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) e
 	if opts.Acc.Role == RoleAdmin {
 		return nil
 	}
-	if opts.Acc.Access == opts.Acl.Owner {
-		return nil
-	}
 
 	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
-	if policyErr != nil && !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-		return policyErr
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
 	}
 
-	// If bucket policy is not set and the ACL is default, only the owner has access
-	if errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	if err := VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
-		return err
-	}
 	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
 		return err
 	}

auth/bucket_policy.go

@@ -16,12 +16,22 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
+	"errors"
 	"net/http"
 
 	"github.com/versity/versitygw/s3err"
 )
 
+var (
+	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidResource = errors.New("Policy has invalid resource")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidPrincipal = errors.New("Invalid principal in policy")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidAction = errors.New("Policy has invalid action")
+)
+
 type BucketPolicy struct {
 	Statement []BucketPolicyItem `json:"Statement"`
 }
@@ -75,11 +85,14 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 
 	for action := range bpi.Actions {
 		isObjectAction := action.IsObjectAction()
-		if isObjectAction && !containsObjectAction {
-			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		if isObjectAction == nil {
+			break
 		}
-		if !isObjectAction && !containsBucketAction {
-			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		if *isObjectAction && !containsObjectAction {
+			return errResourceMismatch
+		}
+		if !*isObjectAction && !containsBucketAction {
+			return errResourceMismatch
 		}
 	}
 
@@ -108,6 +121,11 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 		return getMalformedPolicyError(err)
 	}
 
+	if len(policy.Statement) == 0 {
+		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
+	}
+
 	if err := policy.Validate(bucket, iam); err != nil {
 		return getMalformedPolicyError(err)
 	}
@@ -116,11 +134,6 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 }
 
 func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
-	// If bucket policy is not set
-	if policy == nil {
-		return nil
-	}
-
 	var bucketPolicy BucketPolicy
 	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
 		return err

auth/bucket_policy_actions.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -56,6 +55,8 @@ const (
 	GetObjectRetentionAction               Action = "s3:GetObjectRetention"
 	PutObjectRetentionAction               Action = "s3:PutObjectRetention"
 	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
+	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
+	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
 	AllActions                             Action = "s3:*"
 )
 
@@ -92,6 +93,8 @@ var supportedActionList = map[Action]struct{}{
 	GetObjectRetentionAction:               {},
 	PutObjectRetentionAction:               {},
 	BypassGovernanceRetentionAction:        {},
+	PutBucketOwnershipControlsAction:       {},
+	GetBucketOwnershipControlsAction:       {},
 	AllActions:                             {},
 }
 
@@ -119,7 +122,7 @@ var supportedObjectActionList = map[Action]struct{}{
 // Validates Action: it should either wildcard match with supported actions list or be in it
 func (a Action) IsValid() error {
 	if !strings.HasPrefix(string(a), "s3:") {
-		return fmt.Errorf("invalid action: %v", a)
+		return errInvalidAction
 	}
 
 	if a == AllActions {
@@ -134,31 +137,39 @@ func (a Action) IsValid() error {
 			}
 		}
 
-		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+		return errInvalidAction
 	}
 
 	_, found := supportedActionList[a]
 	if !found {
-		return fmt.Errorf("unsupported action: %v", a)
+		return errInvalidAction
 	}
 	return nil
 }
 
+func getBoolPtr(bl bool) *bool {
+	return &bl
+}
+
 // Checks if the action is object action
-func (a Action) IsObjectAction() bool {
+// nil points to 's3:*'
+func (a Action) IsObjectAction() *bool {
+	if a == AllActions {
+		return nil
+	}
 	if a[len(a)-1] == '*' {
 		pattern := strings.TrimSuffix(string(a), "*")
 		for act := range supportedObjectActionList {
 			if strings.HasPrefix(string(act), pattern) {
-				return true
+				return getBoolPtr(true)
 			}
 		}
 
-		return false
+		return getBoolPtr(false)
 	}
 
 	_, found := supportedObjectActionList[a]
-	return found
+	return &found
 }
 
 func (a Action) WildCardMatch(act Action) bool {
@@ -177,7 +188,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("actions can't be empty")
+			return errInvalidAction
 		}
 		*a = make(Actions)
 		for _, s := range ss {
@@ -190,7 +201,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("actions can't be empty")
+				return errInvalidAction
 			}
 			*a = make(Actions)
 			err = a.Add(s)

auth/bucket_policy_effect.go

@@ -30,5 +30,6 @@ func (bpat BucketPolicyAccessType) Validate() error {
 		return nil
 	}
 
-	return fmt.Errorf("invalid effect: %v", bpat)
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	return fmt.Errorf("Invalid effect: %v", bpat)
 }

auth/bucket_policy_principals.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 )
 
 type Principals map[string]struct{}
@@ -37,7 +36,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		for _, s := range ss {
@@ -46,7 +45,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &s); err == nil {
 		if s == "" {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(s)
@@ -54,7 +53,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &k); err == nil {
 		if k.AWS == "" {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(k.AWS)
@@ -66,7 +65,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		}
 		if err = json.Unmarshal(data, &sk); err == nil {
 			if len(sk.AWS) == 0 {
-				return fmt.Errorf("principals can't be empty")
+				return errInvalidPrincipal
 			}
 			*p = make(Principals)
 			for _, s := range sk.AWS {
@@ -98,7 +97,7 @@ func (p Principals) Validate(iam IAMService) error {
 		if len(p) == 1 {
 			return nil
 		}
-		return fmt.Errorf("principals should either contain * or user access keys")
+		return errInvalidPrincipal
 	}
 
 	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -106,7 +105,7 @@ func (p Principals) Validate(iam IAMService) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user accounts don't exist: %v", accs)
+		return errInvalidPrincipal
 	}
 
 	return nil

auth/bucket_policy_resources.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -30,7 +29,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("resources can't be empty")
+			return errInvalidResource
 		}
 		*r = make(Resources)
 		for _, s := range ss {
@@ -43,7 +42,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("resources can't be empty")
+				return errInvalidResource
 			}
 			*r = make(Resources)
 			err = r.Add(s)
@@ -60,12 +59,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 func (r Resources) Add(rc string) error {
 	ok, pattern := isValidResource(rc)
 	if !ok {
-		return fmt.Errorf("invalid resource: %v", rc)
-	}
-
-	_, found := r[pattern]
-	if found {
-		return fmt.Errorf("duplicate resource: %v", rc)
+		return errInvalidResource
 	}
 
 	r[pattern] = struct{}{}
@@ -99,7 +93,7 @@ func (r Resources) ContainsBucketPattern() bool {
 func (r Resources) Validate(bucket string) error {
 	for resource := range r {
 		if !strings.HasPrefix(resource, bucket) {
-			return fmt.Errorf("incorrect bucket name in %v", resource)
+			return errInvalidResource
 		}
 	}
 

auth/iam.go

@@ -37,12 +37,32 @@ type Account struct {
 	GroupID int    `json:"groupID"`
 }
 
+// Mutable props, which could be changed when updating an IAM account
+type MutableProps struct {
+	Secret  *string `json:"secret"`
+	UserID  *int    `json:"userID"`
+	GroupID *int    `json:"groupID"`
+}
+
+func updateAcc(acc *Account, props MutableProps) {
+	if props.Secret != nil {
+		acc.Secret = *props.Secret
+	}
+	if props.GroupID != nil {
+		acc.GroupID = *props.GroupID
+	}
+	if props.UserID != nil {
+		acc.UserID = *props.UserID
+	}
+}
+
 // IAMService is the interface for all IAM service implementations
 //
 //go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
 type IAMService interface {
 	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
+	UpdateUserAccount(access string, props MutableProps) error
 	DeleteUserAccount(access string) error
 	ListUserAccounts() ([]Account, error)
 	Shutdown() error
@@ -56,6 +76,7 @@ var (
 )
 
 type Opts struct {
+	RootAccount            Account
 	Dir                    string
 	LDAPServerURL          string
 	LDAPBindDN             string
@@ -65,6 +86,8 @@ type Opts struct {
 	LDAPAccessAtr          string
 	LDAPSecretAtr          string
 	LDAPRoleAtr            string
+	LDAPUserIdAtr          string
+	LDAPGroupIdAtr         string
 	VaultEndpointURL       string
 	VaultSecretStoragePath string
 	VaultMountPath         string
@@ -92,20 +115,20 @@ func New(o *Opts) (IAMService, error) {
 
 	switch {
 	case o.Dir != "":
-		svc, err = NewInternal(o.Dir)
+		svc, err = NewInternal(o.RootAccount, o.Dir)
 		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
-		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
-			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
-			o.LDAPObjClasses)
+		svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
+			o.LDAPGroupIdAtr, o.LDAPObjClasses)
 		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
 	case o.S3Endpoint != "":
-		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+		svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
 			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
 		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
 			o.S3Endpoint, o.S3Bucket)
 	case o.VaultEndpointURL != "":
-		svc, err = NewVaultIAMService(o.VaultEndpointURL, o.VaultSecretStoragePath,
+		svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
 			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
 			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
 		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)

auth/iam_cache.go

@@ -66,6 +66,21 @@ func (i *icache) get(k string) (Account, bool) {
 	return v.value, true
 }
 
+func (i *icache) update(k string, props MutableProps) {
+	i.Lock()
+	defer i.Unlock()
+
+	item, found := i.items[k]
+	if found {
+		updateAcc(&item.value, props)
+
+		// refresh the expiration date
+		item.exp = time.Now().Add(i.expire)
+
+		i.items[k] = item
+	}
+}
+
 func (i *icache) Delete(k string) {
 	i.Lock()
 	delete(i.items, k)
@@ -166,6 +181,16 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
 	return nil
 }
 
+func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
+	err := c.service.UpdateUserAccount(access, props)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.update(access, props)
+	return nil
+}
+
 // ListUserAccounts is a passthrough to the underlying service and
 // does not make use of the cache
 func (c *IAMCache) ListUserAccounts() ([]Account, error) {

auth/iam_internal.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"sync"
 	"time"
 )
 
@@ -32,7 +33,15 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	dir string
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+	dir     string
+	rootAcc Account
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -46,9 +55,10 @@ type iAMConfig struct {
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(dir string) (*IAMServiceInternal, error) {
+func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		dir: dir,
+		dir:     dir,
+		rootAcc: rootAcc,
 	}
 
 	err := i.initIAM()
@@ -62,6 +72,13 @@ func NewInternal(dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	if account.Access == s.rootAcc.Access {
+		return ErrUserExists
+	}
+
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -86,6 +103,13 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -99,9 +123,41 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+// UpdateUserAccount updates the specified user account fields. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		acc, found := conf.AccessAccounts[access]
+		if !found {
+			return nil, ErrNoSuchUser
+		}
+
+		updateAcc(&acc, props)
+		conf.AccessAccounts[access] = acc
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -121,6 +177,9 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 
 // ListUserAccounts lists all the user accounts stored.
 func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return []Account{}, fmt.Errorf("get iam data: %w", err)

auth/iam_ldap.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/go-ldap/ldap/v3"
@@ -28,12 +29,16 @@ type LdapIAMService struct {
 	accessAtr  string
 	secretAtr  string
 	roleAtr    string
+	groupIdAtr string
+	userIdAtr  string
+	rootAcc    Account
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
-	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
+		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
 	conn, err := ldap.DialURL(url)
@@ -52,15 +57,23 @@ func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objCl
 		accessAtr:  accAtr,
 		secretAtr:  secAtr,
 		roleAtr:    roleAtr,
+		userIdAtr:  userIdAtr,
+		groupIdAtr: groupIdAtr,
+		rootAcc:    rootAcc,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
-	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	if ld.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
 	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
+	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
+	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -71,6 +84,9 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
 }
 
 func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ld.rootAcc.Access {
+		return ld.rootAcc, nil
+	}
 	searchRequest := ldap.NewSearchRequest(
 		ld.queryBase,
 		ldap.ScopeWholeSubtree,
@@ -79,7 +95,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
 		nil,
 	)
 
@@ -88,14 +104,48 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		return Account{}, err
 	}
 
+	if len(result.Entries) == 0 {
+		return Account{}, ErrNoSuchUser
+	}
+
 	entry := result.Entries[0]
+	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
+	}
+	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
+	}
 	return Account{
-		Access: entry.GetAttributeValue(ld.accessAtr),
-		Secret: entry.GetAttributeValue(ld.secretAtr),
-		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
+		Access:  entry.GetAttributeValue(ld.accessAtr),
+		Secret:  entry.GetAttributeValue(ld.secretAtr),
+		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
+		GroupID: groupId,
+		UserID:  userId,
 	}, nil
 }
 
+func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+	if props.Secret != nil {
+		req.Replace(ld.secretAtr, []string{*props.Secret})
+	}
+	if props.GroupID != nil {
+		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
+	}
+	if props.UserID != nil {
+		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
+	}
+
+	err := ld.conn.Modify(req)
+	//TODO: Handle non existing user case
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (ld *LdapIAMService) DeleteUserAccount(access string) error {
 	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
 
@@ -120,7 +170,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(&%v)", searchFilter),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
 		nil,
 	)
 
@@ -131,10 +181,20 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 
 	result := []Account{}
 	for _, el := range resp.Entries {
+		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
+		}
+		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
+		}
 		result = append(result, Account{
-			Access: el.GetAttributeValue(ld.accessAtr),
-			Secret: el.GetAttributeValue(ld.secretAtr),
-			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
+			Access:  el.GetAttributeValue(ld.accessAtr),
+			Secret:  el.GetAttributeValue(ld.secretAtr),
+			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
+			GroupID: groupId,
+			UserID:  userId,
 		})
 	}
 

auth/iam_s3_object.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"sort"
+	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -41,6 +42,14 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+
 	access        string
 	secret        string
 	region        string
@@ -48,12 +57,13 @@ type IAMServiceS3 struct {
 	endpoint      string
 	sslSkipVerify bool
 	debug         bool
+	rootAcc       Account
 	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}
 
-func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
 	if access == "" {
 		return nil, fmt.Errorf("must provide s3 IAM service access key")
 	}
@@ -78,6 +88,7 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 		endpoint:      endpoint,
 		sslSkipVerify: sslSkipVerify,
 		debug:         debug,
+		rootAcc:       rootAcc,
 	}
 
 	cfg, err := i.getConfig()
@@ -97,6 +108,13 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 }
 
 func (s *IAMServiceS3) CreateAccount(account Account) error {
+	if s.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -112,6 +130,13 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 }
 
 func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return Account{}, err
@@ -125,7 +150,30 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	acc, ok := conf.AccessAccounts[access]
+	if !ok {
+		return ErrNoSuchUser
+	}
+
+	updateAcc(&acc, props)
+	conf.AccessAccounts[access] = acc
+
+	return s.storeAccts(conf)
+}
+
 func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -141,6 +189,9 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
 }
 
 func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return nil, err
@@ -201,7 +252,7 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
 	})
 	if err != nil {
 		// if the error is object not exists,
-		// init empty accounts stuct and return that
+		// init empty accounts struct and return that
 		var nsk *types.NoSuchKey
 		if errors.As(err, &nsk) {
 			return iAMConfig{AccessAccounts: map[string]Account{}}, nil

auth/iam_single.go

@@ -35,6 +35,11 @@ func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
 	return Account{}, ErrNoSuchUser
 }
 
+// UpdateUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
+	return ErrNotSupported
+}
+
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
 	return ErrNotSupported

auth/iam_vault.go

@@ -30,11 +30,12 @@ type VaultIAMService struct {
 	client            *vault.Client
 	reqOpts           []vault.RequestOption
 	secretStoragePath string
+	rootAcc           Account
 }
 
 var _ IAMService = &VaultIAMService{}
 
-func NewVaultIAMService(endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
+func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
 	opts := []vault.ClientOption{
 		vault.WithAddress(endpoint),
 		// set request timeout to 10 secs
@@ -100,10 +101,14 @@ func NewVaultIAMService(endpoint, secretStoragePath, mountPath, rootToken, roleI
 		client:            client,
 		reqOpts:           reqOpts,
 		secretStoragePath: secretStoragePath,
+		rootAcc:           rootAcc,
 	}, nil
 }
 
 func (vt *VaultIAMService) CreateAccount(account Account) error {
+	if vt.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
 		Data: map[string]any{
@@ -125,6 +130,9 @@ func (vt *VaultIAMService) CreateAccount(account Account) error {
 }
 
 func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
+	if vt.rootAcc.Access == access {
+		return vt.rootAcc, nil
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
 	cancel()
@@ -140,6 +148,28 @@ func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
 	return acc, nil
 }
 
+func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	//TODO: We need something like a transaction here ?
+	acc, err := vt.GetUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	updateAcc(&acc, props)
+
+	err = vt.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	err = vt.CreateAccount(acc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (vt *VaultIAMService) DeleteUserAccount(access string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)

backend/backend.go

@@ -18,9 +18,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 	"github.com/versity/versitygw/s3select"
@@ -43,9 +43,12 @@ type Backend interface {
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
+	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
+	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
+	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
 
 	// multipart operations
-	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
@@ -56,12 +59,12 @@ type Backend interface {
 	// standard object operations
 	PutObject(context.Context, *s3.PutObjectInput) (string, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
 	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
-	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
-	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
 	DeleteObject(context.Context, *s3.DeleteObjectInput) error
 	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
@@ -90,7 +93,7 @@ type Backend interface {
 	GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
 
 	// non AWS actions
-	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
 	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
@@ -138,9 +141,18 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
 func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -167,7 +179,7 @@ func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
@@ -179,11 +191,11 @@ func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttr
 func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+	return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -256,7 +268,7 @@ func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object,
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {

backend/common.go

@@ -18,7 +18,9 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"io/fs"
+	"io"
+	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -28,9 +30,10 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
-var (
-	// RFC3339TimeFormat RFC3339 time format
-	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+const (
+	// this is the media type for directories in AWS and Nextcloud
+	DirContentType     = "application/x-directory"
+	DefaultContentType = "binary/octet-stream"
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -61,9 +64,9 @@ var (
 
 // ParseRange parses input range header and returns startoffset, length, and
 // error. If no endoffset specified, then length is set to -1.
-func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
+func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, fi.Size(), nil
+		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
@@ -92,13 +95,21 @@ func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 		return 0, 0, errInvalidRange
 	}
 
-	if endOffset <= startOffset {
+	if endOffset < startOffset {
 		return 0, 0, errInvalidRange
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
+func CreateExceedingRangeErr(objSize int64) s3err.APIError {
+	return s3err.APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
@@ -120,3 +131,16 @@ func md5String(data []byte) string {
 	sum := md5.Sum(data)
 	return hex.EncodeToString(sum[:])
 }
+
+type FileSectionReadCloser struct {
+	R io.Reader
+	F *os.File
+}
+
+func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
+	return f.R.Read(p)
+}
+
+func (f *FileSectionReadCloser) Close() error {
+	return f.F.Close()
+}

backend/s3proxy/s3.go

@@ -55,6 +55,8 @@ type S3Proxy struct {
 	debug           bool
 }
 
+var _ backend.Backend = &S3Proxy{}
+
 func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
 	s := &S3Proxy{
 		access:          access,
@@ -128,9 +130,48 @@ func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput)
 	return handleError(err)
 }
 
-func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
+	_, err := s.client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+		Bucket: &bucket,
+		OwnershipControls: &types.OwnershipControls{
+			Rules: []types.OwnershipControlsRule{
+				{
+					ObjectOwnership: ownership,
+				},
+			},
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	resp, err := s.client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return ownship, handleError(err)
+	}
+	return resp.OwnershipControls.Rules[0].ObjectOwnership, nil
+}
+func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	out, err := s.client.CreateMultipartUpload(ctx, input)
-	return out, handleError(err)
+	if err != nil {
+		return s3response.InitiateMultipartUploadResult{}, handleError(err)
+	}
+
+	return s3response.InitiateMultipartUploadResult{
+		Bucket:   *out.Bucket,
+		Key:      *out.Key,
+		UploadId: *out.UploadId,
+	}, nil
 }
 
 func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
@@ -162,8 +203,8 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
 				ID:          *u.Owner.ID,
 				DisplayName: *u.Owner.DisplayName,
 			},
-			StorageClass: string(u.StorageClass),
-			Initiated:    u.Initiated.Format(backend.RFC3339TimeFormat),
+			StorageClass: u.StorageClass,
+			Initiated:    *u.Initiated,
 		})
 	}
 
@@ -200,7 +241,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
 	for _, p := range output.Parts {
 		parts = append(parts, s3response.Part{
 			PartNumber:   int(*p.PartNumber),
-			LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
+			LastModified: *p.LastModified,
 			ETag:         *p.ETag,
 			Size:         *p.Size,
 		})
@@ -229,7 +270,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
 			ID:          *output.Owner.ID,
 			DisplayName: *output.Owner.DisplayName,
 		},
-		StorageClass:         string(output.StorageClass),
+		StorageClass:         output.StorageClass,
 		PartNumberMarker:     pnm,
 		NextPartNumberMarker: npmn,
 		MaxParts:             int(*output.MaxParts),
@@ -281,17 +322,11 @@ func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	return out, handleError(err)
 }
 
-func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	output, err := s.client.GetObject(ctx, input)
 	if err != nil {
 		return nil, handleError(err)
 	}
-	defer output.Body.Close()
-
-	_, err = io.Copy(w, output.Body)
-	if err != nil {
-		return nil, err
-	}
 
 	return output, nil
 }
@@ -327,7 +362,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
 		ETag:         out.ETag,
 		LastModified: out.LastModified,
 		ObjectSize:   out.ObjectSize,
-		StorageClass: &out.StorageClass,
+		StorageClass: out.StorageClass,
 		VersionId:    out.VersionId,
 		ObjectParts:  &parts,
 	}, handleError(err)
@@ -338,14 +373,47 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s
 	return out, handleError(err)
 }
 
-func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	out, err := s.client.ListObjects(ctx, input)
-	return out, handleError(err)
+	if err != nil {
+		return s3response.ListObjectsResult{}, handleError(err)
+	}
+
+	contents := convertObjects(out.Contents)
+
+	return s3response.ListObjectsResult{
+		CommonPrefixes: out.CommonPrefixes,
+		Contents:       contents,
+		Delimiter:      out.Delimiter,
+		IsTruncated:    out.IsTruncated,
+		Marker:         out.Marker,
+		MaxKeys:        out.MaxKeys,
+		Name:           out.Name,
+		NextMarker:     out.NextMarker,
+		Prefix:         out.Prefix,
+	}, nil
 }
 
-func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 	out, err := s.client.ListObjectsV2(ctx, input)
-	return out, handleError(err)
+	if err != nil {
+		return s3response.ListObjectsV2Result{}, handleError(err)
+	}
+
+	contents := convertObjects(out.Contents)
+
+	return s3response.ListObjectsV2Result{
+		CommonPrefixes:        out.CommonPrefixes,
+		Contents:              contents,
+		Delimiter:             out.Delimiter,
+		IsTruncated:           out.IsTruncated,
+		ContinuationToken:     out.ContinuationToken,
+		MaxKeys:               out.MaxKeys,
+		Name:                  out.Name,
+		NextContinuationToken: out.NextContinuationToken,
+		Prefix:                out.Prefix,
+		KeyCount:              out.KeyCount,
+	}, nil
 }
 
 func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
@@ -592,8 +660,12 @@ func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versio
 	return &status, nil
 }
 
-func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
+func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
+	var acll auth.ACL
+	if err := json.Unmarshal(acl, &acll); err != nil {
+		return fmt.Errorf("unmarshal acl: %w", err)
+	}
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, acll.Owner), nil)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
@@ -623,7 +695,7 @@ func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string
 			return err
 		}
 		defer resp.Body.Close()
-		return fmt.Errorf(string(body))
+		return fmt.Errorf("%v", string(body))
 	}
 
 	return nil
@@ -699,3 +771,21 @@ func base64Decode(encoded string) ([]byte, error) {
 	}
 	return decoded, nil
 }
+
+func convertObjects(objs []types.Object) []s3response.Object {
+	result := make([]s3response.Object, len(objs))
+
+	for _, obj := range objs {
+		result = append(result, s3response.Object{
+			ETag:          obj.ETag,
+			Key:           obj.Key,
+			LastModified:  obj.LastModified,
+			Owner:         obj.Owner,
+			Size:          obj.Size,
+			RestoreStatus: obj.RestoreStatus,
+			StorageClass:  obj.StorageClass,
+		})
+	}
+
+	return result
+}

backend/scoutfs/scoutfs.go

@@ -36,12 +36,14 @@ import (
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
 type ScoutfsOpts struct {
 	ChownUID    bool
 	ChownGID    bool
 	GlacierMode bool
+	BucketLinks bool
 }
 
 type ScoutFS struct {
@@ -457,6 +459,9 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
+		if errors.Is(err, syscall.ENAMETOOLONG) {
+			return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
+		}
 		if err != nil {
 			return nil, fmt.Errorf("stat part: %w", err)
 		}
@@ -486,13 +491,20 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	}
 
 	objPath := filepath.Join(bucket, object)
+
 	fi, err := os.Stat(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
+	if errors.Is(err, syscall.ENAMETOOLONG) {
+		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
+	if strings.HasSuffix(object, "/") && !fi.IsDir() {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
 
 	userMetaData := make(map[string]string)
 	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
@@ -589,7 +601,7 @@ func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, err
 	return entries[0].Name(), sum, nil
 }
 
-func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	acceptRange := *input.Range
@@ -603,15 +615,23 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 	}
 
 	objPath := filepath.Join(bucket, object)
+
 	fi, err := os.Stat(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
+	if errors.Is(err, syscall.ENAMETOOLONG) {
+		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	if strings.HasSuffix(object, "/") && !fi.IsDir() {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
+	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
 	if err != nil {
 		return nil, err
 	}
@@ -658,13 +678,8 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 	if err != nil {
 		return nil, fmt.Errorf("open object: %w", err)
 	}
-	defer f.Close()
 
 	rdr := io.NewSectionReader(f, startOffset, length)
-	_, err = io.Copy(writer, rdr)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
 
 	userMetaData := make(map[string]string)
 
@@ -694,6 +709,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
 		ContentRange:    &contentRange,
+		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
 	}, nil
 }
 
@@ -718,9 +734,9 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
 	return tags, nil
 }
 
-func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
 	bucket := *input.Bucket
 	prefix := ""
@@ -742,20 +758,20 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err)
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
+	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
 		s.fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
-		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+		return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
-	return &s3.ListObjectsOutput{
+	return s3response.ListObjectsResult{
 		CommonPrefixes: results.CommonPrefixes,
 		Contents:       results.Objects,
 		Delimiter:      &delim,
@@ -768,9 +784,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
 	}, nil
 }
 
-func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
 	bucket := *input.Bucket
 	prefix := ""
@@ -792,20 +808,20 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err)
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
+	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys),
 		s.fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
-		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+		return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
-	return &s3.ListObjectsV2Output{
+	return s3response.ListObjectsV2Result{
 		CommonPrefixes:        results.CommonPrefixes,
 		Contents:              results.Objects,
 		Delimiter:             &delim,
@@ -819,50 +835,58 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
 }
 
 func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
-	return func(path string, d fs.DirEntry) (types.Object, error) {
+	return func(path string, d fs.DirEntry) (s3response.Object, error) {
 		objPath := filepath.Join(bucket, path)
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
-			if err != nil {
-				return types.Object{}, fmt.Errorf("get etag: %w", err)
+			etagBytes, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
+			if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
+				return s3response.Object{}, backend.ErrSkipObj
 			}
-			etag := string(b)
+			if err != nil {
+				return s3response.Object{}, fmt.Errorf("get etag: %w", err)
+			}
+			etag := string(etagBytes)
 
 			fi, err := d.Info()
 			if errors.Is(err, fs.ErrNotExist) {
-				return types.Object{}, backend.ErrSkipObj
+				return s3response.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+				return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 			}
 
 			key := path + "/"
+			mtime := fi.ModTime()
 
-			return types.Object{
+			return s3response.Object{
 				ETag:         &etag,
 				Key:          &key,
-				LastModified: backend.GetTimePtr(fi.ModTime()),
+				LastModified: &mtime,
+				StorageClass: types.ObjectStorageClassStandard,
 			}, nil
 		}
 
 		// file object, get object info and fill out object data
 		b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
-			return types.Object{}, backend.ErrSkipObj
+			return s3response.Object{}, backend.ErrSkipObj
 		}
-		if err != nil {
-			return types.Object{}, fmt.Errorf("get etag: %w", err)
+		if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+			return s3response.Object{}, fmt.Errorf("get etag: %w", err)
 		}
+		// note: meta.ErrNoSuchKey will return etagBytes = []byte{}
+		// so this will just set etag to "" if its not already set
+
 		etag := string(b)
 
 		fi, err := d.Info()
 		if errors.Is(err, fs.ErrNotExist) {
-			return types.Object{}, backend.ErrSkipObj
+			return s3response.Object{}, backend.ErrSkipObj
 		}
 		if err != nil {
-			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
 
 		sc := types.ObjectStorageClassStandard
@@ -871,10 +895,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 			// If so, we will return the InvalidObjectState error.
 			st, err := statMore(objPath)
 			if errors.Is(err, fs.ErrNotExist) {
-				return types.Object{}, backend.ErrSkipObj
+				return s3response.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return types.Object{}, fmt.Errorf("stat more: %w", err)
+				return s3response.Object{}, fmt.Errorf("stat more: %w", err)
 			}
 			if st.Offline_blocks != 0 {
 				sc = types.ObjectStorageClassGlacier
@@ -882,11 +906,12 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		}
 
 		size := fi.Size()
+		mtime := fi.ModTime()
 
-		return types.Object{
+		return s3response.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: backend.GetTimePtr(fi.ModTime()),
+			LastModified: &mtime,
 			Size:         &size,
 			StorageClass: sc,
 		}, nil
@@ -969,15 +994,9 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 }
 
 func isNoAttr(err error) bool {
-	if err == nil {
-		return false
-	}
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == errNoData {
-		return true
-	}
 	return false
 }

backend/scoutfs/scoutfs_compat.go

@@ -37,8 +37,9 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	metastore := meta.XattrMeta{}
 
 	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
-		ChownUID: opts.ChownUID,
-		ChownGID: opts.ChownGID,
+		ChownUID:    opts.ChownUID,
+		ChownGID:    opts.ChownGID,
+		BucketLinks: opts.BucketLinks,
 	})
 	if err != nil {
 		return nil, err
@@ -50,12 +51,13 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:    p,
-		rootfd:   f,
-		rootdir:  rootdir,
-		meta:     metastore,
-		chownuid: opts.ChownUID,
-		chowngid: opts.ChownGID,
+		Posix:       p,
+		rootfd:      f,
+		rootdir:     rootdir,
+		meta:        metastore,
+		chownuid:    opts.ChownUID,
+		chowngid:    opts.ChownGID,
+		glaciermode: opts.GlacierMode,
 	}, nil
 }
 

backend/scoutfs/with_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build !freebsd && !openbsd && !netbsd
-// +build !freebsd,!openbsd,!netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENODATA
-)

backend/scoutfs/without_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build freebsd || openbsd || netbsd
-// +build freebsd openbsd netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENOATTR
-)

backend/walk.go

@@ -15,6 +15,7 @@
 package backend
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -23,24 +24,25 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3response"
 )
 
 type WalkResults struct {
 	CommonPrefixes []types.CommonPrefix
-	Objects        []types.Object
+	Objects        []s3response.Object
 	Truncated      bool
 	NextMarker     string
 }
 
-type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
+type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
 
 var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
-	var objects []types.Object
+	var objects []s3response.Object
 
 	var pastMarker bool
 	if marker == "" {
@@ -55,6 +57,9 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		if err != nil {
 			return err
 		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
 		// Ignore the root directory
 		if path == "." {
 			return nil
@@ -91,7 +96,10 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			if err != nil {
 				return fmt.Errorf("readdir %q: %w", path, err)
 			}
-			if len(ents) == 0 {
+
+			path += string(os.PathSeparator)
+
+			if len(ents) == 0 && delimiter == "" {
 				dirobj, err := getObj(path, d)
 				if err == ErrSkipObj {
 					return nil
@@ -100,9 +108,13 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 					return fmt.Errorf("directory to object %q: %w", path, err)
 				}
 				objects = append(objects, dirobj)
+
+				return nil
 			}
 
-			return nil
+			if len(ents) != 0 {
+				return nil
+			}
 		}
 
 		if !pastMarker {
@@ -179,10 +191,24 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
-		// delimiter at the end.
-		cpmap[prefix+before+delimiter] = struct{}{}
+		// delimiter at the end when we add to the map.
+		cprefNoDelim := prefix + before
+		cpref := prefix + before + delimiter
+		if cpref == marker {
+			pastMarker = true
+			return nil
+		}
+
+		if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
+			// skip common prefixes that are before the marker
+			return nil
+		}
+
+		cpmap[cpref] = struct{}{}
 		if (len(objects) + len(cpmap)) == int(max) {
-			pastMax = true
+			newMarker = cpref
+			truncated = true
+			return fs.SkipAll
 		}
 
 		return nil

backend/walk_test.go

@@ -15,15 +15,19 @@
 package backend_test
 
 import (
+	"context"
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
 	"io/fs"
+	"sync"
 	"testing"
 	"testing/fstest"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3response"
 )
 
 type walkTest struct {
@@ -32,19 +36,20 @@ type walkTest struct {
 	getobj   backend.GetObjFunc
 }
 
-func getObj(path string, d fs.DirEntry) (types.Object, error) {
+func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
 	if d.IsDir() {
 		etag := getMD5(path)
 
 		fi, err := d.Info()
 		if err != nil {
-			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
+		mtime := fi.ModTime()
 
-		return types.Object{
+		return s3response.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: backend.GetTimePtr(fi.ModTime()),
+			LastModified: &mtime,
 		}, nil
 	}
 
@@ -52,15 +57,16 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
 
 	fi, err := d.Info()
 	if err != nil {
-		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 	}
 
 	size := fi.Size()
+	mtime := fi.ModTime()
 
-	return types.Object{
+	return s3response.Object{
 		ETag:         &etag,
 		Key:          &path,
-		LastModified: backend.GetTimePtr(fi.ModTime()),
+		LastModified: &mtime,
 		Size:         &size,
 	}, nil
 }
@@ -86,7 +92,7 @@ func TestWalk(t *testing.T) {
 				CommonPrefixes: []types.CommonPrefix{{
 					Prefix: backend.GetStringPtr("photos/"),
 				}},
-				Objects: []types.Object{{
+				Objects: []s3response.Object{{
 					Key: backend.GetStringPtr("sample.jpg"),
 				}},
 			},
@@ -101,14 +107,14 @@ func TestWalk(t *testing.T) {
 				CommonPrefixes: []types.CommonPrefix{{
 					Prefix: backend.GetStringPtr("test/"),
 				}},
-				Objects: []types.Object{},
+				Objects: []s3response.Object{},
 			},
 			getobj: getObj,
 		},
 	}
 
 	for _, tt := range tests {
-		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
+		res, err := backend.Walk(context.Background(), tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
 		if err != nil {
 			t.Fatalf("walk: %v", err)
 		}
@@ -168,7 +174,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
 	return res + "]"
 }
 
-func compareObjects(a, b []types.Object) bool {
+func compareObjects(a, b []s3response.Object) bool {
 	if len(a) == 0 && len(b) == 0 {
 		return true
 	}
@@ -184,7 +190,7 @@ func compareObjects(a, b []types.Object) bool {
 	return false
 }
 
-func containsObject(c types.Object, list []types.Object) bool {
+func containsObject(c s3response.Object, list []s3response.Object) bool {
 	for _, cp := range list {
 		if *c.Key == *cp.Key {
 			return true
@@ -193,7 +199,7 @@ func containsObject(c types.Object, list []types.Object) bool {
 	return false
 }
 
-func printObjects(list []types.Object) string {
+func printObjects(list []s3response.Object) string {
 	res := "["
 	for _, cp := range list {
 		if res == "[" {
@@ -204,3 +210,50 @@ func printObjects(list []types.Object) string {
 	}
 	return res + "]"
 }
+
+type slowFS struct {
+	fstest.MapFS
+}
+
+const (
+	readDirPause = 100 * time.Millisecond
+
+	// walkTimeOut should be less than the tree traversal time
+	// which is the readdirPause time * the number of directories
+	walkTimeOut = 500 * time.Millisecond
+)
+
+func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
+	time.Sleep(readDirPause)
+	return s.MapFS.ReadDir(name)
+}
+
+func TestWalkStop(t *testing.T) {
+	s := &slowFS{MapFS: fstest.MapFS{
+		"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
+	}}
+
+	ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
+	defer cancel()
+
+	var err error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, err = backend.Walk(ctx, s, "", "/", "", 1000,
+			func(path string, d fs.DirEntry) (s3response.Object, error) {
+				return s3response.Object{}, nil
+			}, []string{})
+	}()
+
+	select {
+	case <-time.After(1 * time.Second):
+		t.Fatalf("walk is not terminated in time")
+	case <-ctx.Done():
+	}
+	wg.Wait()
+	if err != ctx.Err() {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}

cmd/versitygw/admin.go

@@ -82,6 +82,34 @@ func adminCommand() *cli.Command {
 					},
 				},
 			},
+			{
+				Name:   "update-user",
+				Usage:  "Updates a user account",
+				Action: updateUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "user access key id to be updated",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:    "secret",
+						Usage:   "secret access key for the new user",
+						Aliases: []string{"s"},
+					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+				},
+			},
 			{
 				Name:   "delete-user",
 				Usage:  "Delete a user",
@@ -276,6 +304,63 @@ func deleteUser(ctx *cli.Context) error {
 	return nil
 }
 
+func updateUser(ctx *cli.Context) error {
+	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
+	props := auth.MutableProps{}
+	if ctx.IsSet("secret") {
+		props.Secret = &secret
+	}
+	if ctx.IsSet("user-id") {
+		props.UserID = &userId
+	}
+	if ctx.IsSet("group-id") {
+		props.GroupID = &groupId
+	}
+
+	propsJSON, err := json.Marshal(props)
+	if err != nil {
+		return fmt.Errorf("failed to parse user attributes: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(propsJSON)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
 func listUsers(ctx *cli.Context) error {
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
 	if err != nil {

cmd/versitygw/main.go

@@ -19,9 +19,11 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
+	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
@@ -45,8 +47,8 @@ var (
 	natsURL, natsTopic                       string
 	eventWebhookURL                          string
 	eventConfigFilePath                      string
-	logWebhookURL                            string
-	accessLog                                string
+	logWebhookURL, accessLog                 string
+	adminLogFile                             string
 	healthPath                               string
 	debug                                    bool
 	pprof                                    string
@@ -56,6 +58,7 @@ var (
 	ldapURL, ldapBindDN, ldapPassword        string
 	ldapQueryBase, ldapObjClasses            string
 	ldapAccessAtr, ldapSecAtr, ldapRoleAtr   string
+	ldapUserIdAtr, ldapGroupIdAtr            string
 	vaultEndpointURL, vaultSecretStoragePath string
 	vaultMountPath, vaultRootToken           string
 	vaultRoleId, vaultRoleSecret             string
@@ -111,10 +114,13 @@ func main() {
 
 func initApp() *cli.App {
 	return &cli.App{
-		Name:  "versitygw",
-		Usage: "Start S3 gateway service with specified backend storage.",
-		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
-to access the supported backend storage as if it was a native S3 service.`,
+		Usage: "Versity S3 Gateway",
+		Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.
+VersityGW is an open-source project licensed under the Apache 2.0 License. The
+source code is hosted on GitHub at https://github.com/versity/versitygw, and
+documentation can be found in the GitHub wiki.`,
+		Copyright: "Copyright (c) 2023-2024 Versity Software",
 		Action: func(ctx *cli.Context) error {
 			return ctx.App.Command("help").Run(ctx)
 		},
@@ -222,6 +228,12 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
 			Destination: &accessLog,
 		},
+		&cli.StringFlag{
+			Name:        "admin-access-log",
+			Usage:       "enable admin server access logging to specified file",
+			EnvVars:     []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
+			Destination: &adminLogFile,
+		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
@@ -331,6 +343,18 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-user-id-atr",
+			Usage:       "ldap server user id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_USER_ID_ATR"},
+			Destination: &ldapUserIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-group-id-atr",
+			Usage:       "ldap server user group id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
+			Destination: &ldapGroupIdAtr,
+		},
 		&cli.StringFlag{
 			Name:        "iam-vault-endpoint-url",
 			Usage:       "vault server url",
@@ -499,10 +523,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	app := fiber.New(fiber.Config{
-		AppName:           "versitygw",
-		ServerHeader:      "VERSITYGW",
-		StreamRequestBody: true,
-		DisableKeepalive:  true,
+		AppName:               "versitygw",
+		ServerHeader:          "VERSITYGW",
+		StreamRequestBody:     true,
+		DisableKeepalive:      true,
+		Network:               fiber.NetworkTCP,
+		DisableStartupMessage: true,
 	})
 
 	var opts []s3api.Option
@@ -538,8 +564,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	admApp := fiber.New(fiber.Config{
-		AppName:      "versitygw",
-		ServerHeader: "VERSITYGW",
+		AppName:               "versitygw",
+		ServerHeader:          "VERSITYGW",
+		Network:               fiber.NetworkTCP,
+		DisableStartupMessage: true,
 	})
 
 	var admOpts []s3api.AdminOpt
@@ -560,6 +588,11 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
+		RootAccount: auth.Account{
+			Access: rootUserAccess,
+			Secret: rootUserSecret,
+			Role:   auth.RoleAdmin,
+		},
 		Dir:                    iamDir,
 		LDAPServerURL:          ldapURL,
 		LDAPBindDN:             ldapBindDN,
@@ -569,6 +602,8 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		LDAPAccessAtr:          ldapAccessAtr,
 		LDAPSecretAtr:          ldapSecAtr,
 		LDAPRoleAtr:            ldapRoleAtr,
+		LDAPUserIdAtr:          ldapUserIdAtr,
+		LDAPGroupIdAtr:         ldapGroupIdAtr,
 		VaultEndpointURL:       vaultEndpointURL,
 		VaultSecretStoragePath: vaultSecretStoragePath,
 		VaultMountPath:         vaultMountPath,
@@ -593,9 +628,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		return fmt.Errorf("setup iam: %w", err)
 	}
 
-	logger, err := s3log.InitLogger(&s3log.LogConfig{
-		LogFile:    accessLog,
-		WebhookURL: logWebhookURL,
+	loggers, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:      accessLog,
+		WebhookURL:   logWebhookURL,
+		AdminLogFile: adminLogFile,
 	})
 	if err != nil {
 		return fmt.Errorf("setup logger: %w", err)
@@ -626,12 +662,16 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
 		Access: rootUserAccess,
 		Secret: rootUserSecret,
-	}, port, region, iam, logger, evSender, metricsManager, opts...)
+	}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
 	if err != nil {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
+
+	if !quiet {
+		printBanner(port, admPort, certFile != "", admCertFile != "")
+	}
 
 	c := make(chan error, 2)
 	go func() { c <- srv.Serve() }()
@@ -648,10 +688,17 @@ Loop:
 		case err = <-c:
 			break Loop
 		case <-sigHup:
-			if logger != nil {
-				err = logger.HangUp()
+			if loggers.S3Logger != nil {
+				err = loggers.S3Logger.HangUp()
 				if err != nil {
-					err = fmt.Errorf("HUP logger: %w", err)
+					err = fmt.Errorf("HUP s3 logger: %w", err)
+					break Loop
+				}
+			}
+			if loggers.AdminLogger != nil {
+				err = loggers.AdminLogger.HangUp()
+				if err != nil {
+					err = fmt.Errorf("HUP admin logger: %w", err)
 					break Loop
 				}
 			}
@@ -669,13 +716,22 @@ Loop:
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
-	if logger != nil {
-		err := logger.Shutdown()
+	if loggers.S3Logger != nil {
+		err := loggers.S3Logger.Shutdown()
 		if err != nil {
 			if saveErr == nil {
 				saveErr = err
 			}
-			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
+			fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
+		}
+	}
+	if loggers.AdminLogger != nil {
+		err := loggers.AdminLogger.Shutdown()
+		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
+			fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
 		}
 	}
 
@@ -695,3 +751,177 @@ Loop:
 
 	return saveErr
 }
+
+func printBanner(port, admPort string, ssl, admSsl bool) {
+	interfaces, err := getMatchingIPs(port)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
+		return
+	}
+
+	var admInterfaces []string
+	if admPort != "" {
+		admInterfaces, err = getMatchingIPs(admPort)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
+			return
+		}
+	}
+
+	title := "VersityGW"
+	version := fmt.Sprintf("Version %v, Build %v", Version, Build)
+	urls := []string{}
+
+	hst, prt, err := net.SplitHostPort(port)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
+		return
+	}
+
+	for _, ip := range interfaces {
+		url := fmt.Sprintf("http://%s:%s", ip, prt)
+		if ssl {
+			url = fmt.Sprintf("https://%s:%s", ip, prt)
+		}
+		urls = append(urls, url)
+	}
+
+	if hst == "" {
+		hst = "0.0.0.0"
+	}
+
+	boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
+
+	lines := []string{
+		centerText(title),
+		centerText(version),
+		centerText(boundHost),
+		centerText(""),
+	}
+
+	if len(admInterfaces) > 0 {
+		lines = append(lines,
+			leftText("S3 service listening on:"),
+		)
+	} else {
+		lines = append(lines,
+			leftText("Admin/S3 service listening on:"),
+		)
+	}
+
+	for _, url := range urls {
+		lines = append(lines, leftText("  "+url))
+	}
+
+	if len(admInterfaces) > 0 {
+		lines = append(lines,
+			centerText(""),
+			leftText("Admin service listening on:"),
+		)
+
+		_, prt, err := net.SplitHostPort(admPort)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
+			return
+		}
+
+		for _, ip := range admInterfaces {
+			url := fmt.Sprintf("http://%s:%s", ip, prt)
+			if admSsl {
+				url = fmt.Sprintf("https://%s:%s", ip, prt)
+			}
+			lines = append(lines, leftText("  "+url))
+		}
+	}
+
+	// Print the top border
+	fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
+
+	// Print each line
+	for _, line := range lines {
+		fmt.Printf("│%-*s│\n", columnWidth-2, line)
+	}
+
+	// Print the bottom border
+	fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
+}
+
+// getMatchingIPs returns all IP addresses for local system interfaces that
+// match the input address specification.
+func getMatchingIPs(spec string) ([]string, error) {
+	// Split the input spec into IP and port
+	host, _, err := net.SplitHostPort(spec)
+	if err != nil {
+		return nil, fmt.Errorf("parse address/port: %v", err)
+	}
+
+	// Handle cases where IP is omitted (e.g., ":1234")
+	if host == "" {
+		host = "0.0.0.0"
+	}
+
+	ipaddr, err := net.ResolveIPAddr("ip", host)
+	if err != nil {
+		return nil, err
+	}
+
+	parsedInputIP := ipaddr.IP
+
+	var result []string
+
+	// Get all network interfaces
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, iface := range interfaces {
+		// Get all addresses associated with the interface
+		addrs, err := iface.Addrs()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, addr := range addrs {
+			// Parse the address to get the IP part
+			ipAddr, _, err := net.ParseCIDR(addr.String())
+			if err != nil {
+				return nil, err
+			}
+
+			if ipAddr.IsLinkLocalUnicast() {
+				continue
+			}
+			if ipAddr.IsInterfaceLocalMulticast() {
+				continue
+			}
+			if ipAddr.IsLinkLocalMulticast() {
+				continue
+			}
+
+			// Check if the IP matches the input specification
+			if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
+				result = append(result, ipAddr.String())
+			}
+		}
+	}
+
+	return result, nil
+}
+
+const columnWidth = 70
+
+func centerText(text string) string {
+	padding := (columnWidth - 2 - len(text)) / 2
+	if padding < 0 {
+		padding = 0
+	}
+	return strings.Repeat(" ", padding) + text
+}
+
+func leftText(text string) string {
+	if len(text) > columnWidth-2 {
+		return text
+	}
+	return text + strings.Repeat(" ", columnWidth-2-len(text))
+}

cmd/versitygw/posix.go

@@ -24,6 +24,7 @@ import (
 
 var (
 	chownuid, chowngid bool
+	bucketlinks        bool
 )
 
 func posixCommand() *cli.Command {
@@ -54,6 +55,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_CHOWN_GID"},
 				Destination: &chowngid,
 			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
 		},
 	}
 }
@@ -70,8 +77,9 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
-		ChownUID: chownuid,
-		ChownGID: chowngid,
+		ChownUID:    chownuid,
+		ChownGID:    chowngid,
+		BucketLinks: bucketlinks,
 	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)

cmd/versitygw/scoutfs.go

@@ -63,6 +63,12 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_CHOWN_GID"},
 				Destination: &chowngid,
 			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
 		},
 	}
 }
@@ -76,6 +82,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.GlacierMode = glacier
 	opts.ChownUID = chownuid
 	opts.ChownGID = chowngid
+	opts.BucketLinks = bucketlinks
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

extra/dashboard/README.md

@@ -0,0 +1,19 @@
+# Versity Gateway Dashboard
+
+This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana. 
+
+The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
+
+This implementation uses the influxql query language. 
+
+## Usage
+
+From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
+
+To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`. 
+
+The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
+
+The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with  username: `admin` and password: `admin`.
+
+To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

extra/dashboard/aws_env_setup.sh

@@ -0,0 +1,4 @@
+export AWS_SECRET_ACCESS_KEY=password
+export AWS_ACCESS_KEY_ID=user
+export AWS_ENDPOINT_URL=http://127.0.0.1:7070
+export AWS_REGION=us-east-1

extra/dashboard/docker-compose-metrics.yml

@@ -0,0 +1,64 @@
+services:
+  telegraf:
+    image: telegraf
+    container_name: telegraf
+    restart: always
+    volumes:
+    - ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      - influxdb
+    links:
+      - influxdb
+    ports:
+    - '8125:8125/udp'
+
+  influxdb:
+    image: influxdb
+    container_name: influxdb
+    restart: always
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
+      - DOCKER_INFLUXDB_INIT_ORG=myorg
+      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb_data:/var/lib/influxdb
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana-server
+    restart: always
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_INSTALL_PLUGINS=
+    links:
+      - influxdb
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./metrics-exploration/grafana_data:/var/lib/grafana
+      - ./metrics-exploration/provisioning:/etc/grafana/provisioning
+
+  versitygw:
+    image: versity/versitygw:latest
+    container_name: versitygw
+    ports:
+      - "7070:7070"
+    environment:
+      - ROOT_ACCESS_KEY=user
+      - ROOT_SECRET_KEY=password
+      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
+    depends_on:
+      - telegraf
+    command: >
+      posix /tmp/vgw
+
+volumes:
+  influxdb_data: {}

extra/dashboard/docker-compose.yml

@@ -0,0 +1,64 @@
+services:
+  telegraf:
+    image: telegraf
+    container_name: telegraf
+    restart: always
+    volumes:
+    - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      - influxdb
+    links:
+      - influxdb
+    ports:
+    - '8125:8125/udp'
+
+  influxdb:
+    image: influxdb
+    container_name: influxdb
+    restart: always
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
+      - DOCKER_INFLUXDB_INIT_ORG=myorg
+      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb_data:/var/lib/influxdb
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana-server
+    restart: always
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_INSTALL_PLUGINS=
+    links:
+      - influxdb
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./grafana_data:/var/lib/grafana
+      - ./provisioning:/etc/grafana/provisioning
+
+  versitygw:
+    image: versity/versitygw:latest
+    container_name: versitygw
+    ports:
+      - "7070:7070"
+    environment:
+      - ROOT_ACCESS_KEY=user
+      - ROOT_SECRET_KEY=password
+      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
+    depends_on:
+      - telegraf
+    command: >
+      posix /tmp/vgw
+
+volumes:
+  influxdb_data: {}

extra/dashboard/provisioning/dashboards/influxql.yaml

@@ -0,0 +1,25 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'influxql'
+    # <int> Org id. Default to 1
+    orgId: 1
+    # <string> name of the dashboard folder.
+    folder: 'influxql'
+    # <string> folder UID. will be automatically generated if not specified
+    folderUid: ''
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: true
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /etc/grafana/provisioning/dashboards/influxql
+      # <bool> use folder names from filesystem to create folders in Grafana
+      foldersFromFilesStructure: true
+

extra/dashboard/provisioning/datasources/influxdb.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+datasources:
+  - name: influxdb
+    type: influxdb
+    isDefault: true
+    access: proxy
+    url: http://influxdb:8086
+    jsonData:
+      dbName: 'metrics'
+      httpHeaderName1: 'Authorization'
+    secureJsonData:
+      httpHeaderValue1: 'Token my-super-secret-auth-token'

extra/dashboard/telegraf.conf

@@ -0,0 +1,34 @@
+[global_tags]
+
+[agent]
+  debug = true
+  quiet = false
+  interval = "60s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  hostname = "versitygw"
+  omit_hostname = false
+
+[[outputs.file]]
+  files = ["stdout"]
+
+[[outputs.influxdb_v2]]
+  urls = ["http://influxdb:8086"]
+  timeout = "5s"
+  token = "my-super-secret-auth-token"
+  organization = "myorg"
+  bucket = "metrics"
+
+[[inputs.statsd]]
+  protocol = "udp4"
+  service_address = ":8125"
+  percentiles = [90]
+  metric_separator = "_"
+  datadog_extensions = false
+  allowed_pending_messages = 10000
+  percentile_limit = 1000

extra/dashboard/test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+. ./aws_env_setup.sh
+
+aws s3 mb s3://test
+aws s3 cp docker-compose.yml s3://test/test.yaml

extra/example.conf

@@ -237,6 +237,8 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_LDAP_ACCESS_ATR=
 #VGW_IAM_LDAP_SECRET_ATR=
 #VGW_IAM_LDAP_ROLE_ATR=
+#VGW_IAM_LDAP_USER_ID_ATR=
+#VGW_IAM_LDAP_GROUP_ID_ATR=
 
 ###############
 # IAM caching #
@@ -305,6 +307,10 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_CHOWN_UID=false
 #VGW_CHOWN_GID=false
 
+# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
+# to directories at the top level gateway directory as buckets.
+#VGW_BUCKET_LINKS=false
+
 ###########
 # scoutfs #
 ###########
@@ -336,6 +342,10 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_CHOWN_UID=false
 #VGW_CHOWN_GID=false
 
+# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
+# to directories at the top level gateway directory as buckets.
+#VGW_BUCKET_LINKS=false
+
 ######
 # s3 #
 ######

go.mod

@@ -3,45 +3,44 @@ module github.com/versity/versitygw
 go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.27.2
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.55.1
-	github.com/aws/smithy-go v1.20.2
+	github.com/aws/aws-sdk-go-v2 v1.30.5
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2
+	github.com/aws/smithy-go v1.20.4
 	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/gofiber/fiber/v2 v2.52.4
+	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.35.0
-	github.com/pkg/xattr v0.4.9
+	github.com/nats-io/nats.go v1.37.0
+	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.3
-	github.com/urfave/cli/v2 v2.27.2
-	github.com/valyala/fasthttp v1.54.0
+	github.com/urfave/cli/v2 v2.27.4
+	github.com/valyala/fasthttp v1.55.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sys v0.21.0
+	golang.org/x/sys v0.25.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
@@ -49,33 +48,33 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.18
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.18
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.24
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.9 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.34
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.32
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.19
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 )

go.sum

@@ -1,13 +1,13 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
@@ -21,46 +21,46 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.27.2 h1:pLsTXqX93rimAOZG2FIYraDQstZaaGVVN4tNw65v0h8=
-github.com/aws/aws-sdk-go-v2 v1.27.2/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
-github.com/aws/aws-sdk-go-v2/config v1.27.18 h1:wFvAnwOKKe7QAyIxziwSKjmer9JBMH1vzIL6W+fYuKk=
-github.com/aws/aws-sdk-go-v2/config v1.27.18/go.mod h1:0xz6cgdX55+kmppvPm2IaKzIXOheGJhAufacPJaXZ7c=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.18 h1:D/ALDWqK4JdY3OFgA2thcPO1c9aYTT5STS/CvnkqY1c=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.18/go.mod h1:JuitCWq+F5QGUrmMPsk945rop6bB57jdscu+Glozdnc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 h1:dDgptDO9dxeFkXy+tEgVkzSClHZje/6JkPW5aZyEvrQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5/go.mod h1:gjvE2KBUgUQhcv89jqxrIxH9GaKs1JbZzWejj/DaHGA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.24 h1:FzNwpVTZDCvm597Ty6mGYvxTolyC1oup0waaKntZI4E=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.24/go.mod h1:wM9NElT/Wn6n3CT1eyVcXtfCy8lSVjjQXfdawQbSShc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 h1:cy8ahBJuhtM8GTTSyOkfy6WVPV1IE+SS5/wfXUYuulw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9/go.mod h1:CZBXGLaJnEZI6EVNcPd7a6B5IC5cA/GkRWtu9fp3S6Y=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 h1:A4SYk07ef04+vxZToz9LWvAXl9LW0NClpPpMsi31cz0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9/go.mod h1:5jJcHuwDagxN+ErjQ3PU3ocf6Ylc/p9x+BLO/+X4iXw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.9 h1:vHyZxoLVOgrI8GqX7OMHLXp4YYoxeEsrjweXKpye+ds=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.9/go.mod h1:z9VXZsWA2BvZNH1dT0ToUYwMu/CR9Skkj/TBX+mceZw=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.11 h1:4vt9Sspk59EZyHCAEMaktHKiq0C09noRTQorXD/qV+s=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.11/go.mod h1:5jHR79Tv+Ccq6rwYh+W7Nptmw++WiFafMfR42XhwNl8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11 h1:o4T+fKxA3gTMcluBNZZXE9DNaMkJuUL1O3mffCUjoJo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11/go.mod h1:84oZdJ+VjuJKs9v1UTC9NaodRZRseOXCTgku+vQJWR8=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.9 h1:TE2i0A9ErH1YfRSvXfCr2SQwfnqsoJT9nPQ9kj0lkxM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.9/go.mod h1:9TzXX3MehQNGPwCZ3ka4CpwQsoAMWSF48/b+De9rfVM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.55.1 h1:UAxBuh0/8sFJk1qOkvOKewP5sWeWaTPDknbQz0ZkDm0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.55.1/go.mod h1:hWjsYGjVuqCgfoveVcVFPXIWgz0aByzwaxKlN1StKcM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.11 h1:gEYM2GSpr4YNWc6hCd5nod4+d4kd9vWIAWrmGuLdlMw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.11/go.mod h1:gVvwPdPNYehHSP9Rs7q27U1EU+3Or2ZpXvzAYJNh63w=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5 h1:iXjh3uaH3vsVcnyZX7MqCoCfcyxIrVE9iOQruRaWPrQ=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5/go.mod h1:5ZXesEuy/QcO0WUnt+4sDkxhdXRHTu2yG0uCSH8B6os=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 h1:M/1u4HBpwLuMtjlxuI2y6HoVLzF5e2mfxHCg7ZVMYmk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.12/go.mod h1:kcfd+eTdEi/40FIbLq4Hif3XMXnl5b/+t/KTfLt9xIk=
-github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
-github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
+github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
+github.com/aws/aws-sdk-go-v2/config v1.27.34 h1:5sLceuETg/215nLtY/QIVB2O6cosS0iC/Tx5oyqUhbw=
+github.com/aws/aws-sdk-go-v2/config v1.27.34/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.19 h1:g4KNa/rtxMNG8x+y6giri1zT0jC/7I22M9tyU1EWeqs=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.19/go.mod h1:BQTlyIEkP7fcOgmHPqkLNGWiZE52efuX2jS7AcGwTyw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o=
+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -71,8 +71,8 @@ github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl5
 github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
 github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
-github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
+github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
@@ -109,13 +109,9 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -123,12 +119,12 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.35.0 h1:XFNqNM7v5B+MQMKqVGAyHwYhyKb48jrenXNxIU20ULk=
-github.com/nats-io/nats.go v1.35.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
+github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -139,8 +135,8 @@ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
-github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
+github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -166,12 +162,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
-github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
+github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
+github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0=
-github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM=
+github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
+github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -182,8 +178,8 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -193,8 +189,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -210,8 +206,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -235,8 +231,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -252,10 +248,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
+golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
@@ -266,9 +262,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

metrics/actions.go

@@ -24,51 +24,54 @@ var (
 )
 
 var (
-	ActionUndetected                 = "ActionUnDetected"
-	ActionAbortMultipartUpload       = "s3_AbortMultipartUpload"
-	ActionCompleteMultipartUpload    = "s3_CompleteMultipartUpload"
-	ActionCopyObject                 = "s3_CopyObject"
-	ActionCreateBucket               = "s3_CreateBucket"
-	ActionCreateMultipartUpload      = "s3_CreateMultipartUpload"
-	ActionDeleteBucket               = "s3_DeleteBucket"
-	ActionDeleteBucketPolicy         = "s3_DeleteBucketPolicy"
-	ActionDeleteBucketTagging        = "s3_DeleteBucketTagging"
-	ActionDeleteObject               = "s3_DeleteObject"
-	ActionDeleteObjectTagging        = "s3_DeleteObjectTagging"
-	ActionDeleteObjects              = "s3_DeleteObjects"
-	ActionGetBucketAcl               = "s3_GetBucketAcl"
-	ActionGetBucketPolicy            = "s3_GetBucketPolicy"
-	ActionGetBucketTagging           = "s3_GetBucketTagging"
-	ActionGetBucketVersioning        = "s3_GetBucketVersioning"
-	ActionGetObject                  = "s3_GetObject"
-	ActionGetObjectAcl               = "s3_GetObjectAcl"
-	ActionGetObjectAttributes        = "s3_GetObjectAttributes"
-	ActionGetObjectLegalHold         = "s3_GetObjectLegalHold"
-	ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
-	ActionGetObjectRetention         = "s3_GetObjectRetention"
-	ActionGetObjectTagging           = "s3_GetObjectTagging"
-	ActionHeadBucket                 = "s3_HeadBucket"
-	ActionHeadObject                 = "s3_HeadObject"
-	ActionListAllMyBuckets           = "s3_ListAllMyBuckets"
-	ActionListMultipartUploads       = "s3_ListMultipartUploads"
-	ActionListObjectVersions         = "s3_ListObjectVersions"
-	ActionListObjects                = "s3_ListObjects"
-	ActionListObjectsV2              = "s3_ListObjectsV2"
-	ActionListParts                  = "s3_ListParts"
-	ActionPutBucketAcl               = "s3_PutBucketAcl"
-	ActionPutBucketPolicy            = "s3_PutBucketPolicy"
-	ActionPutBucketTagging           = "s3_PutBucketTagging"
-	ActionPutBucketVersioning        = "s3_PutBucketVersioning"
-	ActionPutObject                  = "s3_PutObject"
-	ActionPutObjectAcl               = "s3_PutObjectAcl"
-	ActionPutObjectLegalHold         = "s3_PutObjectLegalHold"
-	ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
-	ActionPutObjectRetention         = "s3_PutObjectRetention"
-	ActionPutObjectTagging           = "s3_PutObjectTagging"
-	ActionRestoreObject              = "s3_RestoreObject"
-	ActionSelectObjectContent        = "s3_SelectObjectContent"
-	ActionUploadPart                 = "s3_UploadPart"
-	ActionUploadPartCopy             = "s3_UploadPartCopy"
+	ActionUndetected                    = "ActionUnDetected"
+	ActionAbortMultipartUpload          = "s3_AbortMultipartUpload"
+	ActionCompleteMultipartUpload       = "s3_CompleteMultipartUpload"
+	ActionCopyObject                    = "s3_CopyObject"
+	ActionCreateBucket                  = "s3_CreateBucket"
+	ActionCreateMultipartUpload         = "s3_CreateMultipartUpload"
+	ActionDeleteBucket                  = "s3_DeleteBucket"
+	ActionDeleteBucketPolicy            = "s3_DeleteBucketPolicy"
+	ActionDeleteBucketTagging           = "s3_DeleteBucketTagging"
+	ActionDeleteObject                  = "s3_DeleteObject"
+	ActionDeleteObjectTagging           = "s3_DeleteObjectTagging"
+	ActionDeleteObjects                 = "s3_DeleteObjects"
+	ActionGetBucketAcl                  = "s3_GetBucketAcl"
+	ActionGetBucketPolicy               = "s3_GetBucketPolicy"
+	ActionGetBucketTagging              = "s3_GetBucketTagging"
+	ActionGetBucketVersioning           = "s3_GetBucketVersioning"
+	ActionGetObject                     = "s3_GetObject"
+	ActionGetObjectAcl                  = "s3_GetObjectAcl"
+	ActionGetObjectAttributes           = "s3_GetObjectAttributes"
+	ActionGetObjectLegalHold            = "s3_GetObjectLegalHold"
+	ActionGetObjectLockConfiguration    = "s3_GetObjectLockConfiguration"
+	ActionGetObjectRetention            = "s3_GetObjectRetention"
+	ActionGetObjectTagging              = "s3_GetObjectTagging"
+	ActionHeadBucket                    = "s3_HeadBucket"
+	ActionHeadObject                    = "s3_HeadObject"
+	ActionListAllMyBuckets              = "s3_ListAllMyBuckets"
+	ActionListMultipartUploads          = "s3_ListMultipartUploads"
+	ActionListObjectVersions            = "s3_ListObjectVersions"
+	ActionListObjects                   = "s3_ListObjects"
+	ActionListObjectsV2                 = "s3_ListObjectsV2"
+	ActionListParts                     = "s3_ListParts"
+	ActionPutBucketAcl                  = "s3_PutBucketAcl"
+	ActionPutBucketPolicy               = "s3_PutBucketPolicy"
+	ActionPutBucketTagging              = "s3_PutBucketTagging"
+	ActionPutBucketVersioning           = "s3_PutBucketVersioning"
+	ActionPutObject                     = "s3_PutObject"
+	ActionPutObjectAcl                  = "s3_PutObjectAcl"
+	ActionPutObjectLegalHold            = "s3_PutObjectLegalHold"
+	ActionPutObjectLockConfiguration    = "s3_PutObjectLockConfiguration"
+	ActionPutObjectRetention            = "s3_PutObjectRetention"
+	ActionPutObjectTagging              = "s3_PutObjectTagging"
+	ActionRestoreObject                 = "s3_RestoreObject"
+	ActionSelectObjectContent           = "s3_SelectObjectContent"
+	ActionUploadPart                    = "s3_UploadPart"
+	ActionUploadPartCopy                = "s3_UploadPartCopy"
+	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
+	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
+	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
 )
 
 func init() {

s3api/admin-router.go

@@ -19,12 +19,13 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminRouter struct{}
 
-func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
-	controller := controllers.NewAdminController(iam, be)
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
+	controller := controllers.NewAdminController(iam, be, logger)
 
 	// CreateUser admin api
 	app.Patch("/create-user", controller.CreateUser)
@@ -32,6 +33,9 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
 	// DeleteUsers admin api
 	app.Patch("/delete-user", controller.DeleteUser)
 
+	// UpdateUser admin api
+	app.Patch("/update-user", controller.UpdateUser)
+
 	// ListUsers admin api
 	app.Patch("/list-users", controller.ListUsers)
 

s3api/admin-server.go

@@ -22,6 +22,7 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminServer struct {
@@ -32,7 +33,7 @@ type S3AdminServer struct {
 	cert    *tls.Certificate
 }
 
-func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
 	server := &S3AdminServer{
 		app:     app,
 		backend: be,
@@ -46,13 +47,13 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 
 	// Logging middlewares
 	app.Use(logger.New())
-	app.Use(middlewares.DecodeURL(nil, nil))
+	app.Use(middlewares.DecodeURL(l, nil))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyV4Signature(root, iam, nil, nil, region, false))
-	app.Use(middlewares.VerifyMD5Body(nil))
+	app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(l))
 
-	server.router.Init(app, be, iam)
+	server.router.Init(app, be, iam, l)
 
 	return server
 }

s3api/controllers/admin.go

@@ -16,107 +16,310 @@ package controllers
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3log"
 )
 
 type AdminController struct {
 	iam auth.IAMService
 	be  backend.Backend
+	l   s3log.AuditLogger
 }
 
-func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
-	return AdminController{iam: iam, be: be}
+func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
+	return AdminController{iam: iam, be: be, l: l}
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:CreateUser",
+			})
 	}
 	var usr auth.Account
 	err := json.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return fmt.Errorf("failed to parse request body: %w", err)
+		return sendResponse(ctx, fmt.Errorf("failed to parse request body: %w", err), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusBadRequest,
+				action: "admin:CreateUser",
+			})
 	}
 
 	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
+		return sendResponse(ctx, errors.New("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusBadRequest,
+				action: "admin:CreateUser",
+			})
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		return fmt.Errorf("failed to create user: %w", err)
+		status := fiber.StatusInternalServerError
+		err = fmt.Errorf("failed to create user: %w", err)
+
+		if strings.Contains(err.Error(), "user already exists") {
+			status = fiber.StatusConflict
+		}
+
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				status: status,
+				logger: c.l,
+				action: "admin:CreateUser",
+			})
 	}
 
-	return ctx.SendString("The user has been created successfully")
+	return sendResponse(ctx, nil, "The user has been created successfully", &metaOptions{
+		status: fiber.StatusCreated,
+		logger: c.l,
+		action: "admin:CreateUser",
+	})
+}
+
+func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:UpdateUser",
+			})
+	}
+
+	access := ctx.Query("access")
+	if access == "" {
+		return sendResponse(ctx, errors.New("missing user access parameter"), nil,
+			&metaOptions{
+				status: fiber.StatusBadRequest,
+				logger: c.l,
+				action: "admin:UpdateUser",
+			})
+	}
+
+	var props auth.MutableProps
+	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
+		return sendResponse(ctx, fmt.Errorf("invalid request body %w", err), nil,
+			&metaOptions{
+				status: fiber.StatusBadRequest,
+				logger: c.l,
+				action: "admin:UpdateUser",
+			})
+	}
+
+	err := c.iam.UpdateUserAccount(access, props)
+	if err != nil {
+		status := fiber.StatusInternalServerError
+		err = fmt.Errorf("failed to update user account: %w", err)
+
+		if strings.Contains(err.Error(), "user not found") {
+			status = fiber.StatusNotFound
+		}
+
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				status: status,
+				logger: c.l,
+				action: "admin:UpdateUser",
+			})
+	}
+
+	return sendResponse(ctx, nil, "the user has been updated successfully",
+		&metaOptions{
+			logger: c.l,
+			action: "admin:UpdateUser",
+		})
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:DeleteUser",
+			})
 	}
 
 	err := c.iam.DeleteUserAccount(access)
 	if err != nil {
-		return err
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:DeleteUser",
+			})
 	}
 
-	return ctx.SendString("The user has been deleted successfully")
+	return sendResponse(ctx, nil, "The user has been deleted successfully",
+		&metaOptions{
+			logger: c.l,
+			action: "admin:DeleteUser",
+		})
 }
 
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:ListUsers",
+			})
 	}
 	accs, err := c.iam.ListUserAccounts()
-	if err != nil {
-		return err
-	}
-
-	return ctx.JSON(accs)
+	return sendResponse(ctx, err, accs,
+		&metaOptions{
+			logger: c.l,
+			action: "admin:ListUsers",
+		})
 }
 
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:ChangeBucketOwner",
+			})
 	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
 
 	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
 	if err != nil {
-		return err
+		return sendResponse(ctx, err, nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:ChangeBucketOwner",
+			})
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user specified as the new bucket owner does not exist")
+		return sendResponse(ctx, errors.New("user specified as the new bucket owner does not exist"), nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:ChangeBucketOwner",
+				status: fiber.StatusNotFound,
+			})
 	}
 
-	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
+	acl := auth.ACL{
+		Owner: owner,
+		Grantees: []auth.Grantee{
+			{
+				Permission: types.PermissionFullControl,
+				Access:     owner,
+				Type:       types.TypeCanonicalUser,
+			},
+		},
+	}
+
+	aclParsed, err := json.Marshal(acl)
 	if err != nil {
-		return err
+		return sendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err), nil,
+			&metaOptions{
+				logger: c.l,
+				action: "admin:ChangeBucketOwner",
+			})
 	}
 
-	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
+	return sendResponse(ctx, err, "Bucket owner has been updated successfully",
+		&metaOptions{
+			logger: c.l,
+			action: "admin:ChangeBucketOwner",
+		})
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
+			&metaOptions{
+				logger: c.l,
+				status: fiber.StatusForbidden,
+				action: "admin:ListBuckets",
+			})
 	}
 
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
+	return sendResponse(ctx, err, buckets,
+		&metaOptions{
+			logger: c.l,
+			action: "admin:ListBuckets",
+		})
+}
+
+type metaOptions struct {
+	action string
+	status int
+	logger s3log.AuditLogger
+}
+
+func sendResponse(ctx *fiber.Ctx, err error, data any, m *metaOptions) error {
+	status := m.status
+	if err != nil {
+		if status == 0 {
+			status = fiber.StatusInternalServerError
+		}
+		if m.logger != nil {
+			m.logger.Log(ctx, err, []byte(err.Error()), s3log.LogMeta{
+				Action:     m.action,
+				HttpStatus: status,
+			})
+		}
+
+		return ctx.Status(status).SendString(err.Error())
+	}
+
+	if status == 0 {
+		status = fiber.StatusOK
+	}
+
+	msg, ok := data.(string)
+	if ok {
+		if m.logger != nil {
+			m.logger.Log(ctx, nil, []byte(msg), s3log.LogMeta{
+				Action:     m.action,
+				HttpStatus: status,
+			})
+		}
+
+		return ctx.Status(status).SendString(msg)
+	}
+
+	dataJSON, err := json.Marshal(data)
 	if err != nil {
 		return err
 	}
 
-	return ctx.JSON(buckets)
+	if m.logger != nil {
+		m.logger.Log(ctx, nil, dataJSON, s3log.LogMeta{
+			HttpStatus: status,
+			Action:     m.action,
+		})
+	}
+
+	ctx.Set(fiber.HeaderContentType, fiber.MIMEApplicationJSON)
+
+	return ctx.Status(status).Send(dataJSON)
 }

s3api/controllers/admin_test.go

@@ -85,7 +85,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 201,
 		},
 		{
 			name: "Admin-create-user-invalid-user-role",
@@ -94,7 +94,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
@@ -103,7 +103,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -119,6 +119,122 @@ func TestAdminController_CreateUser(t *testing.T) {
 	}
 }
 
+func TestAdminController_UpdateUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/update-user", adminController.UpdateUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/update-user", adminController.UpdateUser)
+
+	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	appNotFound := fiber.New()
+
+	appNotFound.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-update-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-update-user-missing-access",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-request-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
+		{
+			name: "Admin-update-user-not-found",
+			app:  appNotFound,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 404,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
 func TestAdminController_DeleteUser(t *testing.T) {
 	type args struct {
 		req *http.Request
@@ -173,7 +289,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -251,7 +367,7 @@ func TestAdminController_ListUsers(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Admin-list-users-iam-error",
@@ -291,7 +407,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 	adminController := AdminController{
 		be: &BackendMock{
-			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
+			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
 				return nil
 			},
 		},
@@ -368,7 +484,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -386,7 +502,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 404,
 		},
 		{
 			name: "Change-bucket-owner-success",
@@ -395,7 +511,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 201,
+			statusCode: 200,
 		},
 	}
 	for _, tt := range tests {
@@ -455,7 +571,7 @@ func TestAdminController_ListBuckets(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "List-buckets-success",

s3api/controllers/backend_moq_test.go

@@ -7,9 +7,9 @@ import (
 	"bufio"
 	"context"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3response"
-	"io"
 	"sync"
 )
 
@@ -26,7 +26,7 @@ var _ backend.Backend = &BackendMock{}
 //			AbortMultipartUploadFunc: func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error {
 //				panic("mock out the AbortMultipartUpload method")
 //			},
-//			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, newOwner string) error {
+//			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
 //				panic("mock out the ChangeBucketOwner method")
 //			},
 //			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
@@ -38,12 +38,15 @@ var _ backend.Backend = &BackendMock{}
 //			CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
 //				panic("mock out the CreateBucket method")
 //			},
-//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 //				panic("mock out the CreateMultipartUpload method")
 //			},
 //			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketOwnershipControls method")
+//			},
 //			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketPolicy method")
 //			},
@@ -62,6 +65,9 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+//				panic("mock out the GetBucketOwnershipControls method")
+//			},
 //			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 //				panic("mock out the GetBucketPolicy method")
 //			},
@@ -71,7 +77,7 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
 //				panic("mock out the GetBucketVersioning method")
 //			},
-//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 //				panic("mock out the GetObject method")
 //			},
 //			GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
@@ -110,10 +116,10 @@ var _ backend.Backend = &BackendMock{}
 //			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
 //				panic("mock out the ListObjectVersions method")
 //			},
-//			ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+//			ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 //				panic("mock out the ListObjects method")
 //			},
-//			ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+//			ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 //				panic("mock out the ListObjectsV2 method")
 //			},
 //			ListPartsFunc: func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error) {
@@ -122,6 +128,9 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+//				panic("mock out the PutBucketOwnershipControls method")
+//			},
 //			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
 //				panic("mock out the PutBucketPolicy method")
 //			},
@@ -178,7 +187,7 @@ type BackendMock struct {
 	AbortMultipartUploadFunc func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error
 
 	// ChangeBucketOwnerFunc mocks the ChangeBucketOwner method.
-	ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, newOwner string) error
+	ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, acl []byte) error
 
 	// CompleteMultipartUploadFunc mocks the CompleteMultipartUpload method.
 	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
@@ -190,11 +199,14 @@ type BackendMock struct {
 	CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error
 
 	// CreateMultipartUploadFunc mocks the CreateMultipartUpload method.
-	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
+	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
+	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
 	DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -213,6 +225,9 @@ type BackendMock struct {
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketOwnershipControlsFunc mocks the GetBucketOwnershipControls method.
+	GetBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error)
+
 	// GetBucketPolicyFunc mocks the GetBucketPolicy method.
 	GetBucketPolicyFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
 
@@ -223,7 +238,7 @@ type BackendMock struct {
 	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 
 	// GetObjectFunc mocks the GetObject method.
-	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 
 	// GetObjectAclFunc mocks the GetObjectAcl method.
 	GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
@@ -262,10 +277,10 @@ type BackendMock struct {
 	ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
 
 	// ListObjectsFunc mocks the ListObjects method.
-	ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 
 	// ListObjectsV2Func mocks the ListObjectsV2 method.
-	ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
 
 	// ListPartsFunc mocks the ListParts method.
 	ListPartsFunc func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error)
@@ -273,6 +288,9 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketOwnershipControlsFunc mocks the PutBucketOwnershipControls method.
+	PutBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error
+
 	// PutBucketPolicyFunc mocks the PutBucketPolicy method.
 	PutBucketPolicyFunc func(contextMoqParam context.Context, bucket string, policy []byte) error
 
@@ -333,8 +351,8 @@ type BackendMock struct {
 			ContextMoqParam context.Context
 			// Bucket is the bucket argument value.
 			Bucket string
-			// NewOwner is the newOwner argument value.
-			NewOwner string
+			// ACL is the acl argument value.
+			ACL []byte
 		}
 		// CompleteMultipartUpload holds details about calls to the CompleteMultipartUpload method.
 		CompleteMultipartUpload []struct {
@@ -373,6 +391,13 @@ type BackendMock struct {
 			// DeleteBucketInput is the deleteBucketInput argument value.
 			DeleteBucketInput *s3.DeleteBucketInput
 		}
+		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
+		DeleteBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
 		DeleteBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -417,6 +442,13 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketOwnershipControls holds details about calls to the GetBucketOwnershipControls method.
+		GetBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetBucketPolicy holds details about calls to the GetBucketPolicy method.
 		GetBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -444,8 +476,6 @@ type BackendMock struct {
 			ContextMoqParam context.Context
 			// GetObjectInput is the getObjectInput argument value.
 			GetObjectInput *s3.GetObjectInput
-			// Writer is the writer argument value.
-			Writer io.Writer
 		}
 		// GetObjectAcl holds details about calls to the GetObjectAcl method.
 		GetObjectAcl []struct {
@@ -571,6 +601,15 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketOwnershipControls holds details about calls to the PutBucketOwnershipControls method.
+		PutBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Ownership is the ownership argument value.
+			Ownership types.ObjectOwnership
+		}
 		// PutBucketPolicy holds details about calls to the PutBucketPolicy method.
 		PutBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -693,54 +732,57 @@ type BackendMock struct {
 			UploadPartCopyInput *s3.UploadPartCopyInput
 		}
 	}
-	lockAbortMultipartUpload       sync.RWMutex
-	lockChangeBucketOwner          sync.RWMutex
-	lockCompleteMultipartUpload    sync.RWMutex
-	lockCopyObject                 sync.RWMutex
-	lockCreateBucket               sync.RWMutex
-	lockCreateMultipartUpload      sync.RWMutex
-	lockDeleteBucket               sync.RWMutex
-	lockDeleteBucketPolicy         sync.RWMutex
-	lockDeleteBucketTagging        sync.RWMutex
-	lockDeleteObject               sync.RWMutex
-	lockDeleteObjectTagging        sync.RWMutex
-	lockDeleteObjects              sync.RWMutex
-	lockGetBucketAcl               sync.RWMutex
-	lockGetBucketPolicy            sync.RWMutex
-	lockGetBucketTagging           sync.RWMutex
-	lockGetBucketVersioning        sync.RWMutex
-	lockGetObject                  sync.RWMutex
-	lockGetObjectAcl               sync.RWMutex
-	lockGetObjectAttributes        sync.RWMutex
-	lockGetObjectLegalHold         sync.RWMutex
-	lockGetObjectLockConfiguration sync.RWMutex
-	lockGetObjectRetention         sync.RWMutex
-	lockGetObjectTagging           sync.RWMutex
-	lockHeadBucket                 sync.RWMutex
-	lockHeadObject                 sync.RWMutex
-	lockListBuckets                sync.RWMutex
-	lockListBucketsAndOwners       sync.RWMutex
-	lockListMultipartUploads       sync.RWMutex
-	lockListObjectVersions         sync.RWMutex
-	lockListObjects                sync.RWMutex
-	lockListObjectsV2              sync.RWMutex
-	lockListParts                  sync.RWMutex
-	lockPutBucketAcl               sync.RWMutex
-	lockPutBucketPolicy            sync.RWMutex
-	lockPutBucketTagging           sync.RWMutex
-	lockPutBucketVersioning        sync.RWMutex
-	lockPutObject                  sync.RWMutex
-	lockPutObjectAcl               sync.RWMutex
-	lockPutObjectLegalHold         sync.RWMutex
-	lockPutObjectLockConfiguration sync.RWMutex
-	lockPutObjectRetention         sync.RWMutex
-	lockPutObjectTagging           sync.RWMutex
-	lockRestoreObject              sync.RWMutex
-	lockSelectObjectContent        sync.RWMutex
-	lockShutdown                   sync.RWMutex
-	lockString                     sync.RWMutex
-	lockUploadPart                 sync.RWMutex
-	lockUploadPartCopy             sync.RWMutex
+	lockAbortMultipartUpload          sync.RWMutex
+	lockChangeBucketOwner             sync.RWMutex
+	lockCompleteMultipartUpload       sync.RWMutex
+	lockCopyObject                    sync.RWMutex
+	lockCreateBucket                  sync.RWMutex
+	lockCreateMultipartUpload         sync.RWMutex
+	lockDeleteBucket                  sync.RWMutex
+	lockDeleteBucketOwnershipControls sync.RWMutex
+	lockDeleteBucketPolicy            sync.RWMutex
+	lockDeleteBucketTagging           sync.RWMutex
+	lockDeleteObject                  sync.RWMutex
+	lockDeleteObjectTagging           sync.RWMutex
+	lockDeleteObjects                 sync.RWMutex
+	lockGetBucketAcl                  sync.RWMutex
+	lockGetBucketOwnershipControls    sync.RWMutex
+	lockGetBucketPolicy               sync.RWMutex
+	lockGetBucketTagging              sync.RWMutex
+	lockGetBucketVersioning           sync.RWMutex
+	lockGetObject                     sync.RWMutex
+	lockGetObjectAcl                  sync.RWMutex
+	lockGetObjectAttributes           sync.RWMutex
+	lockGetObjectLegalHold            sync.RWMutex
+	lockGetObjectLockConfiguration    sync.RWMutex
+	lockGetObjectRetention            sync.RWMutex
+	lockGetObjectTagging              sync.RWMutex
+	lockHeadBucket                    sync.RWMutex
+	lockHeadObject                    sync.RWMutex
+	lockListBuckets                   sync.RWMutex
+	lockListBucketsAndOwners          sync.RWMutex
+	lockListMultipartUploads          sync.RWMutex
+	lockListObjectVersions            sync.RWMutex
+	lockListObjects                   sync.RWMutex
+	lockListObjectsV2                 sync.RWMutex
+	lockListParts                     sync.RWMutex
+	lockPutBucketAcl                  sync.RWMutex
+	lockPutBucketOwnershipControls    sync.RWMutex
+	lockPutBucketPolicy               sync.RWMutex
+	lockPutBucketTagging              sync.RWMutex
+	lockPutBucketVersioning           sync.RWMutex
+	lockPutObject                     sync.RWMutex
+	lockPutObjectAcl                  sync.RWMutex
+	lockPutObjectLegalHold            sync.RWMutex
+	lockPutObjectLockConfiguration    sync.RWMutex
+	lockPutObjectRetention            sync.RWMutex
+	lockPutObjectTagging              sync.RWMutex
+	lockRestoreObject                 sync.RWMutex
+	lockSelectObjectContent           sync.RWMutex
+	lockShutdown                      sync.RWMutex
+	lockString                        sync.RWMutex
+	lockUploadPart                    sync.RWMutex
+	lockUploadPartCopy                sync.RWMutex
 }
 
 // AbortMultipartUpload calls AbortMultipartUploadFunc.
@@ -780,23 +822,23 @@ func (mock *BackendMock) AbortMultipartUploadCalls() []struct {
 }
 
 // ChangeBucketOwner calls ChangeBucketOwnerFunc.
-func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, bucket string, newOwner string) error {
+func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, bucket string, acl []byte) error {
 	if mock.ChangeBucketOwnerFunc == nil {
 		panic("BackendMock.ChangeBucketOwnerFunc: method is nil but Backend.ChangeBucketOwner was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		Bucket          string
-		NewOwner        string
+		ACL             []byte
 	}{
 		ContextMoqParam: contextMoqParam,
 		Bucket:          bucket,
-		NewOwner:        newOwner,
+		ACL:             acl,
 	}
 	mock.lockChangeBucketOwner.Lock()
 	mock.calls.ChangeBucketOwner = append(mock.calls.ChangeBucketOwner, callInfo)
 	mock.lockChangeBucketOwner.Unlock()
-	return mock.ChangeBucketOwnerFunc(contextMoqParam, bucket, newOwner)
+	return mock.ChangeBucketOwnerFunc(contextMoqParam, bucket, acl)
 }
 
 // ChangeBucketOwnerCalls gets all the calls that were made to ChangeBucketOwner.
@@ -806,12 +848,12 @@ func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, buck
 func (mock *BackendMock) ChangeBucketOwnerCalls() []struct {
 	ContextMoqParam context.Context
 	Bucket          string
-	NewOwner        string
+	ACL             []byte
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		Bucket          string
-		NewOwner        string
+		ACL             []byte
 	}
 	mock.lockChangeBucketOwner.RLock()
 	calls = mock.calls.ChangeBucketOwner
@@ -932,7 +974,7 @@ func (mock *BackendMock) CreateBucketCalls() []struct {
 }
 
 // CreateMultipartUpload calls CreateMultipartUploadFunc.
-func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	if mock.CreateMultipartUploadFunc == nil {
 		panic("BackendMock.CreateMultipartUploadFunc: method is nil but Backend.CreateMultipartUpload was just called")
 	}
@@ -1003,6 +1045,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketOwnershipControls calls DeleteBucketOwnershipControlsFunc.
+func (mock *BackendMock) DeleteBucketOwnershipControls(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.DeleteBucketOwnershipControlsFunc: method is nil but Backend.DeleteBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketOwnershipControls.Lock()
+	mock.calls.DeleteBucketOwnershipControls = append(mock.calls.DeleteBucketOwnershipControls, callInfo)
+	mock.lockDeleteBucketOwnershipControls.Unlock()
+	return mock.DeleteBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketOwnershipControlsCalls gets all the calls that were made to DeleteBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketOwnershipControlsCalls())
+func (mock *BackendMock) DeleteBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketOwnershipControls.RLock()
+	calls = mock.calls.DeleteBucketOwnershipControls
+	mock.lockDeleteBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // DeleteBucketPolicy calls DeleteBucketPolicyFunc.
 func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketPolicyFunc == nil {
@@ -1223,6 +1301,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketOwnershipControls calls GetBucketOwnershipControlsFunc.
+func (mock *BackendMock) GetBucketOwnershipControls(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+	if mock.GetBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.GetBucketOwnershipControlsFunc: method is nil but Backend.GetBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketOwnershipControls.Lock()
+	mock.calls.GetBucketOwnershipControls = append(mock.calls.GetBucketOwnershipControls, callInfo)
+	mock.lockGetBucketOwnershipControls.Unlock()
+	return mock.GetBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// GetBucketOwnershipControlsCalls gets all the calls that were made to GetBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketOwnershipControlsCalls())
+func (mock *BackendMock) GetBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketOwnershipControls.RLock()
+	calls = mock.calls.GetBucketOwnershipControls
+	mock.lockGetBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // GetBucketPolicy calls GetBucketPolicyFunc.
 func (mock *BackendMock) GetBucketPolicy(contextMoqParam context.Context, bucket string) ([]byte, error) {
 	if mock.GetBucketPolicyFunc == nil {
@@ -1332,23 +1446,21 @@ func (mock *BackendMock) GetBucketVersioningCalls() []struct {
 }
 
 // GetObject calls GetObjectFunc.
-func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	if mock.GetObjectFunc == nil {
 		panic("BackendMock.GetObjectFunc: method is nil but Backend.GetObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}{
 		ContextMoqParam: contextMoqParam,
 		GetObjectInput:  getObjectInput,
-		Writer:          writer,
 	}
 	mock.lockGetObject.Lock()
 	mock.calls.GetObject = append(mock.calls.GetObject, callInfo)
 	mock.lockGetObject.Unlock()
-	return mock.GetObjectFunc(contextMoqParam, getObjectInput, writer)
+	return mock.GetObjectFunc(contextMoqParam, getObjectInput)
 }
 
 // GetObjectCalls gets all the calls that were made to GetObject.
@@ -1358,12 +1470,10 @@ func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInp
 func (mock *BackendMock) GetObjectCalls() []struct {
 	ContextMoqParam context.Context
 	GetObjectInput  *s3.GetObjectInput
-	Writer          io.Writer
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}
 	mock.lockGetObject.RLock()
 	calls = mock.calls.GetObject
@@ -1824,7 +1934,7 @@ func (mock *BackendMock) ListObjectVersionsCalls() []struct {
 }
 
 // ListObjects calls ListObjectsFunc.
-func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	if mock.ListObjectsFunc == nil {
 		panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called")
 	}
@@ -1860,7 +1970,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct {
 }
 
 // ListObjectsV2 calls ListObjectsV2Func.
-func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 	if mock.ListObjectsV2Func == nil {
 		panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called")
 	}
@@ -1971,6 +2081,46 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketOwnershipControls calls PutBucketOwnershipControlsFunc.
+func (mock *BackendMock) PutBucketOwnershipControls(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+	if mock.PutBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.PutBucketOwnershipControlsFunc: method is nil but Backend.PutBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Ownership:       ownership,
+	}
+	mock.lockPutBucketOwnershipControls.Lock()
+	mock.calls.PutBucketOwnershipControls = append(mock.calls.PutBucketOwnershipControls, callInfo)
+	mock.lockPutBucketOwnershipControls.Unlock()
+	return mock.PutBucketOwnershipControlsFunc(contextMoqParam, bucket, ownership)
+}
+
+// PutBucketOwnershipControlsCalls gets all the calls that were made to PutBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketOwnershipControlsCalls())
+func (mock *BackendMock) PutBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Ownership       types.ObjectOwnership
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}
+	mock.lockPutBucketOwnershipControls.RLock()
+	calls = mock.calls.PutBucketOwnershipControls
+	mock.lockPutBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // PutBucketPolicy calls PutBucketPolicyFunc.
 func (mock *BackendMock) PutBucketPolicy(contextMoqParam context.Context, bucket string, policy []byte) error {
 	if mock.PutBucketPolicyFunc == nil {

s3api/controllers/base.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -50,7 +51,8 @@ type S3ApiController struct {
 }
 
 const (
-	iso8601Format = "20060102T150405Z"
+	iso8601Format      = "20060102T150405Z"
+	defaultContentType = "binary/octet-stream"
 )
 
 func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) S3ApiController {
@@ -91,6 +93,10 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
+	path := ctx.Path()
+	if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" {
+		key = key + "/"
+	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
@@ -402,7 +408,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		Key:       &key,
 		Range:     &acceptRange,
 		VersionId: &versionId,
-	}, ctx.Response().BodyWriter())
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -412,71 +418,76 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 				BucketOwner: parsedAcl.Owner,
 			})
 	}
-	if res == nil {
-		return SendResponse(ctx, fmt.Errorf("get object nil response"),
-			&MetaOpts{
-				Logger:      c.logger,
-				MetricsMng:  c.mm,
-				Action:      metrics.ActionGetObject,
-				BucketOwner: parsedAcl.Owner,
-			})
+
+	contentType := getstring(res.ContentType)
+	if contentType == "" {
+		contentType = defaultContentType
+	}
+	acceptRanges := getstring(res.AcceptRanges)
+	if acceptRanges == "" {
+		acceptRanges = "bytes"
 	}
 
-	utils.SetMetaHeaders(ctx, res.Metadata)
-	var lastmod string
-	if res.LastModified != nil {
-		lastmod = res.LastModified.Format(timefmt)
-	}
-
-	utils.SetResponseHeaders(ctx, []utils.CustomHeader{
-		{
-			Key:   "Content-Length",
-			Value: fmt.Sprint(getint64(res.ContentLength)),
-		},
+	hdrs := []utils.CustomHeader{
 		{
 			Key:   "Content-Type",
-			Value: getstring(res.ContentType),
-		},
-		{
-			Key:   "Content-Encoding",
-			Value: getstring(res.ContentEncoding),
+			Value: contentType,
 		},
 		{
 			Key:   "ETag",
 			Value: getstring(res.ETag),
 		},
 		{
-			Key:   "Last-Modified",
-			Value: lastmod,
+			Key:   "accept-ranges",
+			Value: acceptRanges,
 		},
-		{
-			Key:   "x-amz-storage-class",
-			Value: string(res.StorageClass),
-		},
-		{
+	}
+
+	if getstring(res.ContentRange) != "" {
+		hdrs = append(hdrs, utils.CustomHeader{
 			Key:   "Content-Range",
 			Value: getstring(res.ContentRange),
-		},
-		{
-			Key:   "accept-ranges",
-			Value: getstring(res.AcceptRanges),
-		},
-	})
-
-	if res.TagCount != nil {
-		utils.SetResponseHeaders(ctx, []utils.CustomHeader{
-			{
-				Key:   "x-amz-tagging-count",
-				Value: fmt.Sprint(*res.TagCount),
-			},
 		})
 	}
+	if res.LastModified != nil {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "Last-Modified",
+			Value: res.LastModified.Format(timefmt),
+		})
+	}
+	if getstring(res.ContentEncoding) != "" {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "Content-Encoding",
+			Value: getstring(res.ContentEncoding),
+		})
+	}
+	if res.TagCount != nil {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-tagging-count",
+			Value: fmt.Sprint(*res.TagCount),
+		})
+	}
+	if res.StorageClass != "" {
+		hdrs = append(hdrs, utils.CustomHeader{
+			Key:   "x-amz-storage-class",
+			Value: string(res.StorageClass),
+		})
+	}
+
+	// Set x-amz-meta-... headers
+	utils.SetMetaHeaders(ctx, res.Metadata)
+	// Set other response headers
+	utils.SetResponseHeaders(ctx, hdrs)
 
 	status := http.StatusOK
 	if acceptRange != "" {
 		status = http.StatusPartialContent
 	}
 
+	if res.Body != nil {
+		ctx.Response().SetBodyStream(res.Body, int(getint64(res.ContentLength)))
+	}
+
 	return SendResponse(ctx, nil,
 		&MetaOpts{
 			Logger:        c.logger,
@@ -566,6 +577,43 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Readonly:      c.readonly,
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketOwnershipControlsAction,
+		})
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionGetBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		data, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket)
+		return SendXMLResponse(ctx,
+			s3response.OwnershipControls{
+				Rules: []types.OwnershipControlsRule{
+					{
+						ObjectOwnership: data,
+					},
+				},
+			}, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionGetBucketOwnershipControls,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
 			Readonly:      c.readonly,
@@ -913,10 +961,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			Delimiter: &delimiter,
 			MaxKeys:   &maxkeys,
 		})
-	return SendXMLResponse(ctx, struct {
-		*s3.ListObjectsOutput
-		XMLName struct{} `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
-	}{ListObjectsOutput: res}, err,
+	return SendXMLResponse(ctx, res, err,
 		&MetaOpts{
 			Logger:      c.logger,
 			MetricsMng:  c.mm,
@@ -933,6 +978,9 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp")
 	granWrite := ctx.Get("X-Amz-Grant-Write")
 	grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp")
+	objectOwnership := types.ObjectOwnership(
+		ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)),
+	)
 	mfa := ctx.Get("X-Amz-Mfa")
 	contentMD5 := ctx.Get("Content-MD5")
 	acct := ctx.Locals("account").(auth.Account)
@@ -1000,6 +1048,57 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
+		var ownershipControls s3response.OwnershipControls
+		if err := xml.Unmarshal(ctx.Body(), &ownershipControls); err != nil {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		if len(ownershipControls.Rules) != 1 || !utils.IsValidOwnership(ownershipControls.Rules[0].ObjectOwnership) {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Readonly:      c.readonly,
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketOwnershipControlsAction,
+		}); err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err := c.be.PutBucketOwnershipControls(ctx.Context(), bucket, ownershipControls.Rules[0].ObjectOwnership)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionPutBucketOwnershipControls,
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
@@ -1141,10 +1240,33 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		var input *s3.PutBucketAclInput
-
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-		err := auth.VerifyAccess(ctx.Context(), c.be,
+		var input *auth.PutBucketAclInput
+
+		ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket)
+		if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketAcl,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+		if ownership == types.ObjectOwnershipBucketOwnerEnforced {
+			if c.debug {
+				log.Println("bucket acls are disabled")
+			}
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAclNotSupported),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketAcl,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = auth.VerifyAccess(ctx.Context(), c.be,
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
@@ -1171,7 +1293,37 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				if c.debug {
 					log.Printf("error unmarshalling access control policy: %v", err)
 				}
-				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedACL),
+					&MetaOpts{
+						Logger:      c.logger,
+						Action:      metrics.ActionPutBucketAcl,
+						BucketOwner: parsedAcl.Owner,
+					})
+			}
+
+			if accessControlPolicy.Owner == nil ||
+				accessControlPolicy.Owner.ID == nil ||
+				*accessControlPolicy.Owner.ID == "" {
+				if c.debug {
+					log.Println("empty access control policy owner")
+				}
+				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedACL),
+					&MetaOpts{
+						Logger:      c.logger,
+						Action:      metrics.ActionPutBucketAcl,
+						BucketOwner: parsedAcl.Owner,
+					})
+			}
+
+			if *accessControlPolicy.Owner.ID != parsedAcl.Owner {
+				if c.debug {
+					log.Printf("invalid access control policy owner id: %v, expected %v", *accessControlPolicy.Owner.ID, parsedAcl.Owner)
+				}
+				return SendResponse(ctx, s3err.APIError{
+					Code:           "InvalidArgument",
+					Description:    "Invalid id",
+					HTTPStatusCode: http.StatusBadRequest,
+				},
 					&MetaOpts{
 						Logger:      c.logger,
 						Action:      metrics.ActionPutBucketAcl,
@@ -1185,7 +1337,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 						grants, acl)
 				}
 				return SendResponse(ctx,
-					s3err.GetAPIError(s3err.ErrInvalidRequest),
+					s3err.GetAPIError(s3err.ErrUnexpectedContent),
 					&MetaOpts{
 						Logger:      c.logger,
 						MetricsMng:  c.mm,
@@ -1194,16 +1346,11 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 					})
 			}
 
-			input = &s3.PutBucketAclInput{
-				Bucket: &bucket,
-				ACL:    "",
-				AccessControlPolicy: &types.AccessControlPolicy{
-					Owner:  &accessControlPolicy.Owner,
-					Grants: accessControlPolicy.AccessControlList.Grants,
-				},
+			input = &auth.PutBucketAclInput{
+				Bucket:              &bucket,
+				AccessControlPolicy: &accessControlPolicy,
 			}
-		}
-		if acl != "" {
+		} else if acl != "" {
 			if acl != "private" && acl != "public-read" && acl != "public-read-write" {
 				if c.debug {
 					log.Printf("invalid acl: %q", acl)
@@ -1217,13 +1364,13 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 						BucketOwner: parsedAcl.Owner,
 					})
 			}
-			if len(ctx.Body()) > 0 || grants != "" {
+			if grants != "" {
 				if c.debug {
 					log.Printf("invalid request: %q (grants) %q (acl)",
 						grants, acl)
 				}
 				return SendResponse(ctx,
-					s3err.GetAPIError(s3err.ErrInvalidRequest),
+					s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants),
 					&MetaOpts{
 						Logger:      c.logger,
 						MetricsMng:  c.mm,
@@ -1232,35 +1379,34 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 					})
 			}
 
-			input = &s3.PutBucketAclInput{
+			input = &auth.PutBucketAclInput{
 				Bucket: &bucket,
 				ACL:    types.BucketCannedACL(acl),
-				AccessControlPolicy: &types.AccessControlPolicy{
-					Owner: &types.Owner{
-						ID: &acct.Access,
-					},
-				},
 			}
-		}
-		if grants != "" {
-			input = &s3.PutBucketAclInput{
+		} else if grants != "" {
+			input = &auth.PutBucketAclInput{
 				Bucket:           &bucket,
 				GrantFullControl: &grantFullControl,
 				GrantRead:        &grantRead,
 				GrantReadACP:     &grantReadACP,
 				GrantWrite:       &granWrite,
 				GrantWriteACP:    &grantWriteACP,
-				AccessControlPolicy: &types.AccessControlPolicy{
-					Owner: &types.Owner{
-						ID: &acct.Access,
-					},
-				},
-				ACL: "",
 			}
+		} else {
+			if c.debug {
+				log.Println("none of the bucket acl options has been specified: canned, req headers, req body")
+			}
+			return SendResponse(ctx,
+				s3err.GetAPIError(s3err.ErrMissingSecurityHeader),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionPutBucketAcl,
+					BucketOwner: parsedAcl.Owner,
+				})
 		}
 
-		fmt.Println(*input, parsedAcl)
-		updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam)
+		updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam, acct.Role == auth.RoleAdmin)
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1289,16 +1435,45 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				Action:     metrics.ActionCreateBucket,
 			})
 	}
+	if ok := utils.IsValidOwnership(objectOwnership); !ok {
+		if c.debug {
+			log.Printf("invalid bucket object ownership: %v", objectOwnership)
+		}
+		return SendResponse(ctx, s3err.APIError{
+			Code:           "InvalidArgument",
+			Description:    fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership),
+			HTTPStatusCode: http.StatusBadRequest,
+		},
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionCreateBucket,
+				BucketOwner: acct.Access,
+			})
+	}
+
+	if acl+grants != "" && objectOwnership == types.ObjectOwnershipBucketOwnerEnforced {
+		if c.debug {
+			log.Printf("bucket acls are disabled for %v object ownership", objectOwnership)
+		}
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership),
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionCreateBucket,
+				BucketOwner: acct.Access,
+			})
+	}
 
 	if acl != "" && grants != "" {
 		if c.debug {
 			log.Printf("invalid request: %q (grants) %q (acl)", grants, acl)
 		}
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest),
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants),
 			&MetaOpts{
 				Logger:      c.logger,
 				MetricsMng:  c.mm,
-				Action:      metrics.ActionPutBucketAcl,
+				Action:      metrics.ActionCreateBucket,
 				BucketOwner: acct.Access,
 			})
 	}
@@ -1307,17 +1482,18 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		Owner: acct.Access,
 	}
 
-	updAcl, err := auth.UpdateACL(&s3.PutBucketAclInput{
+	updAcl, err := auth.UpdateACL(&auth.PutBucketAclInput{
 		GrantFullControl: &grantFullControl,
 		GrantRead:        &grantRead,
 		GrantReadACP:     &grantReadACP,
 		GrantWrite:       &granWrite,
 		GrantWriteACP:    &grantWriteACP,
-		AccessControlPolicy: &types.AccessControlPolicy{Owner: &types.Owner{
-			ID: &acct.Access,
-		}},
+		AccessControlPolicy: &auth.AccessControlPolicy{
+			Owner: &types.Owner{
+				ID: &acct.Access,
+			}},
 		ACL: types.BucketCannedACL(acl),
-	}, defACL, c.iam)
+	}, defACL, c.iam, acct.Role == auth.RoleAdmin)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1334,7 +1510,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 
 	err = c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{
 		Bucket:                     &bucket,
-		ObjectOwnership:            types.ObjectOwnership(acct.Access),
+		ObjectOwnership:            objectOwnership,
 		ObjectLockEnabledForBucket: &lockEnabled,
 	}, updAcl)
 	return SendResponse(ctx, err,
@@ -1354,16 +1530,22 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	versionId := ctx.Query("versionId")
 	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
+	contentType := ctx.Get("Content-Type")
+	contentEncoding := ctx.Get("Content-Encoding")
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	tagging := ctx.Get("x-amz-tagging")
 
 	// Copy source headers
 	copySource := ctx.Get("X-Amz-Copy-Source")
+	if len(copySource) > 0 && copySource[0] == '/' {
+		copySource = copySource[1:]
+	}
 	copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match")
 	copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match")
 	copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since")
 	copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since")
 	copySrcRange := ctx.Get("X-Amz-Copy-Source-Range")
+	directive := ctx.Get("X-Amz-Metadata-Directive")
 
 	// Permission headers
 	acl := ctx.Get("X-Amz-Acl")
@@ -1373,11 +1555,19 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	granWrite := ctx.Get("X-Amz-Grant-Write")
 	grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp")
 
-	// Other headers
+	// Content Length
 	contentLengthStr := ctx.Get("Content-Length")
 	if contentLengthStr == "" {
 		contentLengthStr = "0"
 	}
+	// Use decoded content length if available because the
+	// middleware will decode the chunked transfer encoding
+	decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
+	if decodedLength != "" {
+		contentLengthStr = decodedLength
+	}
+
+	// Other headers
 	bucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner")
 	storageClass := ctx.Get("X-Amz-Storage-Class")
 
@@ -1479,7 +1669,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		}
 
 		bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention")
-		bypass := bypassHdr == "true"
+		bypass := strings.EqualFold(bypassHdr, "true")
 		if bypass {
 			policy, err := c.be.GetBucketPolicy(ctx.Context(), bucket)
 			if err != nil {
@@ -1563,6 +1753,24 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	if ctx.Request().URI().QueryArgs().Has("uploadId") &&
 		ctx.Request().URI().QueryArgs().Has("partNumber") &&
 		copySource != "" {
+
+		cs := copySource
+		copySource, err := url.QueryUnescape(copySource)
+		if err != nil {
+			if c.debug {
+				log.Printf("error unescaping copy source %q: %v",
+					cs, err)
+			}
+			return SendXMLResponse(ctx, nil,
+				s3err.GetAPIError(s3err.ErrInvalidCopySource),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionUploadPartCopy,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
 		partNumber := int32(ctx.QueryInt("partNumber", -1))
 		if partNumber < 1 || partNumber > 10000 {
 			if c.debug {
@@ -1578,7 +1786,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err := auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource,
+		err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource,
 			auth.AccessOptions{
 				Acl:           parsedAcl,
 				AclPermission: types.PermissionWrite,
@@ -1734,13 +1942,26 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 					})
 			}
 
+			//TODO: This part will be changed when object acls are implemented
+
+			grants := []types.Grant{}
+			for _, grt := range accessControlPolicy.AccessControlList.Grants {
+				grants = append(grants, types.Grant{
+					Grantee: &types.Grantee{
+						ID:   &grt.Grantee.ID,
+						Type: grt.Grantee.Type,
+					},
+					Permission: grt.Permission,
+				})
+			}
+
 			input = &s3.PutObjectAclInput{
 				Bucket: &bucket,
 				Key:    &keyStart,
 				ACL:    "",
 				AccessControlPolicy: &types.AccessControlPolicy{
-					Owner:  &accessControlPolicy.Owner,
-					Grants: accessControlPolicy.AccessControlList.Grants,
+					Owner:  accessControlPolicy.Owner,
+					Grants: grants,
 				},
 			}
 		}
@@ -1811,7 +2032,24 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if copySource != "" {
-		err := auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource,
+		cs := copySource
+		copySource, err := url.QueryUnescape(copySource)
+		if err != nil {
+			if c.debug {
+				log.Printf("error unescaping copy source %q: %v",
+					cs, err)
+			}
+			return SendXMLResponse(ctx, nil,
+				s3err.GetAPIError(s3err.ErrInvalidCopySource),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionCopyObject,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource,
 			auth.AccessOptions{
 				Acl:           parsedAcl,
 				AclPermission: types.PermissionWrite,
@@ -1872,6 +2110,22 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 
 		metadata := utils.GetUserMetaData(&ctx.Request().Header)
 
+		if directive != "" && directive != "COPY" && directive != "REPLACE" {
+			return SendXMLResponse(ctx, nil,
+				s3err.GetAPIError(s3err.ErrInvalidMetadataDirective),
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionCopyObject,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		metaDirective := types.MetadataDirectiveCopy
+		if directive == "REPLACE" {
+			metaDirective = types.MetadataDirectiveReplace
+		}
+
 		res, err := c.be.CopyObject(ctx.Context(),
 			&s3.CopyObjectInput{
 				Bucket:                      &bucket,
@@ -1883,6 +2137,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				CopySourceIfUnmodifiedSince: umtime,
 				ExpectedBucketOwner:         &acct.Access,
 				Metadata:                    metadata,
+				MetadataDirective:           metaDirective,
 				StorageClass:                types.StorageClass(storageClass),
 			})
 		if err == nil {
@@ -1982,6 +2237,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			Bucket:                    &bucket,
 			Key:                       &keyStart,
 			ContentLength:             &contentLength,
+			ContentType:               &contentType,
+			ContentEncoding:           &contentEncoding,
 			Metadata:                  metadata,
 			Body:                      body,
 			Tagging:                   &tagging,
@@ -2042,6 +2299,38 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("ownershipControls") {
+		err := auth.VerifyAccess(ctx.Context(), c.be,
+			auth.AccessOptions{
+				Readonly:      c.readonly,
+				Acl:           parsedAcl,
+				AclPermission: types.PermissionWrite,
+				IsRoot:        isRoot,
+				Acc:           acct,
+				Bucket:        bucket,
+				Action:        auth.PutBucketOwnershipControlsAction,
+			})
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					MetricsMng:  c.mm,
+					Action:      metrics.ActionDeleteBucketOwnershipControls,
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = c.be.DeleteBucketOwnershipControls(ctx.Context(), bucket)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				MetricsMng:  c.mm,
+				Action:      metrics.ActionDeleteBucketOwnershipControls,
+				BucketOwner: parsedAcl.Owner,
+				Status:      http.StatusNoContent,
+			})
+	}
+
 	if ctx.Request().URI().QueryArgs().Has("policy") {
 		err := auth.VerifyAccess(ctx.Context(), c.be,
 			auth.AccessOptions{
@@ -2113,7 +2402,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-	bypass := ctx.Get("X-Amz-Bypass-Governance-Retention")
+	bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention")
 	var dObj s3response.DeleteObjects
 
 	err := xml.Unmarshal(ctx.Body(), &dObj)
@@ -2150,7 +2439,10 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, utils.ParseDeleteObjects(dObj.Objects), bypass == "true", c.be)
+	// The AWS CLI sends 'True', while Go SDK sends 'true'
+	bypass := strings.EqualFold(bypassHdr, "true")
+
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, utils.ParseDeleteObjects(dObj.Objects), bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2189,11 +2481,15 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-	bypass := ctx.Get("X-Amz-Bypass-Governance-Retention")
+	bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention")
 
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
+	path := ctx.Path()
+	if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" {
+		key = key + "/"
+	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
 		err := auth.VerifyAccess(ctx.Context(), c.be,
@@ -2294,7 +2590,10 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{key}, bypass == "true", c.be)
+	// The AWS CLI sends 'True', while Go SDK sends 'true'
+	bypass := strings.EqualFold(bypassHdr, "true")
+
+	err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []string{key}, bypass, c.be)
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -2389,6 +2688,10 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
+	path := ctx.Path()
+	if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" {
+		key = key + "/"
+	}
 
 	var partNumber *int32
 	if ctx.Request().URI().QueryArgs().Has("partNumber") {
@@ -2464,10 +2767,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 			Key:   "ETag",
 			Value: getstring(res.ETag),
 		},
-		{
-			Key:   "x-amz-storage-class",
-			Value: string(res.StorageClass),
-		},
 		{
 			Key:   "x-amz-restore",
 			Value: getstring(res.Restore),
@@ -2511,12 +2810,22 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 			Value: getstring(res.ContentEncoding),
 		})
 	}
-	if res.ContentType != nil {
+	if res.StorageClass != "" {
 		headers = append(headers, utils.CustomHeader{
-			Key:   "Content-Type",
-			Value: getstring(res.ContentType),
+			Key:   "x-amz-storage-class",
+			Value: string(res.StorageClass),
 		})
 	}
+
+	contentType := getstring(res.ContentType)
+	if contentType == "" {
+		contentType = defaultContentType
+	}
+	headers = append(headers, utils.CustomHeader{
+		Key:   "Content-Type",
+		Value: contentType,
+	})
+
 	utils.SetResponseHeaders(ctx, headers)
 
 	return SendResponse(ctx, nil,
@@ -2537,6 +2846,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	contentType := ctx.Get("Content-Type")
+	contentEncoding := ctx.Get("Content-Encoding")
 	tagging := ctx.Get("X-Amz-Tagging")
 
 	if keyEnd != "" {
@@ -2548,23 +2858,20 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 		key = key + "/"
 	}
 
-	var restoreRequest s3.RestoreObjectInput
 	if ctx.Request().URI().QueryArgs().Has("restore") {
-		err := xml.Unmarshal(ctx.Body(), &restoreRequest)
-		if err != nil {
-			if c.debug {
-				log.Printf("error unmarshalling restore object: %v", err)
+		var restoreRequest types.RestoreRequest
+		if err := xml.Unmarshal(ctx.Body(), &restoreRequest); err != nil {
+			if !errors.Is(err, io.EOF) {
+				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+					&MetaOpts{
+						Logger:      c.logger,
+						MetricsMng:  c.mm,
+						Action:      metrics.ActionRestoreObject,
+						BucketOwner: parsedAcl.Owner,
+					})
 			}
-			return SendResponse(ctx, err,
-				&MetaOpts{
-					Logger:      c.logger,
-					MetricsMng:  c.mm,
-					Action:      metrics.ActionRestoreObject,
-					BucketOwner: parsedAcl.Owner,
-				})
 		}
-
-		err = auth.VerifyAccess(ctx.Context(), c.be,
+		err := auth.VerifyAccess(ctx.Context(), c.be,
 			auth.AccessOptions{
 				Readonly:      c.readonly,
 				Acl:           parsedAcl,
@@ -2585,10 +2892,11 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		restoreRequest.Bucket = &bucket
-		restoreRequest.Key = &key
-
-		err = c.be.RestoreObject(ctx.Context(), &restoreRequest)
+		err = c.be.RestoreObject(ctx.Context(), &s3.RestoreObjectInput{
+			Bucket:         &bucket,
+			Key:            &key,
+			RestoreRequest: &restoreRequest,
+		})
 		return SendResponse(ctx, err,
 			&MetaOpts{
 				Logger:      c.logger,
@@ -2768,6 +3076,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			Key:                       &key,
 			Tagging:                   &tagging,
 			ContentType:               &contentType,
+			ContentEncoding:           &contentEncoding,
 			ObjectLockRetainUntilDate: &objLockState.RetainUntilDate,
 			ObjectLockMode:            objLockState.ObjectLockMode,
 			ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,

s3api/controllers/base_test.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -191,7 +190,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
 				return s3response.GetObjectAttributesResult{}, nil
 			},
-			GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+			GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
 					Metadata:        map[string]string{"hello": "world"},
 					ContentType:     getPtr("application/xml"),
@@ -374,11 +373,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 				return s3response.ListMultipartUploadsResult{}, nil
 			},
-			ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-				return &s3.ListObjectsV2Output{}, nil
+			ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+				return s3response.ListObjectsV2Result{}, nil
 			},
-			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-				return &s3.ListObjectsOutput{}, nil
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+				return s3response.ListObjectsResult{}, nil
 			},
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return map[string]string{}, nil
@@ -395,6 +394,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 				return objectLockResult, nil
 			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerEnforced, nil
+			},
 		},
 	}
 
@@ -414,8 +416,8 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+				return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 			},
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -448,6 +450,15 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 404,
 		},
+		{
+			name: "Get-bucket-ownership-control-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-bucket-tagging-success",
 			app:  app,
@@ -562,7 +573,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	app := fiber.New()
 
 	// Mock valid acl
-	acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
+	acl := auth.ACL{Owner: "valid access"}
 	acldata, err := json.Marshal(acl)
 	if err != nil {
 		t.Errorf("Failed to parse the params: %v", err.Error())
@@ -636,6 +647,22 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</ObjectLockConfiguration>
 	`
 
+	ownershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
+	invalidOwnershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>invalid_value</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -659,6 +686,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
 				return nil
 			},
+			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+				return nil
+			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerPreferred, nil
+			},
 		},
 	}
 	// Mock ctx.Locals
@@ -691,6 +724,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
 	errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
+	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -716,6 +752,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Put-bucket-ownership-controls-invalid-ownership",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Put-object-lock-configuration-invalid-body",
 			app:  app,
@@ -804,7 +858,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 				req: incorrectBucketOwner,
 			},
 			wantErr:    false,
-			statusCode: 403,
+			statusCode: 400,
 		},
 		{
 			name: "Put-bucket-acl-success",
@@ -816,7 +870,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 200,
 		},
 		{
-			name: "Put-bucket-invalid-bucket-name",
+			name: "Create-bucket-invalid-acl-ownership-combination",
+			app:  app,
+			args: args{
+				req: invAclOwnershipReq,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Create-bucket-invalid-bucket-name",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
@@ -825,7 +888,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 400,
 		},
 		{
-			name: "Put-bucket-success",
+			name: "Create-bucket-success",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
@@ -1160,6 +1223,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 				return nil
 			},
+			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
+			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
 		},
 	}
 
@@ -1198,6 +1267,23 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			wantErr:    false,
 			statusCode: 204,
 		},
+		{
+			name: "Delete-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		}, {
+			name: "Delete-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -1611,8 +1697,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 				return &s3.CompleteMultipartUploadOutput{}, nil
 			},
-			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-				return &s3.CreateMultipartUploadOutput{}, nil
+			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+				return s3response.InitiateMultipartUploadResult{}, nil
 			},
 			SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
 				return func(w *bufio.Writer) {}
@@ -1646,20 +1732,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 		{
 			name: "Restore-object-success",
 			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
-		{
-			name: "Restore-object-error",
-			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 200,
 		},
 		{
 			name: "Select-object-content-invalid-body",

s3api/controllers/iam_moq_test.go

@@ -33,6 +33,9 @@ var _ auth.IAMService = &IAMServiceMock{}
 //			ShutdownFunc: func() error {
 //				panic("mock out the Shutdown method")
 //			},
+//			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+//				panic("mock out the UpdateUserAccount method")
+//			},
 //		}
 //
 //		// use mockedIAMService in code that requires auth.IAMService
@@ -55,6 +58,9 @@ type IAMServiceMock struct {
 	// ShutdownFunc mocks the Shutdown method.
 	ShutdownFunc func() error
 
+	// UpdateUserAccountFunc mocks the UpdateUserAccount method.
+	UpdateUserAccountFunc func(access string, props auth.MutableProps) error
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// CreateAccount holds details about calls to the CreateAccount method.
@@ -78,12 +84,20 @@ type IAMServiceMock struct {
 		// Shutdown holds details about calls to the Shutdown method.
 		Shutdown []struct {
 		}
+		// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
+		UpdateUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+			// Props is the props argument value.
+			Props auth.MutableProps
+		}
 	}
 	lockCreateAccount     sync.RWMutex
 	lockDeleteUserAccount sync.RWMutex
 	lockGetUserAccount    sync.RWMutex
 	lockListUserAccounts  sync.RWMutex
 	lockShutdown          sync.RWMutex
+	lockUpdateUserAccount sync.RWMutex
 }
 
 // CreateAccount calls CreateAccountFunc.
@@ -235,3 +249,39 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
 	mock.lockShutdown.RUnlock()
 	return calls
 }
+
+// UpdateUserAccount calls UpdateUserAccountFunc.
+func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
+	if mock.UpdateUserAccountFunc == nil {
+		panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+		Props  auth.MutableProps
+	}{
+		Access: access,
+		Props:  props,
+	}
+	mock.lockUpdateUserAccount.Lock()
+	mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
+	mock.lockUpdateUserAccount.Unlock()
+	return mock.UpdateUserAccountFunc(access, props)
+}
+
+// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.UpdateUserAccountCalls())
+func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
+	Access string
+	Props  auth.MutableProps
+} {
+	var calls []struct {
+		Access string
+		Props  auth.MutableProps
+	}
+	mock.lockUpdateUserAccount.RLock()
+	calls = mock.calls.UpdateUserAccount
+	mock.lockUpdateUserAccount.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -50,7 +50,8 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			!ctx.Request().URI().QueryArgs().Has("tagging") &&
 			!ctx.Request().URI().QueryArgs().Has("versioning") &&
 			!ctx.Request().URI().QueryArgs().Has("policy") &&
-			!ctx.Request().URI().QueryArgs().Has("object-lock") {
+			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
+			!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}

s3api/middlewares/url-decoder.go

@@ -26,12 +26,11 @@ import (
 
 func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		reqURL := ctx.Request().URI().String()
-		decoded, err := url.Parse(reqURL)
+		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
 		if err != nil {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}
-		ctx.Path(decoded.Path)
+		ctx.Path(unescp)
 		return ctx.Next()
 	}
 }

s3api/router.go

@@ -28,11 +28,11 @@ type S3ApiRouter struct {
 	WithAdmSrv bool
 }
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
 	s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
 
 	if sa.WithAdmSrv {
-		adminController := controllers.NewAdminController(iam, be)
+		adminController := controllers.NewAdminController(iam, be, aLogger)
 
 		// CreateUser admin api
 		app.Patch("/create-user", adminController.CreateUser)
@@ -40,6 +40,9 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		// DeleteUsers admin api
 		app.Patch("/delete-user", adminController.DeleteUser)
 
+		// UpdateUser admin api
+		app.Patch("update-user", adminController.UpdateUser)
+
 		// ListUsers admin api
 		app.Patch("/list-users", adminController.ListUsers)
 

s3api/router_test.go

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, false, false)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false)
 		})
 	}
 }

s3api/server.go

@@ -47,6 +47,7 @@ func New(
 	port, region string,
 	iam auth.IAMService,
 	l s3log.AuditLogger,
+	adminLogger s3log.AuditLogger,
 	evs s3event.S3EventSender,
 	mm *metrics.Manager,
 	opts ...Option,
@@ -64,7 +65,9 @@ func New(
 
 	// Logging middlewares
 	if !server.quiet {
-		app.Use(logger.New())
+		app.Use(logger.New(logger.Config{
+			Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
+		}))
 	}
 	// Set up health endpoint if specified
 	if server.health != "" {
@@ -82,7 +85,7 @@ func New(
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l, server.readonly))
 
-	server.router.Init(app, be, iam, l, evs, mm, server.debug, server.readonly)
+	server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly)
 
 	return server, nil
 }

s3api/server_test.go

@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
-				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil)
+				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
 				return

s3api/utils/utils.go

@@ -39,6 +39,10 @@ var (
 	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
+const (
+	upperhex = "0123456789ABCDEF"
+)
+
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
 	headers.DisableNormalizing()
@@ -64,7 +68,9 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		body = bytes.NewReader(req.Body())
 	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
+	escapedURI := escapeOriginalURI(ctx)
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -259,7 +265,7 @@ func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3
 		output.ObjectSize = nil
 	}
 	if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
-		output.StorageClass = nil
+		output.StorageClass = ""
 	}
 
 	return output
@@ -326,3 +332,87 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 		LegalHoldStatus: legalHold,
 	}, nil
 }
+
+func IsValidOwnership(val types.ObjectOwnership) bool {
+	switch val {
+	case types.ObjectOwnershipBucketOwnerEnforced:
+		return true
+	case types.ObjectOwnershipBucketOwnerPreferred:
+		return true
+	case types.ObjectOwnershipObjectWriter:
+		return true
+	default:
+		return false
+	}
+}
+
+func escapeOriginalURI(ctx *fiber.Ctx) string {
+	path := ctx.Path()
+
+	// Escape the URI original path
+	escapedURI := escapePath(path)
+
+	// Add the URI query params
+	query := string(ctx.Request().URI().QueryArgs().QueryString())
+	if query != "" {
+		escapedURI = escapedURI + "?" + query
+	}
+
+	return escapedURI
+}
+
+// Escapes the path string
+// Most of the parts copied from std url
+func escapePath(s string) string {
+	hexCount := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			hexCount++
+		}
+	}
+
+	if hexCount == 0 {
+		return s
+	}
+
+	var buf [64]byte
+	var t []byte
+
+	required := len(s) + 2*hexCount
+	if required <= len(buf) {
+		t = buf[:required]
+	} else {
+		t = make([]byte, required)
+	}
+
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case shouldEscape(c):
+			t[j] = '%'
+			t[j+1] = upperhex[c>>4]
+			t[j+2] = upperhex[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+
+	return string(t)
+}
+
+// Checks if the character needs to be escaped
+func shouldEscape(c byte) bool {
+	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '_', '.', '~', '/':
+		return false
+	}
+
+	return true
+}

s3api/utils/utils_test.go

@@ -335,3 +335,172 @@ func TestFilterObjectAttributes(t *testing.T) {
 		})
 	}
 }
+
+func TestIsValidOwnership(t *testing.T) {
+	type args struct {
+		val types.ObjectOwnership
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "valid-BucketOwnerEnforced",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerEnforced,
+			},
+			want: true,
+		},
+		{
+			name: "valid-BucketOwnerPreferred",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerPreferred,
+			},
+			want: true,
+		},
+		{
+			name: "valid-ObjectWriter",
+			args: args{
+				val: types.ObjectOwnershipObjectWriter,
+			},
+			want: true,
+		},
+		{
+			name: "invalid_value",
+			args: args{
+				val: types.ObjectOwnership("invalid_value"),
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidOwnership(tt.args.val); got != tt.want {
+				t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_shouldEscape(t *testing.T) {
+	type args struct {
+		c byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "shouldn't-escape-alphanum",
+			args: args{
+				c: 'h',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-char",
+			args: args{
+				c: '_',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-number",
+			args: args{
+				c: '0',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-path-separator",
+			args: args{
+				c: '/',
+			},
+			want: false,
+		},
+		{
+			name: "should-escape-special-char-1",
+			args: args{
+				c: '&',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-2",
+			args: args{
+				c: '*',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-3",
+			args: args{
+				c: '(',
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldEscape(tt.args.c); got != tt.want {
+				t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_escapePath(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "empty-string",
+			args: args{
+				s: "",
+			},
+			want: "",
+		},
+		{
+			name: "alphanum-path",
+			args: args{
+				s: "/test-bucket/test-key",
+			},
+			want: "/test-bucket/test-key",
+		},
+		{
+			name: "path-with-unescapable-chars",
+			args: args{
+				s: "/test~bucket/test.key",
+			},
+			want: "/test~bucket/test.key",
+		},
+		{
+			name: "path-with-escapable-chars",
+			args: args{
+				s: "/bucket-*(/test=key&",
+			},
+			want: "/bucket-%2A%28/test%3Dkey%26",
+		},
+		{
+			name: "path-with-space",
+			args: args{
+				s: "/test-bucket/my key",
+			},
+			want: "/test-bucket/my%20key",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := escapePath(tt.args.s); got != tt.want {
+				t.Errorf("escapePath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -123,6 +123,15 @@ const (
 	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
 	ErrRequestTimeTooSkewed
+	ErrInvalidBucketAclWithObjectOwnership
+	ErrBothCannedAndHeaderGrants
+	ErrOwnershipControlsNotFound
+	ErrAclNotSupported
+	ErrMalformedACL
+	ErrUnexpectedContent
+	ErrMissingSecurityHeader
+	ErrInvalidMetadataDirective
+	ErrKeyTooLong
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
@@ -144,7 +153,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
-		Description:    "The bucket you tried to delete is not empty",
+		Description:    "The bucket you tried to delete is not empty.",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrBucketAlreadyExists: {
@@ -169,17 +178,17 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrInvalidMaxUploads: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxKeys: {
 		Code:           "InvalidArgument",
-		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartNumberMarker: {
@@ -189,7 +198,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrNoSuchBucket: {
 		Code:           "NoSuchBucket",
-		Description:    "The specified bucket does not exist",
+		Description:    "The specified bucket does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchKey: {
@@ -214,7 +223,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrInvalidPartNumber: {
 		Code:           "InvalidArgument",
-		Description:    "Part number must be an integer between 1 and 10000, inclusive",
+		Description:    "Part number must be an integer between 1 and 10000, inclusive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidCopyDest: {
@@ -259,7 +268,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
-		Description:    "Invalid according to Policy: Policy Condition failed",
+		Description:    "Invalid according to Policy: Policy Condition failed.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrEntityTooSmall: {
@@ -294,7 +303,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrMalformedPresignedDate: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
+		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSignHeadersTag: {
@@ -309,7 +318,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
-		Description:    "There were headers present in the request which were not signed",
+		Description:    "There were headers present in the request which were not signed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidQueryParams: {
@@ -324,22 +333,22 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrExpiredPresignRequest: {
 		Code:           "AccessDenied",
-		Description:    "Request has expired",
+		Description:    "Request has expired.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMalformedExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires should be a number",
+		Description:    "X-Amz-Expires should be a number.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNegativeExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires must be non-negative",
+		Description:    "X-Amz-Expires must be non-negative.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
+		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidAccessKeyID: {
@@ -349,7 +358,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrRequestNotReadyYet: {
 		Code:           "AccessDenied",
-		Description:    "Request is not valid yet",
+		Description:    "Request is not valid yet.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureDoesNotMatch: {
@@ -359,17 +368,17 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrSignatureDateDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
+		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureTerminationStr: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'",
+		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureIncorrService: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Credential should be scoped to correct service: s3",
+		Description:    "Credential should be scoped to correct service: s3.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrContentSHA256Mismatch: {
@@ -379,32 +388,32 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrMissingDateHeader: {
 		Code:           "AccessDenied",
-		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidRequest: {
 		Code:           "InvalidRequest",
-		Description:    "Invalid Request",
+		Description:    "Invalid Request.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAuthNotSetup: {
 		Code:           "InvalidRequest",
-		Description:    "Signed request requires setting up SeaweedFS S3 authentication",
+		Description:    "Signed request requires setting up SeaweedFS S3 authentication.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
-		Description:    "A header you provided implies functionality that is not implemented",
+		Description:    "A header you provided implies functionality that is not implemented.",
 		HTTPStatusCode: http.StatusNotImplemented,
 	},
 	ErrPreconditionFailed: {
 		Code:           "PreconditionFailed",
-		Description:    "At least one of the pre-conditions you specified did not hold",
+		Description:    "At least one of the pre-conditions you specified did not hold.",
 		HTTPStatusCode: http.StatusPreconditionFailed,
 	},
 	ErrInvalidObjectState: {
 		Code:           "InvalidObjectState",
-		Description:    "The operation is not valid for the current state of the object",
+		Description:    "The operation is not valid for the current state of the object.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrInvalidRange: {
@@ -419,52 +428,52 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrObjectLockConfigurationNotFound: {
 		Code:           "ObjectLockConfigurationNotFoundError",
-		Description:    "Object Lock configuration does not exist for this bucket",
+		Description:    "Object Lock configuration does not exist for this bucket.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
-		Description:    "The specified object does not have an ObjectLock configuration",
+		Description:    "The specified object does not have an ObjectLock configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidBucketObjectLockConfiguration: {
 		Code:           "InvalidRequest",
-		Description:    "Bucket is missing ObjectLockConfiguration",
+		Description:    "Bucket is missing ObjectLockConfiguration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectLockConfigurationNotAllowed: {
 		Code:           "InvalidBucketState",
-		Description:    "Object Lock configuration cannot be enabled on existing buckets",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrObjectLocked: {
 		Code:           "InvalidRequest",
-		Description:    "Object is WORM protected and cannot be overwritten",
+		Description:    "Object is WORM protected and cannot be overwritten.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrPastObjectLockRetainDate: {
 		Code:           "InvalidRequest",
-		Description:    "the retain until date must be in the future",
+		Description:    "the retain until date must be in the future.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectLockInvalidRetentionPeriod: {
 		Code:           "InvalidRetentionPeriod",
-		Description:    "the retention days/years must be positive integer",
+		Description:    "the retention days/years must be positive integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNoSuchBucketPolicy: {
 		Code:           "NoSuchBucketPolicy",
-		Description:    "The bucket policy does not exist",
+		Description:    "The bucket policy does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrBucketTaggingNotFound: {
 		Code:           "NoSuchTagSet",
-		Description:    "The TagSet does not exist",
+		Description:    "The TagSet does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrObjectLockInvalidHeaders: {
 		Code:           "InvalidRequest",
-		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
+		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrRequestTimeTooSkewed: {
@@ -472,6 +481,51 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The difference between the request time and the server's time is too large.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidBucketAclWithObjectOwnership: {
+		Code:           "ErrInvalidBucketAclWithObjectOwnership",
+		Description:    "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBothCannedAndHeaderGrants: {
+		Code:           "InvalidRequest",
+		Description:    "Specifying both Canned ACLs and Header Grants is not allowed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOwnershipControlsNotFound: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAclNotSupported: {
+		Code:           "AccessControlListNotSupported",
+		Description:    "The bucket does not allow ACLs.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedACL: {
+		Code:           "MalformedACLError",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrUnexpectedContent: {
+		Code:           "UnexpectedContent",
+		Description:    "This request does not support content.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSecurityHeader: {
+		Code:           "MissingSecurityHeader",
+		Description:    "Your request was missing a required header.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrInvalidMetadataDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown metadata directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrKeyTooLong: {
+		Code:           "KeyTooLongError",
+		Description:    "Your key is too long.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {

s3log/audit-logger.go

@@ -35,11 +35,13 @@ type LogMeta struct {
 	BucketOwner string
 	ObjectSize  int64
 	Action      string
+	HttpStatus  int
 }
 
 type LogConfig struct {
-	LogFile    string
-	WebhookURL string
+	LogFile      string
+	WebhookURL   string
+	AdminLogFile string
 }
 
 type LogFields struct {
@@ -71,18 +73,66 @@ type LogFields struct {
 	AclRequired        string
 }
 
-func InitLogger(cfg *LogConfig) (AuditLogger, error) {
+type AdminLogFields struct {
+	Time               time.Time
+	RemoteIP           string
+	Requester          string
+	RequestID          string
+	Operation          string
+	RequestURI         string
+	HttpStatus         int
+	ErrorCode          string
+	BytesSent          int
+	TotalTime          int64
+	TurnAroundTime     int64
+	Referer            string
+	UserAgent          string
+	SignatureVersion   string
+	CipherSuite        string
+	AuthenticationType string
+	TLSVersion         string
+}
+
+type Loggers struct {
+	S3Logger    AuditLogger
+	AdminLogger AuditLogger
+}
+
+func InitLogger(cfg *LogConfig) (*Loggers, error) {
 	if cfg.WebhookURL != "" && cfg.LogFile != "" {
 		return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
 	}
-	if cfg.WebhookURL != "" {
-		return InitWebhookLogger(cfg.WebhookURL)
-	}
-	if cfg.LogFile != "" {
-		return InitFileLogger(cfg.LogFile)
+	loggers := new(Loggers)
+
+	switch {
+	case cfg.WebhookURL != "":
+		fmt.Printf("initializing S3 access logs with '%v' webhook url\n", cfg.WebhookURL)
+		l, err := InitWebhookLogger(cfg.WebhookURL)
+		if err != nil {
+			return nil, err
+		}
+		loggers.S3Logger = l
+	case cfg.LogFile != "":
+		fmt.Printf("initializing S3 access logs with '%v' file\n", cfg.LogFile)
+		l, err := InitFileLogger(cfg.LogFile)
+		if err != nil {
+			return nil, err
+		}
+
+		loggers.S3Logger = l
 	}
 
-	return nil, nil
+	if cfg.AdminLogFile != "" {
+		fmt.Printf("initializing admin access logs with '%v' file\n", cfg.AdminLogFile)
+		l, err := InitAdminFileLogger(cfg.AdminLogFile)
+		if err != nil {
+			return nil, err
+		}
+
+		loggers.AdminLogger = l
+	}
+
+	return loggers, nil
 }
 
 func genID() string {

s3log/file_admin.go

@@ -0,0 +1,151 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"crypto/tls"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+)
+
+// FileLogger is a local file audit log
+type AdminFileLogger struct {
+	FileLogger
+}
+
+var _ AuditLogger = &AdminFileLogger{}
+
+// InitFileLogger initializes audit logs to local file
+func InitAdminFileLogger(logname string) (AuditLogger, error) {
+	f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("open log: %w", err)
+	}
+
+	f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return &AdminFileLogger{FileLogger: FileLogger{logfile: logname, f: f}}, nil
+}
+
+// Log sends log message to file logger
+func (f *AdminFileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if f.gotErr {
+		return
+	}
+
+	lf := AdminLogFields{}
+
+	access := "-"
+	reqURI := ctx.OriginalURL()
+	errorCode := ""
+	startTime := ctx.Locals("startTime").(time.Time)
+	tlsConnState := ctx.Context().TLSConnectionState()
+	if tlsConnState != nil {
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
+	}
+
+	if err != nil {
+		errorCode = err.Error()
+	}
+
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
+	}
+
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.RequestURI = reqURI
+	lf.HttpStatus = meta.HttpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+
+	f.writeLog(lf)
+}
+
+func (f *AdminFileLogger) writeLog(lf AdminLogFields) {
+	if lf.RemoteIP == "" {
+		lf.RemoteIP = "-"
+	}
+	if lf.Requester == "" {
+		lf.Requester = "-"
+	}
+	if lf.Operation == "" {
+		lf.Operation = "-"
+	}
+	if lf.RequestURI == "" {
+		lf.RequestURI = "-"
+	}
+	if lf.ErrorCode == "" {
+		lf.ErrorCode = "-"
+	}
+	if lf.Referer == "" {
+		lf.Referer = "-"
+	}
+	if lf.UserAgent == "" {
+		lf.UserAgent = "-"
+	}
+	if lf.CipherSuite == "" {
+		lf.CipherSuite = "-"
+	}
+	if lf.TLSVersion == "" {
+		lf.TLSVersion = "-"
+	}
+
+	log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
+		fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
+		lf.RemoteIP,
+		lf.Requester,
+		lf.RequestID,
+		lf.Operation,
+		lf.RequestURI,
+		lf.HttpStatus,
+		lf.ErrorCode,
+		lf.BytesSent,
+		lf.TotalTime,
+		lf.TurnAroundTime,
+		lf.Referer,
+		lf.UserAgent,
+		lf.SignatureVersion,
+		lf.CipherSuite,
+		lf.AuthenticationType,
+		lf.TLSVersion,
+	)
+
+	_, err := f.f.WriteString(log)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
+		// TODO: do we need to terminate on log error?
+		// set err for now so that we don't spew errors
+		f.gotErr = true
+	}
+}

s3response/s3response.go

@@ -21,14 +21,30 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
+const RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+
 // Part describes part metadata.
 type Part struct {
 	PartNumber   int
-	LastModified string
+	LastModified time.Time
 	ETag         string
 	Size         int64
 }
 
+func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias Part
+	aux := &struct {
+		LastModified string `xml:"LastModified"`
+		*Alias
+	}{
+		Alias: (*Alias)(&p),
+	}
+
+	aux.LastModified = p.LastModified.UTC().Format(RFC3339TimeFormat)
+
+	return e.EncodeElement(aux, start)
+}
+
 // ListPartsResponse - s3 api list parts response.
 type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
@@ -41,7 +57,7 @@ type ListPartsResult struct {
 	Owner     Owner
 
 	// The class of storage used to store the object.
-	StorageClass string
+	StorageClass types.StorageClass
 
 	PartNumberMarker     int
 	NextPartNumberMarker int
@@ -56,11 +72,28 @@ type GetObjectAttributesResult struct {
 	ETag         *string
 	LastModified *time.Time
 	ObjectSize   *int64
-	StorageClass *types.StorageClass
+	StorageClass types.StorageClass
 	VersionId    *string
 	ObjectParts  *ObjectParts
 }
 
+func (r GetObjectAttributesResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias GetObjectAttributesResult
+	aux := &struct {
+		LastModified *string `xml:"LastModified"`
+		*Alias
+	}{
+		Alias: (*Alias)(&r),
+	}
+
+	if r.LastModified != nil {
+		formattedTime := r.LastModified.UTC().Format(RFC3339TimeFormat)
+		aux.LastModified = &formattedTime
+	}
+
+	return e.EncodeElement(aux, start)
+}
+
 type ObjectParts struct {
 	PartNumberMarker     int
 	NextPartNumberMarker int
@@ -91,14 +124,85 @@ type ListMultipartUploadsResult struct {
 	CommonPrefixes []CommonPrefix
 }
 
+type ListObjectsResult struct {
+	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	Name           *string
+	Prefix         *string
+	Marker         *string
+	NextMarker     *string
+	MaxKeys        *int32
+	Delimiter      *string
+	IsTruncated    *bool
+	Contents       []Object
+	CommonPrefixes []types.CommonPrefix
+	EncodingType   types.EncodingType
+}
+
+type ListObjectsV2Result struct {
+	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	Name                  *string
+	Prefix                *string
+	StartAfter            *string
+	ContinuationToken     *string
+	NextContinuationToken *string
+	KeyCount              *int32
+	MaxKeys               *int32
+	Delimiter             *string
+	IsTruncated           *bool
+	Contents              []Object
+	CommonPrefixes        []types.CommonPrefix
+	EncodingType          types.EncodingType
+}
+
+type Object struct {
+	ETag          *string
+	Key           *string
+	LastModified  *time.Time
+	Owner         *types.Owner
+	RestoreStatus *types.RestoreStatus
+	Size          *int64
+	StorageClass  types.ObjectStorageClass
+}
+
+func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias Object
+	aux := &struct {
+		LastModified *string `xml:"LastModified,omitempty"`
+		*Alias
+	}{
+		Alias: (*Alias)(&o),
+	}
+
+	if o.LastModified != nil {
+		formattedTime := o.LastModified.UTC().Format(RFC3339TimeFormat)
+		aux.LastModified = &formattedTime
+	}
+
+	return e.EncodeElement(aux, start)
+}
+
 // Upload describes in progress multipart upload
 type Upload struct {
 	Key          string
 	UploadID     string `xml:"UploadId"`
 	Initiator    Initiator
 	Owner        Owner
-	StorageClass string
-	Initiated    string
+	StorageClass types.StorageClass
+	Initiated    time.Time
+}
+
+func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias Upload
+	aux := &struct {
+		Initiated string `xml:"Initiated"`
+		*Alias
+	}{
+		Alias: (*Alias)(&u),
+	}
+
+	aux.Initiated = u.Initiated.UTC().Format(RFC3339TimeFormat)
+
+	return e.EncodeElement(aux, start)
 }
 
 // CommonPrefix ListObjectsResponse common prefixes (directory abstraction)
@@ -174,6 +278,20 @@ type ListAllMyBucketsEntry struct {
 	CreationDate time.Time
 }
 
+func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias ListAllMyBucketsEntry
+	aux := &struct {
+		CreationDate string `xml:"CreationDate"`
+		*Alias
+	}{
+		Alias: (*Alias)(&r),
+	}
+
+	aux.CreationDate = r.CreationDate.UTC().Format(RFC3339TimeFormat)
+
+	return e.EncodeElement(aux, start)
+}
+
 type ListAllMyBucketsList struct {
 	Bucket []ListAllMyBucketsEntry
 }
@@ -189,6 +307,20 @@ type CopyObjectResult struct {
 	ETag         string
 }
 
+func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias CopyObjectResult
+	aux := &struct {
+		LastModified string `xml:"LastModified"`
+		*Alias
+	}{
+		Alias: (*Alias)(&r),
+	}
+
+	aux.LastModified = r.LastModified.UTC().Format(RFC3339TimeFormat)
+
+	return e.EncodeElement(aux, start)
+}
+
 type AccessControlPolicy struct {
 	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
 	Owner             CanonicalUser
@@ -217,3 +349,14 @@ type Grantee struct {
 	ID          string
 	DisplayName string
 }
+
+type OwnershipControls struct {
+	Rules []types.OwnershipControlsRule `xml:"Rule"`
+}
+
+type InitiateMultipartUploadResult struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+	Bucket   string
+	Key      string
+	UploadId string
+}

tests/.env.default

@@ -17,4 +17,13 @@ GOCOVERDIR=$PWD/cover
 USERS_FOLDER=$PWD/iam
 #TEST_LOG_FILE=test.log
 #VERSITY_LOG_FILE=versity.log
-IAM_TYPE=folder
+IAM_TYPE=folder
+DIRECT=false
+#DIRECT_DISPLAY_NAME=
+#COVERAGE_DB=coverage.sql
+USERNAME_ONE=ABCDEFG
+PASSWORD_ONE=HIJKLMN
+USERNAME_TWO=HIJKLMN
+PASSWORD_TWO=OPQRSTU
+TEST_FILE_FOLDER=$PWD/versity-gwtest-files
+REMOVE_TEST_FILE_FOLDER=true

tests/.env.docker.default

@@ -0,0 +1,28 @@
+AWS_PROFILE=versity
+AWS_ENDPOINT_URL=https://127.0.0.1:7070
+VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
+BACKEND=posix
+LOCAL_FOLDER=/tmp/gw
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+CERT=$PWD/cert-docker.pem
+KEY=$PWD/versitygw-docker.pem
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets
+MC_ALIAS=versity
+LOG_LEVEL=2
+USERS_FOLDER=$PWD/iam
+#TEST_LOG_FILE=test.log
+#VERSITY_LOG_FILE=versity.log
+IAM_TYPE=folder
+DIRECT=false
+#DIRECT_DISPLAY_NAME=
+#COVERAGE_DB=coverage.sql
+USERNAME_ONE=ABCDEFG
+PASSWORD_ONE=HIJKLMN
+USERNAME_TWO=HIJKLMN
+PASSWORD_TWO=OPQRSTU
+TEST_FILE_FOLDER=$PWD/versity-gwtest-files
+RECREATE_BUCKETS=true
+REMOVE_TEST_FILE_FOLDER=true

tests/.secrets.default

@@ -0,0 +1,5 @@
+# change to your account attributes
+AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+AWS_REGION=us-east-1
+AWS_PROFILE=versity

tests/Dockerfile_direct

@@ -0,0 +1,73 @@
+FROM ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG SECRETS_FILE=tests/.secrets.direct
+ARG CONFIG_FILE=tests/.env.direct
+ARG AWS_CLI=awscli-exe-linux-aarch64.zip
+ARG MC_FOLDER=linux-arm64
+
+ENV TZ=Etc/UTC
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    git \
+    wget \
+    curl \
+    unzip \
+    tzdata \
+    s3cmd \
+    jq \
+    bc \
+    libxml2-utils \
+    ca-certificates && \
+    update-ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /tmp
+
+# Install AWS cli
+RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+
+# Install mc
+RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
+  --create-dirs \
+  -o /usr/local/minio-binaries/mc && \
+chmod -R 755 /usr/local/minio-binaries
+ENV PATH=/usr/local/minio-binaries:${PATH}
+
+# Create tester user
+RUN groupadd -r tester && useradd -r -g tester tester
+RUN mkdir /home/tester && chown tester:tester /home/tester
+ENV HOME=/home/tester
+
+# install bats
+RUN git clone https://github.com/bats-core/bats-core.git && \
+    cd bats-core && \
+    ./install.sh /home/tester
+
+USER tester
+RUN mkdir -p /home/tester/tests
+COPY --chown=tester:tester . /home/tester/tests
+
+# add bats support libraries
+RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
+RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
+
+WORKDIR /home/tester
+
+RUN . $SECRETS_FILE && \
+    export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
+    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
+    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
+    aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
+
+RUN mkdir /tmp/gw
+
+RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen_bits:2048 && \
+    openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
+        -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+ENV WORKSPACE=.
+ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
+
+CMD ["tests/run_all.sh"]

tests/Dockerfile_test_bats

@@ -1,8 +1,11 @@
-FROM --platform=linux/arm64 ubuntu:latest
+FROM ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
 ARG SECRETS_FILE=tests/.secrets
 ARG CONFIG_FILE=tests/.env.docker
+ARG GO_LIBRARY=go1.23.1.linux-arm64.tar.gz
+ARG AWS_CLI=awscli-exe-linux-aarch64.zip
+ARG MC_FOLDER=linux-arm64
 
 ENV TZ=Etc/UTC
 RUN apt-get update && \
@@ -16,6 +19,7 @@ RUN apt-get update && \
     s3cmd \
     jq \
     bc \
+    libxml2-utils \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*
@@ -24,20 +28,20 @@ RUN apt-get update && \
 WORKDIR /tmp
 
 # Install AWS cli
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
 
 # Install mc
-RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
+RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
   --create-dirs \
   -o /usr/local/minio-binaries/mc && \
 chmod -R 755 /usr/local/minio-binaries
-ENV PATH="/usr/local/minio-binaries":${PATH}
+ENV PATH=/usr/local/minio-binaries:${PATH}
 
 # Download Go 1.21 (adjust the version and platform as needed)
-RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
+RUN wget https://golang.org/dl/${GO_LIBRARY}
 
 # Extract the downloaded archive
-RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
+RUN tar -xvf $GO_LIBRARY -C /usr/local
 
 # Set Go environment variables
 ENV PATH="/usr/local/go/bin:${PATH}"
@@ -60,6 +64,10 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
 USER tester
 COPY --chown=tester:tester . /home/tester
 
+# add bats support libraries
+RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
+RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
+
 WORKDIR /home/tester
 RUN make
 

tests/README.md

@@ -9,9 +9,11 @@
    * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
    * **s3cmd**:  Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
    * **mc**:  Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
-3. Install BATS.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
-4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
-5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
+3. Install **BATS**.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
+4. Install **bats-support** and **bats-assert**.  This can be done by saving the root folder of each repo (https://github.com/bats-core/bats-support and https://github.com/ztombol/bats-assert) in the `tests` folder.
+5. If running on Mac OS, install **jq** with the command `brew install jq`.
+6. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, and `AWS_PROFILE` values to the file.
+7. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
 ```
     export AWS_PROFILE=versity-test
     export AWS_ACCESS_KEY_ID=<your account ID>
@@ -21,14 +23,18 @@
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE
     aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
 ```
-6. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
-7. If using SSL, create a local private key and certificate, such as with the commands below.  Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively.
+8. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
+9. If using SSL, create a local private key and certificate, such as with the commands below.  Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively.
 ```
     openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
     openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
 ```
-8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
-9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
+11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+
+### Static Bucket Mode
+
+To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`.  Two utility functions are included, if needed, to create, and delete buckets for this:  `tests/setup_static.sh` and `tests/remove_static.sh`.  Note that this creates a bucket with object lock enabled, and some tests may fail if the bucket being tested doesn't have object lock enabled.
 
 ### S3 Backend
 
@@ -52,8 +58,9 @@ To communicate directly with s3, in order to compare the gateway results to dire
 
 ## Instructions - Running With Docker
 
-1.  Create a `.secrets` file in the `tests` folder, and add the `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and the `AWS_PROFILE` fields, as well as the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
-2.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
+1.  Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
+2.  By default, the dockerfile uses the **arm** architecture (usually modern Mac).  If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="<param>=<amd library or folder>"`  Also, you can determine which is used by your OS with `uname -a`.
+3.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
 
 ## Instructions - Running with docker-compose
 
@@ -71,3 +78,7 @@ To run in insecure mode, comment out the `CERT` and `KEY` parameters in the `.en
 To use static buckets set the `RECREATE_BUCKETS` value to `false`.
 
 For the s3 backend, see the **S3 Backend** instructions above.
+
+If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries.
+
+A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`

tests/commands/abort_multipart_upload.sh

@@ -1,14 +1,41 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 abort_multipart_upload() {
+  record_command "abort-multipart-upload" "client:s3api"
   if [ $# -ne 3 ]; then
-    echo "command to run abort requires bucket, key, upload ID"
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID"
     return 1
   fi
+  if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $error"
+    return 1
+  fi
+  return 0
+}
 
-  error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3") || local aborted=$?
-  if [[ $aborted -ne 0 ]]; then
-    echo "Error aborting upload: $error"
+abort_multipart_upload_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID, username, password"
+    return 1
+  fi
+  record_command "abort-multipart-upload" "client:s3api"
+  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $abort_multipart_upload_error"
+    export abort_multipart_upload_error
     return 1
   fi
   return 0

tests/commands/complete_multipart_upload.sh

@@ -1,11 +1,26 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 complete_multipart_upload() {
   if [[ $# -ne 4 ]]; then
     log 2 "'complete multipart upload' command requires bucket, key, upload ID, parts list"
     return 1
   fi
   log 5 "complete multipart upload id: $3, parts: $4"
+  record_command "complete-multipart-upload" "client:s3api"
   error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$?
   if [[ $completed -ne 0 ]]; then
     log 2 "error completing multipart upload: $error"

tests/commands/copy_object.sh

@@ -1,5 +1,19 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 copy_object() {
   if [ $# -ne 4 ]; then
     echo "copy object command requires command type, source, bucket, key"
@@ -7,6 +21,7 @@ copy_object() {
   fi
   local exit_code=0
   local error
+  record_command "copy-object" "client:$1"
   if [[ $1 == 's3' ]]; then
     error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
@@ -29,6 +44,7 @@ copy_object() {
 }
 
 copy_object_empty() {
+  record-command "copy-object" "client:s3api"
   error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
   if [[ $result -eq 0 ]]; then
     log 2 "copy object with empty parameters returned no error"

tests/commands/create_bucket.sh

@@ -1,5 +1,21 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/report.sh
+
 # create an AWS bucket
 # param:  bucket name
 # return 0 for success, 1 for failure
@@ -9,6 +25,7 @@ create_bucket() {
     return 1
   fi
 
+  record_command "create-bucket" "client:$1"
   local exit_code=0
   local error
   log 6 "create bucket"
@@ -32,7 +49,31 @@ create_bucket() {
   return 0
 }
 
+create_bucket_with_user() {
+  if [ $# -ne 4 ]; then
+    log 2 "create bucket missing command type, bucket name, access, secret"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == "aws" ]] || [[ $1 == "s3api" ]]; then
+    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "mc" ]]; then
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    log 2 "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error creating bucket: $error"
+    return 1
+  fi
+  return 0
+}
+
 create_bucket_object_lock_enabled() {
+  record_command "create-bucket" "client:s3api"
   if [ $# -ne 1 ]; then
     log 2 "create bucket missing bucket name"
     return 1

tests/commands/create_multipart_upload.sh

@@ -1,27 +1,64 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # initialize a multipart upload
 # params:  bucket, key
 # return 0 for success, 1 for failure
 create_multipart_upload() {
+  record_command "create-multipart-upload" "client:s3api"
   if [ $# -ne 2 ]; then
     log 2 "create multipart upload function must have bucket, key"
     return 1
   fi
 
-  local multipart_data
-  multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2") || local created=$?
-  if [[ $created -ne 0 ]]; then
-    log 2 "Error creating multipart upload: $upload_id"
+  if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+    log 2 "Error creating multipart upload: $multipart_data"
     return 1
   fi
 
-  upload_id=$(echo "$multipart_data" | jq '.UploadId')
+  if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
+    log 2 "error parsing upload ID: $upload_id"
+    return 1
+  fi
   upload_id="${upload_id//\"/}"
-  export upload_id
+  return 0
+}
+
+create_multipart_upload_with_user() {
+  record_command "create-multipart-upload" "client:s3api"
+  if [ $# -ne 4 ]; then
+    log 2 "create multipart upload function must have bucket, key, username, password"
+    return 1
+  fi
+
+  if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
+    log 2 "Error creating multipart upload: $multipart_data"
+    return 1
+  fi
+
+  if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
+    log 2 "error parsing upload ID: $upload_id"
+    return 1
+  fi
+  upload_id="${upload_id//\"/}"
+  return 0
 }
 
 create_multipart_upload_params() {
+  record_command "create-multipart-upload" "client:s3api"
   if [ $# -ne 8 ]; then
     log 2 "create multipart upload function with params must have bucket, key, content type, metadata, object lock legal hold status, " \
       "object lock mode, object lock retain until date, and tagging"
@@ -41,14 +78,13 @@ create_multipart_upload_params() {
     log 2 "error creating multipart upload with params: $multipart_data"
     return 1
   fi
-  export multipart_data
   upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId')
   upload_id="${upload_id//\"/}"
-  export upload_id
   return 0
 }
 
 create_multipart_upload_custom() {
+  record_command "create-multipart-upload" "client:s3api"
   if [ $# -lt 2 ]; then
     log 2 "create multipart upload custom function must have at least bucket and key"
     return 1
@@ -65,11 +101,9 @@ create_multipart_upload_custom() {
     log 2 "error creating custom multipart data command: $multipart_data"
     return 1
   fi
-  export multipart_data
   log 5 "multipart data: $multipart_data"
   upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId')
   upload_id="${upload_id//\"/}"
   log 5 "upload id: $upload_id"
-  export upload_id
   return 0
 }

tests/commands/delete_bucket.sh

@@ -1,18 +1,37 @@
 #!/usr/bin/env bash
 
-# delete an AWS bucket
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # param:  bucket name
-# return 0 for success, 1 for failure
+# fail if params are bad, or bucket exists and user is unable to delete bucket
 delete_bucket() {
+  log 6 "delete_bucket"
+  record_command "delete-bucket" "client:$1"
   if [ $# -ne 2 ]; then
-    log 2 "delete bucket missing command type, bucket name"
+    log 2 "'delete_bucket' command requires client, bucket"
     return 1
   fi
 
-  local exit_code=0
-  local error
+  if [[ ( $RECREATE_BUCKETS == "false" ) && (( "$2" == "$BUCKET_ONE_NAME" ) || ( "$2" == "$BUCKET_TWO_NAME" )) ]]; then
+    log 2 "attempt to delete main buckets in static mode"
+    return 1
+  fi
+
+  exit_code=0
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 rb s3://"$2" 2>&1) || exit_code=$?
+    error=$(aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$?
   elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
     error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
@@ -26,10 +45,9 @@ delete_bucket() {
   if [ $exit_code -ne 0 ]; then
     if [[ "$error" == *"The specified bucket does not exist"* ]]; then
       return 0
-    else
-      log 2 "error deleting bucket: $error"
-      return 1
     fi
+    log 2 "error deleting bucket: $error"
+    return 1
   fi
   return 0
 }

tests/commands/delete_bucket_policy.sh

@@ -1,18 +1,34 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 delete_bucket_policy() {
+  record_command "delete-bucket-policy" "client:$1"
   if [[ $# -ne 2 ]]; then
     log 2 "delete bucket policy command requires command type, bucket"
     return 1
   fi
-  if [[ $1 == 'aws' ]]; then
+  local delete_result=0
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
     error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
     error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
   else
-    log 2 "command 'get bucket policy' not implemented for '$1'"
+    log 2 "command 'delete bucket policy' not implemented for '$1'"
     return 1
   fi
   if [[ $delete_result -ne 0 ]]; then
@@ -20,4 +36,18 @@ delete_bucket_policy() {
     return 1
   fi
   return 0
+}
+
+delete_bucket_policy_with_user() {
+  record_command "delete-bucket-policy" "client:s3api"
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete bucket policy with user' command requires bucket, username, password"
+    return 1
+  fi
+  if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
+    log 2 "error deleting bucket policy: $delete_bucket_policy_error"
+    export delete_bucket_policy_error
+    return 1
+  fi
+  return 0
 }

tests/commands/delete_bucket_tagging.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+delete_bucket_tagging() {
+  record_command "delete-bucket-tagging" "client:$1"
+  if [ $# -ne 2 ]; then
+    log 2 "delete bucket tagging command missing command type, bucket name"
+    return 1
+  fi
+  local result
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
+  else
+    log 2 "invalid command type $1"
+    return 1
+  fi
+  if [[ $result -ne 0 ]]; then
+    log 2 "error deleting bucket tagging: $tags"
+    return 1
+  fi
+  return 0
+}
+
+delete_bucket_tagging_with_user() {
+  log 6 "delete_bucket_tagging_with_user"
+  record_command "delete-bucket-tagging" "client:s3api"
+  if [ $# -ne 3 ]; then
+    log 2 "delete bucket tagging command missing username, password, bucket name"
+    return 1
+  fi
+  if ! error=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then
+    log 2 "error deleting bucket tagging with user: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/delete_object.sh

@@ -1,6 +1,23 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# params:  client, bucket, key
 delete_object() {
+  log 6 "delete_object"
+  record_command "delete-object" "client:$1"
   if [ $# -ne 3 ]; then
     log 2 "delete object command requires command type, bucket, key"
     return 1
@@ -27,7 +44,20 @@ delete_object() {
   return 0
 }
 
+delete_object_bypass_retention() {
+  if [[ $# -ne 4 ]]; then
+    log 2 "'delete-object with bypass retention' requires bucket, key, user, password"
+    return 1
+  fi
+  if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then
+    log 2 "error deleting object with bypass retention: $delete_object_error"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_with_user() {
+  record_command "delete-object" "client:$1"
   if [ $# -ne 5 ]; then
     log 2 "delete object with user command requires command type, bucket, key, access ID, secret key"
     return 1
@@ -36,7 +66,7 @@ delete_object_with_user() {
   if [[ $1 == 's3' ]]; then
     delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
+    delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
   else

tests/commands/delete_object_tagging.sh

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 delete_object_tagging() {
+  record_command "delete-object-tagging" "client:$1"
   if [[ $# -ne 3 ]]; then
     echo "delete object tagging command missing command type, bucket, key"
     return 1

tests/commands/delete_objects.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+delete_objects() {
+  record_command "delete-objects" "client:s3api"
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete-objects' command requires bucket name, two object keys"
+    return 1
+  fi
+  if ! error=$(aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
+      \"Objects\": [
+        {\"Key\": \"$2\"},
+        {\"Key\": \"$3\"}
+      ]
+    }" 2>&1); then
+    log 2 "error deleting objects: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/get_bucket_acl.sh

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_bucket_acl() {
+  record_command "get-bucket-acl" "client:$1"
   if [ $# -ne 2 ]; then
     log 2 "bucket ACL command missing command type, bucket name"
     return 1
@@ -18,5 +33,18 @@ get_bucket_acl() {
     log 2 "Error getting bucket ACLs: $acl"
     return 1
   fi
-  export acl
+  acl=$(echo "$acl" | grep -v "InsecureRequestWarning")
+}
+
+get_bucket_acl_with_user() {
+  record_command "get-bucket-acl" "client:s3api"
+  if [ $# -ne 3 ]; then
+    log 2 "'get bucket ACL with user' command requires bucket name, username, password"
+    return 1
+  fi
+  if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then
+    log 2 "error getting bucket ACLs: $bucket_acl"
+    return 1
+  fi
+  return 0
 }

tests/commands/get_bucket_location.sh

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_bucket_location() {
+  record_command "get-bucket-location" "client:$1"
   if [[ $# -ne 2 ]]; then
     echo "get bucket location command requires command type, bucket name"
     return 1
@@ -19,10 +34,10 @@ get_bucket_location() {
     return 1
   fi
   location=$(echo "$location_json" | jq -r '.LocationConstraint')
-  export location
 }
 
 get_bucket_location_aws() {
+  record_command "get-bucket-location" "client:s3api"
   if [[ $# -ne 1 ]]; then
     echo "get bucket location (aws) requires bucket name"
     return 1
@@ -33,11 +48,11 @@ get_bucket_location_aws() {
     return 1
   fi
   bucket_location=$(echo "$location_json" | jq -r '.LocationConstraint')
-  export bucket_location
   return 0
 }
 
 get_bucket_location_s3cmd() {
+  record_command "get-bucket-location" "client:s3cmd"
   if [[ $# -ne 1 ]]; then
     echo "get bucket location (s3cmd) requires bucket name"
     return 1
@@ -48,11 +63,11 @@ get_bucket_location_s3cmd() {
     return 1
   fi
   bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}')
-  export bucket_location
   return 0
 }
 
 get_bucket_location_mc() {
+  record_command "get-bucket-location" "client:mc"
   if [[ $# -ne 1 ]]; then
     echo "get bucket location (mc) requires bucket name"
     return 1
@@ -62,7 +77,7 @@ get_bucket_location_mc() {
     echo "error getting s3cmd info: $info"
     return 1
   fi
+  # shellcheck disable=SC2034
   bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}')
-  export bucket_location
   return 0
 }

tests/commands/get_bucket_ownership_controls.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+get_bucket_ownership_controls() {
+  if [[ -n "$SKIP_BUCKET_OWNERSHIP_CONTROLS" ]]; then
+    log 5 "Skipping get bucket ownership controls"
+    return 0
+  fi
+
+  record_command "get-bucket-ownership-controls" "client:s3api"
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get bucket ownership controls' command requires bucket name"
+    return 1
+  fi
+
+  if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
+    log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
+    return 1
+  fi
+
+  log 5 "Raw bucket Ownership Controls:  $raw_bucket_ownership_controls"
+  bucket_ownership_controls=$(echo "$raw_bucket_ownership_controls" | grep -v "InsecureRequestWarning")
+  return 0
+}
+
+get_object_ownership_rule() {
+  if [[ -n "$SKIP_BUCKET_OWNERSHIP_CONTROLS" ]]; then
+    log 5 "Skipping get bucket ownership controls"
+    return 0
+  fi
+
+  if [[ $# -ne 1 ]]; then
+    log 2 "'get object ownership rule' command requires bucket name"
+    return 1
+  fi
+  if ! get_bucket_ownership_controls "$1"; then
+    log 2 "error getting bucket ownership controls"
+    return 1
+  fi
+  if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then
+    log 2 "error getting object ownership rule: $object_ownership_rule"
+    return 1
+  fi
+  log 5 "object ownership rule: $object_ownership_rule"
+  return 0
+}

tests/commands/get_bucket_policy.sh

@@ -1,74 +1,116 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_bucket_policy() {
+  record_command "get-bucket-policy" "client:$1"
   if [[ $# -ne 2 ]]; then
-    echo "get bucket policy command requires command type, bucket"
+    log 2 "get bucket policy command requires command type, bucket"
     return 1
   fi
   local get_bucket_policy_result=0
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
     get_bucket_policy_aws "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
     get_bucket_policy_mc "$2" || get_bucket_policy_result=$?
   else
-    echo "command 'get bucket policy' not implemented for '$1'"
+    log 2 "command 'get bucket policy' not implemented for '$1'"
     return 1
   fi
   if [[ $get_bucket_policy_result -ne 0 ]]; then
-    echo "error getting policy: $bucket_policy"
+    log 2 "error getting policy: $bucket_policy"
     return 1
   fi
-  export bucket_policy
   return 0
 }
 
 get_bucket_policy_aws() {
+  record_command "get-bucket-policy" "client:s3api"
   if [[ $# -ne 1 ]]; then
-    echo "aws 'get bucket policy' command requires bucket"
+    log 2 "aws 'get bucket policy' command requires bucket"
     return 1
   fi
-  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || get_result=$?
+  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
   policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
   log 5 "$policy_json"
   if [[ $get_result -ne 0 ]]; then
     if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
       bucket_policy=
     else
-      echo "error getting policy: $policy_json"
+      log 2 "error getting policy: $policy_json"
       return 1
     fi
   else
     bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
   fi
-  export bucket_policy
+  return 0
+}
+
+get_bucket_policy_with_user() {
+  record_command "get-bucket-policy" "client:s3api"
+  if [[ $# -ne 3 ]]; then
+    log 2 "'get bucket policy with user' command requires bucket, username, password"
+    return 1
+  fi
+  if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
+    policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
+    bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
+  else
+    if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
+      bucket_policy=
+    else
+      log 2 "error getting policy for user $2: $policy_json"
+      return 1
+    fi
+  fi
   return 0
 }
 
 get_bucket_policy_s3cmd() {
+  record_command "get-bucket-policy" "client:s3cmd"
   if [[ $# -ne 1 ]]; then
-    echo "s3cmd 'get bucket policy' command requires bucket"
+    log 2 "s3cmd 'get bucket policy' command requires bucket"
     return 1
   fi
 
-  info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1") || get_result=$?
-  if [[ $get_result -ne 0 ]]; then
-    echo "error getting bucket policy: $info"
+  if ! info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then
+    log 2 "error getting bucket policy: $info"
     return 1
   fi
 
+  log 5 "policy info: $info"
   bucket_policy=""
   policy_brackets=false
+  # NOTE:  versitygw sends policies back in multiple lines here, direct in single line
   while IFS= read -r line; do
     if [[ $policy_brackets == false ]]; then
       policy_line=$(echo "$line" | grep 'Policy: ')
       if [[ $policy_line != "" ]]; then
-        if [[ $policy_line != *'{' ]]; then
+        if [[ $policy_line != *'{'* ]]; then
           break
         fi
-        policy_brackets=true
-        bucket_policy+="{"
+        if [[ $policy_line == *'}'* ]]; then
+          log 5 "policy on single line"
+          bucket_policy=${policy_line//Policy:/}
+          break
+        else
+          policy_brackets=true
+          bucket_policy+="{"
+        fi
       fi
     else
       bucket_policy+=$line
@@ -77,11 +119,12 @@ get_bucket_policy_s3cmd() {
       fi
     fi
   done <<< "$info"
-  export bucket_policy
+  log 5 "bucket policy: $bucket_policy"
   return 0
 }
 
 get_bucket_policy_mc() {
+  record_command "get-bucket-policy" "client:mc"
   if [[ $# -ne 1 ]]; then
     echo "aws 'get bucket policy' command requires bucket"
     return 1
@@ -91,6 +134,5 @@ get_bucket_policy_mc() {
     echo "error getting policy: $bucket_policy"
     return 1
   fi
-  export bucket_policy
   return 0
 }

tests/commands/get_bucket_tagging.sh

@@ -1,21 +1,32 @@
 #!/usr/bin/env bash
 
-# get bucket tags
-# params:  bucket
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# params:  client, bucket
 # export 'tags' on success, return 1 for error
 get_bucket_tagging() {
-  if [ $# -ne 2 ]; then
-    echo "get bucket tag command missing command type, bucket name"
-    return 1
-  fi
+  log 6 "get_bucket_tagging"
+  assert [ $# -eq 2 ]
+  record_command "get-bucket-tagging" "client:$1"
   local result
   if [[ $1 == 'aws' ]]; then
     tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
     tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
   else
-    echo "invalid command type $1"
-    return 1
+    fail "invalid command type $1"
   fi
   log 5 "Tags: $tags"
   tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
@@ -28,4 +39,27 @@ get_bucket_tagging() {
     return 1
   fi
   export tags
-}
+}
+
+get_bucket_tagging_with_user() {
+  log 6 "get_bucket_tagging_with_user"
+  if [ $# -ne 3 ]; then
+    log 2 "'get_bucket_tagging_with_user' command requires ID, key, bucket"
+    return 1
+  fi
+  record_command "get-bucket-tagging" "client:s3api"
+  local result
+  if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then
+    log 5 "tags error: $tags"
+    if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
+      export tags=
+      return 0
+    fi
+    fail "unrecognized error getting bucket tagging with user: $tags"
+    return 1
+  fi
+  log 5 "raw tags data: $tags"
+  tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
+  log 5 "modified tags data: $tags"
+  return 0
+}

tests/commands/get_bucket_versioning.sh

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_bucket_versioning() {
+  record_command "get-bucket-versioning" "client:s3api"
   if [[ $# -ne 2 ]]; then
     log 2 "put bucket versioning command requires command type, bucket name"
     return 1

tests/commands/get_object.sh

@@ -1,40 +1,84 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_object() {
+  log 6 "get_object"
+  record_command "get-object" "client:$1"
   if [ $# -ne 4 ]; then
     log 2 "get object command requires command type, bucket, key, destination"
     return 1
   fi
   local exit_code=0
-  local error
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
   else
     log 2 "'get object' command not implemented for '$1'"
     return 1
   fi
   log 5 "get object exit code: $exit_code"
   if [ $exit_code -ne 0 ]; then
-    log 2 "error getting object: $error"
+    log 2 "error getting object: $get_object_error"
     return 1
   fi
   return 0
 }
 
 get_object_with_range() {
+  record_command "get-object" "client:s3api"
   if [[ $# -ne 4 ]]; then
     log 2 "'get object with range' requires bucket, key, range, outfile"
     return 1
   fi
-  error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1) || local exit_code=$?
+  get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1) || local exit_code=$?
   if [[ $exit_code -ne 0 ]]; then
-    log 2 "error getting object with range: $error"
+    log 2 "error getting object with range: $get_object_error"
+    return 1
+  fi
+  return 0
+}
+
+get_object_with_user() {
+  log 6 "get_object_with_user"
+  record_command "get-object" "client:$1"
+  if [ $# -ne 6 ]; then
+    log 2 "'get object with user' command requires command type, bucket, key, save location, aws ID, aws secret key"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 's3' ]] || [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    log 5 "s3cmd filename: $3"
+    get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == "mc" ]]; then
+    log 5 "save location: $4"
+    get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+  else
+    log 2 "'get_object_with_user' not implemented for client '$1'"
+    return 1
+  fi
+  log 5 "get object exit code: $exit_code"
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error getting object: $get_object_error"
     return 1
   fi
   return 0

tests/commands/get_object_attributes.sh

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_object_attributes() {
+  record_command "get-object-attributes" "client:s3api"
   if [[ $# -ne 2 ]]; then
     log 2 "'get object attributes' command requires bucket, key"
     return 1
@@ -12,6 +27,5 @@ get_object_attributes() {
   fi
   attributes=$(echo "$attributes" | grep -v "InsecureRequestWarning")
   log 5 "$attributes"
-  export attributes
   return 0
 }

tests/commands/get_object_legal_hold.sh

@@ -1,15 +1,29 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_object_legal_hold() {
   if [[ $# -ne 2 ]]; then
     log 2 "'get object legal hold' command requires bucket, key"
     return 1
   fi
+  record_command "get-object-legal-hold" "client:s3api"
   legal_hold=$(aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$?
   if [[ $get_result -ne 0 ]]; then
     log 2 "error getting object legal hold: $legal_hold"
     return 1
   fi
-  export legal_hold
   return 0
 }

tests/commands/get_object_lock_configuration.sh

@@ -1,15 +1,31 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_object_lock_configuration() {
+  record_command "get-object-lock-configuration" "client:s3api"
   if [[ $# -ne 1 ]]; then
     log 2 "'get object lock configuration' command missing bucket name"
     return 1
   fi
-  lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1") || local get_result=$?
-  if [[ $get_result -ne 0 ]]; then
+  if ! lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then
     log 2 "error obtaining lock config: $lock_config"
+    # shellcheck disable=SC2034
+    get_object_lock_config_err=$lock_config
     return 1
   fi
-  export lock_config
+  lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning")
   return 0
 }

tests/commands/get_object_retention.sh

@@ -1,15 +1,30 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_object_retention() {
+  record_command "get-object-retention" "client:s3api"
   if [[ $# -ne 2 ]]; then
     log 2 "'get object retention' command requires bucket, key"
     return 1
   fi
-  retention=$(aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1) || local get_result=$?
-  if [[ $get_result -ne 0 ]]; then
+  if ! retention=$(aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then
     log 2 "error getting object retention: $retention"
+    get_object_retention_error=$retention
+    export get_object_retention_error
     return 1
   fi
-  export retention
   return 0
 }

tests/commands/get_object_tagging.sh

@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 get_object_tagging() {
+  record_command "get-object-tagging" "client:$1"
   if [ $# -ne 3 ]; then
     log 2 "get object tag command missing command type, bucket, and/or key"
     return 1

tests/commands/head_bucket.sh

@@ -1,8 +1,31 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/report.sh
+
+# params: client, bucket name
+# fail for invalid params, return
+#   0 - bucket exists
+#   1 - bucket does not exist
+#   2 - misc error
 head_bucket() {
+  log 6 "head_bucket"
+  record_command "head-bucket" "client:$1"
   if [ $# -ne 2 ]; then
-    echo "head bucket command missing command type, bucket name"
+    log 2 "'head_bucket' command requires client, bucket name"
     return 1
   fi
   local exit_code=0
@@ -13,13 +36,14 @@ head_bucket() {
   elif [[ $1 == 'mc' ]]; then
     bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
-    echo "invalid command type $1"
-    return 1
+    fail "invalid command type $1"
   fi
   if [ $exit_code -ne 0 ]; then
-    echo "error getting bucket info: $bucket_info"
-    return 1
+    if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
+      return 1
+    fi
+    log 2 "error getting bucket info: $bucket_info"
+    return 2
   fi
-  export bucket_info
   return 0
 }