Merge pull request #1151 from versity/sis/posix-tag-parsing-refactoring

fix: Reuses the ParseObjectTags method to parse the object tags in Pu…

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,27 +1,23 @@
 ---
-name: Bug report
+name: Bug Report
 about: Create a report to help us improve
-title: ''
+title: '[Bug] - <Short Description>'
 labels: bug
 assignees: ''
 
 ---
 
 **Describe the bug**
-A clear and concise description of what the bug is.
+<!-- A clear and concise description of what the bug is. -->
 
 **To Reproduce**
-Steps to reproduce the behavior.
+<!-- Steps to reproduce the behavior. -->
 
 **Expected behavior**
-A clear and concise description of what you expected to happen.
+<!-- A clear and concise description of what you expected to happen. -->
 
 **Server Version**
- output of
-```
-./versitygw -version
-uname -a
-```
+<!-- output of: './versitygw -version && uname -a'  -->
 
 **Additional context**
-Describe s3 client and version if applicable.
+<!-- Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,14 +1,14 @@
 ---
-name: Feature request
+name: Feature Request
 about: Suggest an idea for this project
-title: ''
+title: '[Feature] - <Short Description>'
 labels: enhancement
 assignees: ''
 
 ---
 
 **Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+<!-- A clear and concise description of what you want to happen. -->
 
 **Additional context**
-Add any other context or screenshots about the feature request here.
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/test_case.md

@@ -0,0 +1,33 @@
+---
+name: Test Case Request
+about: Request new test cases or additional test coverage
+title: '[Test Case] - <Short Description>'
+labels: 'testcase'
+assignees: ''
+
+---
+
+## Description
+<!-- Please provide a detailed description of the test case or test coverage request. -->
+
+## Purpose
+<!-- Explain why this test case is important and what it aims to achieve. -->
+
+## Scope
+<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
+
+## Acceptance Criteria
+<!-- List the criteria that must be met for the test case to be considered complete. -->
+
+1. 
+2. 
+3. 
+
+## Additional Context
+<!-- Add any other context or screenshots about the feature request here. -->
+
+## Resources
+<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
+
+
+**Thank you for contributing to our project!**

.github/workflows/azurite.yml

@@ -0,0 +1,37 @@
+name: azurite functional tests
+
+on: pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+              
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Set up Docker Compose
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
+
+      - name: Wait for Azurite to be ready
+        run: sleep 40
+
+      - name: Get Dependencies
+        run: |
+          go mod download
+  
+      - name: Build and Run
+        run: |
+          make
+          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
+  
+      - name: Shut down services
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/docker-bats.yml

@@ -0,0 +1,29 @@
+name: docker bats tests
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        run: |
+          cp tests/.env.docker.default tests/.env.docker
+          cp tests/.secrets.default tests/.secrets
+          # see https://github.com/versity/versitygw/issues/1034
+          docker build \
+            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
+            --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain \
+            -f tests/Dockerfile_test_bats \
+            -t bats_test .
+
+      - name: Run Docker Container
+        run: |
+          docker compose -f tests/docker-compose-bats.yml --project-directory . \
+          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/docker.yml

@@ -15,6 +15,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -43,6 +50,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
           build-args: |
             VERSION=${{ github.event.release.tag_name }}
             TIME=${{ github.event.release.published_at }}

.github/workflows/functional.yml

@@ -1,7 +1,8 @@
 name: functional tests
-on: pull_request
-jobs:
 
+on: pull_request
+
+jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
@@ -18,7 +19,7 @@ jobs:
 
     - name: Get Dependencies
       run: |
-        go get -v -t -d ./...
+        go mod download
 
     - name: Build and Run
       run: |

.github/workflows/system.yml

@@ -8,83 +8,97 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - set: 1
-            LOCAL_FOLDER: /tmp/gw1
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+          - set: "mc, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam1
-            AWS_ENDPOINT_URL: https://127.0.0.1:7070
-            RUN_SET: "s3cmd"
+            RUN_SET: "mc-non-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7070
             BACKEND: "posix"
-          - set: 2
-            LOCAL_FOLDER: /tmp/gw2
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
+          - set: "mc, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam2
-            AWS_ENDPOINT_URL: https://127.0.0.1:7071
-            RUN_SET: "s3"
+            RUN_SET: "mc-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7071
             BACKEND: "posix"
-          - set: 3
-            LOCAL_FOLDER: /tmp/gw3
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
+          - set: "REST, posix, non-static, all, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam3
-            AWS_ENDPOINT_URL: https://127.0.0.1:7072
-            RUN_SET: "s3api"
+            RUN_SET: "rest"
             RECREATE_BUCKETS: "true"
-            PORT: 7072
             BACKEND: "posix"
-          - set: 4
-            LOCAL_FOLDER: /tmp/gw4
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+          - set: "s3, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam4
-            AWS_ENDPOINT_URL: https://127.0.0.1:7073
-            RUN_SET: "mc"
+            RUN_SET: "s3-non-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7073
             BACKEND: "posix"
-          - set: 5
-            LOCAL_FOLDER: /tmp/gw5
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
+          - set: "s3, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, non-static, s3 IAM"
             IAM_TYPE: s3
-            USERS_BUCKET: versity-gwtest-iam
-            AWS_ENDPOINT_URL: https://127.0.0.1:7074
-            RUN_SET: "aws-user"
+            RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "true"
-            PORT: 7074
             BACKEND: "posix"
-          - set: 6
-            LOCAL_FOLDER: /tmp/gw6
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+          - set: "s3api, posix, bucket, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam6
-            AWS_ENDPOINT_URL: https://127.0.0.1:7075
-            RUN_SET: "aws"
+            RUN_SET: "s3api-bucket"
             RECREATE_BUCKETS: "false"
-            PORT: 7075
             BACKEND: "posix"
-          - set: 7
-            LOCAL_FOLDER: /tmp/gw7
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+          - set: "s3api, posix, multipart, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam7
-            AWS_ENDPOINT_URL: https://127.0.0.1:7076
-            RUN_SET: "aws"
+            RUN_SET: "s3api-multipart"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, object, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-object"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          # TODO fix/debug s3 gateway
+          #- set: "s3api, s3, multipart|object, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
+          #- set: "s3api, s3, policy|user, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-policy,s3api-user"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
+          - set: "s3cmd, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7076
-            BACKEND: "s3"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, non-user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-non-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -103,6 +117,8 @@ jobs:
         run: |
           git clone https://github.com/bats-core/bats-core.git
           cd bats-core && ./install.sh $HOME
+          git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
+          git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
 
       - name: Install s3cmd
         run: |
@@ -113,17 +129,22 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Build and run, posix backend
+      - name: Install xmllint (for rest)
+        run: |
+          sudo apt-get install libxml2-utils
+
+      # see https://github.com/versity/versitygw/issues/1034
+      - name: Install AWS cli
+        run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" 
+          unzip -o awscliv2.zip 
+          ./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+      - name: Build and run
         env:
-          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
-          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
-          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
-          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
-          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
           IAM_TYPE: ${{ matrix.IAM_TYPE }}
-          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
           RUN_SET: ${{ matrix.RUN_SET }}
-          PORT: ${{ matrix.PORT }}
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
@@ -131,10 +152,27 @@ jobs:
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
+          LOCAL_FOLDER: /tmp/gw
+          BUCKET_ONE_NAME: versity-gwtest-bucket-one
+          BUCKET_TWO_NAME: versity-gwtest-bucket-two
+          USERS_FOLDER: /tmp/iam
+          USERS_BUCKET: versity-gwtest-iam
+          AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          PORT: 7070
           S3CMD_CONFIG: tests/s3cfg.local.default
           MC_ALIAS: versity
           LOG_LEVEL: 4
           GOCOVERDIR: ${{ github.workspace }}/cover
+          USERNAME_ONE: ABCDEFG
+          PASSWORD_ONE: 1234567
+          USERNAME_TWO: HIJKLMN
+          PASSWORD_TWO: 8901234
+          TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
+          REMOVE_TEST_FILE_FOLDER: true
+          VERSIONING_DIR: ${{ github.workspace }}/versioning
+          COMMAND_LOG: command.log
+          TIME_LOG: time.log
+          PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -142,6 +180,7 @@ jobs:
           export AWS_REGION=us-east-1
           export AWS_ACCESS_KEY_ID_TWO=user
           export AWS_SECRET_ACCESS_KEY_TWO=pass
+          export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity
@@ -155,6 +194,9 @@ jobs:
           fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
+      - name: Time report
+        run: cat ${{ github.workspace }}/time.log
+
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

.gitignore

@@ -45,6 +45,7 @@ tests/.secrets*
 
 # IAM users files often created in testing
 users.json
+users.json.backup
 
 # env files for testing
 **/.env*
@@ -59,3 +60,10 @@ tests/!s3cfg.local.default
 
 # patches
 *.patch
+
+# grafana's local database (kept on filesystem for survival between instantiations)
+metrics-exploration/grafana_data/**
+
+# bats tools
+/tests/bats-assert
+/tests/bats-support

Dockerfile

@@ -32,4 +32,4 @@ RUN mkdir -p $SETUP_DIR
 
 COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
 
-ENTRYPOINT [ "/app/versitygw" ]
+ENTRYPOINT [ "/app/versitygw" ]

Makefile

@@ -18,6 +18,10 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 
+# docker-compose
+DCCMD=docker-compose
+DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
+
 BIN=versitygw
 
 VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -71,19 +75,19 @@ dist:
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
-	docker compose --env-file .env.dev up posix
+	$(DOCKERCOMPOSE) up posix
 
 # Creates and runs S3 gateway proxy instance in a docker container
 .PHONY: up-proxy
 up-proxy:
-	docker compose --env-file .env.dev up proxy
+	$(DOCKERCOMPOSE) up proxy
 
 # Creates and runs S3 gateway to azurite instance in a docker container
 .PHONY: up-azurite
 up-azurite:
-	docker compose --env-file .env.dev up azurite azuritegw
+	$(DOCKERCOMPOSE) up azurite azuritegw
 
 # Creates and runs both S3 gateway and proxy server instances in docker containers
 .PHONY: up-app
 up-app:
-	docker compose --env-file .env.dev up
+	$(DOCKERCOMPOSE) up

README.md

@@ -40,7 +40,7 @@ Versity Gateway, a simple to use tool for seamless inline translation between AW
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
 
-The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versity’s open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.  
 
 The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
 

auth/acl.go

@@ -17,6 +17,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"strings"
@@ -28,27 +29,156 @@ import (
 )
 
 type ACL struct {
-	ACL      types.BucketCannedACL
 	Owner    string
 	Grantees []Grantee
 }
 
 type Grantee struct {
-	Permission types.Permission
+	Permission Permission
 	Access     string
+	Type       types.Type
 }
 
 type GetBucketAclOutput struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
 	Owner             *types.Owner
 	AccessControlList AccessControlList
 }
 
-type AccessControlList struct {
-	Grants []types.Grant `xml:"Grant"`
+type PutBucketAclInput struct {
+	Bucket              *string
+	ACL                 types.BucketCannedACL
+	AccessControlPolicy *AccessControlPolicy
+	GrantFullControl    *string
+	GrantRead           *string
+	GrantReadACP        *string
+	GrantWrite          *string
+	GrantWriteACP       *string
 }
+
 type AccessControlPolicy struct {
 	AccessControlList AccessControlList `xml:"AccessControlList"`
-	Owner             types.Owner
+	Owner             *types.Owner
+}
+
+func (acp *AccessControlPolicy) Validate() error {
+	if !acp.AccessControlList.isValid() {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner can't be nil
+	if acp.Owner == nil {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner ID can't be empty
+	if acp.Owner.ID == nil || *acp.Owner.ID == "" {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	return nil
+}
+
+type AccessControlList struct {
+	Grants []Grant `xml:"Grant"`
+}
+
+// Validates the AccessControlList
+func (acl *AccessControlList) isValid() bool {
+	for _, el := range acl.Grants {
+		if !el.isValid() {
+			return false
+		}
+	}
+
+	return true
+}
+
+type Permission string
+
+const (
+	PermissionFullControl Permission = "FULL_CONTROL"
+	PermissionWrite       Permission = "WRITE"
+	PermissionWriteAcp    Permission = "WRITE_ACP"
+	PermissionRead        Permission = "READ"
+	PermissionReadAcp     Permission = "READ_ACP"
+)
+
+// Check if the permission is valid
+func (p Permission) isValid() bool {
+	return p == PermissionFullControl ||
+		p == PermissionRead ||
+		p == PermissionReadAcp ||
+		p == PermissionWrite ||
+		p == PermissionWriteAcp
+}
+
+type Grant struct {
+	Grantee    *Grt       `xml:"Grantee"`
+	Permission Permission `xml:"Permission"`
+}
+
+// Checks if Grant is valid
+func (g *Grant) isValid() bool {
+	return g.Permission.isValid() && g.Grantee.isValid()
+}
+
+type Grt struct {
+	XMLNS string     `xml:"xmlns:xsi,attr"`
+	Type  types.Type `xml:"xsi:type,attr"`
+	ID    string     `xml:"ID"`
+}
+
+// Custom Unmarshalling for Grt to parse xsi:type properly
+func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Iterate through the XML tokens to process the attributes
+	for _, attr := range start.Attr {
+		// Check if the attribute is xsi:type and belongs to the xsi namespace
+		if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
+			g.Type = types.Type(attr.Value)
+		}
+		// Handle xmlns:xsi
+		if attr.Name.Local == "xmlns:xsi" {
+			g.XMLNS = attr.Value
+		}
+	}
+
+	// Decode the inner XML elements like ID
+	for {
+		t, err := d.Token()
+		if err != nil {
+			return err
+		}
+
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "ID" {
+				if err := d.DecodeElement(&g.ID, &se); err != nil {
+					return err
+				}
+			}
+		case xml.EndElement:
+			if se.Name.Local == start.Name.Local {
+				return nil
+			}
+		}
+	}
+}
+
+// Validates Grt
+func (g *Grt) isValid() bool {
+	// Validate the Type
+	// Only these 2 types are supported in the gateway
+	if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
+		return false
+	}
+
+	// The ID prop shouldn't be empty
+	if g.ID == "" {
+		return false
+	}
+
+	return true
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -63,17 +193,35 @@ func ParseACL(data []byte) (ACL, error) {
 	return acl, nil
 }
 
-func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
+func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
+	grants := []Grant{}
+
+	if len(data) == 0 {
+		return GetBucketAclOutput{
+			Owner: &types.Owner{
+				ID: &owner,
+			},
+			AccessControlList: AccessControlList{
+				Grants: grants,
+			},
+		}, nil
+	}
+
 	var acl ACL
 	if err := json.Unmarshal(data, &acl); err != nil {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
-	grants := []types.Grant{}
-
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
-		grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
+		grants = append(grants, Grant{
+			Grantee: &Grt{
+				XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
+				ID:    acs,
+				Type:  elem.Type,
+			},
+			Permission: elem.Permission,
+		})
 	}
 
 	return GetBucketAclOutput{
@@ -86,20 +234,43 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 	}, nil
 }
 
-func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
+func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
 	if input == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
-	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
-		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+
+	defaultGrantees := []Grantee{
+		{
+			Permission: PermissionFullControl,
+			Access:     acl.Owner,
+			Type:       types.TypeCanonicalUser,
+		},
 	}
 
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
-		acl.ACL = input.ACL
-		acl.Grantees = []Grantee{}
+		switch input.ACL {
+		case types.BucketCannedACLPublicRead:
+			defaultGrantees = append(defaultGrantees, Grantee{
+				Permission: PermissionRead,
+				Access:     "all-users",
+				Type:       types.TypeGroup,
+			})
+		case types.BucketCannedACLPublicReadWrite:
+			defaultGrantees = append(defaultGrantees, []Grantee{
+				{
+					Permission: PermissionRead,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+				{
+					Permission: PermissionWrite,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+			}...)
+		}
 	} else {
-		grantees := []Grantee{}
 		accs := []string{}
 
 		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -108,45 +279,71 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionFullControl,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantRead != nil && *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionRead,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionReadAcp,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantWrite != nil && *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionWrite,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionWriteAcp,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 
 			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
 		} else {
 			cache := make(map[string]bool)
-			for _, grt := range input.AccessControlPolicy.Grants {
-				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
+			for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
+				if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
 					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 				}
-				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
-				if _, ok := cache[*grt.Grantee.ID]; !ok {
-					cache[*grt.Grantee.ID] = true
-					accs = append(accs, *grt.Grantee.ID)
+
+				access := grt.Grantee.ID
+				defaultGrantees = append(defaultGrantees, Grantee{
+					Access:     access,
+					Permission: grt.Permission,
+					Type:       types.TypeCanonicalUser,
+				})
+				if _, ok := cache[access]; !ok {
+					cache[access] = true
+					accs = append(accs, access)
 				}
 			}
 		}
@@ -159,11 +356,10 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		if len(accList) > 0 {
 			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
 		}
-
-		acl.Grantees = grantees
-		acl.ACL = ""
 	}
 
+	acl.Grantees = defaultGrantees
+
 	result, err := json.Marshal(acl)
 	if err != nil {
 		return nil, err
@@ -182,8 +378,8 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
 				result = append(result, acc)
 				continue
 			}
-			if err == ErrNotSupported {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
+				return nil, err
 			}
 			return nil, fmt.Errorf("check user account: %w", err)
 		}
@@ -206,35 +402,34 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func verifyACL(acl ACL, access string, permission types.Permission) error {
-	if acl.ACL != "" {
-		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
+func verifyACL(acl ACL, access string, permission Permission) error {
+	grantee := Grantee{
+		Access:     access,
+		Permission: permission,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeFullCtrl := Grantee{
+		Access:     access,
+		Permission: PermissionFullControl,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeAllUsers := Grantee{
+		Access:     "all-users",
+		Permission: permission,
+		Type:       types.TypeGroup,
+	}
 
+	isFound := false
+
+	for _, grt := range acl.Grantees {
+		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
+			isFound = true
+			break
+		}
+	}
+
+	if isFound {
 		return nil
-	} else {
-		if len(acl.Grantees) == 0 {
-			return nil
-		}
-		grantee := Grantee{Access: access, Permission: permission}
-		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
-
-		isFound := false
-
-		for _, grt := range acl.Grantees {
-			if grt == grantee || grt == granteeFullCtrl {
-				isFound = true
-				break
-			}
-		}
-
-		if isFound {
-			return nil
-		}
 	}
 
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -274,7 +469,7 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 
 type AccessOptions struct {
 	Acl           ACL
-	AclPermission types.Permission
+	AclPermission Permission
 	IsRoot        bool
 	Acc           Account
 	Bucket        string
@@ -285,7 +480,7 @@ type AccessOptions struct {
 
 func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
 	if opts.Readonly {
-		if opts.AclPermission == types.PermissionWrite || opts.AclPermission == types.PermissionWriteAcp {
+		if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
 		}
 	}
@@ -295,23 +490,16 @@ func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) e
 	if opts.Acc.Role == RoleAdmin {
 		return nil
 	}
-	if opts.Acc.Access == opts.Acl.Owner {
-		return nil
-	}
 
 	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
-	if policyErr != nil && !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-		return policyErr
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
 	}
 
-	// If bucket policy is not set and the ACL is default, only the owner has access
-	if errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	if err := VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
-		return err
-	}
 	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
 		return err
 	}
@@ -350,7 +538,7 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource
 
 	if err := VerifyAccess(ctx, be, AccessOptions{
 		Acl:           srcBucketAcl,
-		AclPermission: types.PermissionRead,
+		AclPermission: PermissionRead,
 		IsRoot:        opts.IsRoot,
 		Acc:           opts.Acc,
 		Bucket:        srcBucket,

auth/bucket_policy.go

@@ -48,18 +48,19 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 }
 
 func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	var isAllowed bool
 	for _, statement := range bp.Statement {
 		if statement.findMatch(principal, action, resource) {
 			switch statement.Effect {
 			case BucketPolicyAccessTypeAllow:
-				return true
+				isAllowed = true
 			case BucketPolicyAccessTypeDeny:
 				return false
 			}
 		}
 	}
 
-	return false
+	return isAllowed
 }
 
 type BucketPolicyItem struct {
@@ -134,11 +135,6 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 }
 
 func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
-	// If bucket policy is not set
-	if policy == nil {
-		return nil
-	}
-
 	var bucketPolicy BucketPolicy
 	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
 		return err

auth/bucket_policy_actions.go

@@ -36,6 +36,7 @@ const (
 	ListBucketMultipartUploadsAction       Action = "s3:ListBucketMultipartUploads"
 	PutObjectAction                        Action = "s3:PutObject"
 	GetObjectAction                        Action = "s3:GetObject"
+	GetObjectVersionAction                 Action = "s3:GetObjectVersion"
 	DeleteObjectAction                     Action = "s3:DeleteObject"
 	GetObjectAclAction                     Action = "s3:GetObjectAcl"
 	GetObjectAttributesAction              Action = "s3:GetObjectAttributes"
@@ -55,6 +56,8 @@ const (
 	GetObjectRetentionAction               Action = "s3:GetObjectRetention"
 	PutObjectRetentionAction               Action = "s3:PutObjectRetention"
 	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
+	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
+	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
 	AllActions                             Action = "s3:*"
 )
 
@@ -73,6 +76,7 @@ var supportedActionList = map[Action]struct{}{
 	ListBucketMultipartUploadsAction:       {},
 	PutObjectAction:                        {},
 	GetObjectAction:                        {},
+	GetObjectVersionAction:                 {},
 	DeleteObjectAction:                     {},
 	GetObjectAclAction:                     {},
 	GetObjectAttributesAction:              {},
@@ -91,6 +95,8 @@ var supportedActionList = map[Action]struct{}{
 	GetObjectRetentionAction:               {},
 	PutObjectRetentionAction:               {},
 	BypassGovernanceRetentionAction:        {},
+	PutBucketOwnershipControlsAction:       {},
+	GetBucketOwnershipControlsAction:       {},
 	AllActions:                             {},
 }
 
@@ -99,6 +105,7 @@ var supportedObjectActionList = map[Action]struct{}{
 	ListMultipartUploadPartsAction:  {},
 	PutObjectAction:                 {},
 	GetObjectAction:                 {},
+	GetObjectVersionAction:          {},
 	DeleteObjectAction:              {},
 	GetObjectAclAction:              {},
 	GetObjectAttributesAction:       {},

auth/bucket_policy_resources.go

@@ -102,21 +102,45 @@ func (r Resources) Validate(bucket string) error {
 
 func (r Resources) FindMatch(resource string) bool {
 	for res := range r {
-		if strings.HasSuffix(res, "*") {
-			pattern := strings.TrimSuffix(res, "*")
-			if strings.HasPrefix(resource, pattern) {
-				return true
-			}
-		} else {
-			if res == resource {
-				return true
-			}
+		if r.Match(res, resource) {
+			return true
 		}
 	}
 
 	return false
 }
 
+// Match checks if the input string matches the given pattern with wildcards (`*`, `?`).
+// - `?` matches exactly one occurrence of any character.
+// - `*` matches arbitrary many (including zero) occurrences of any character.
+func (r Resources) Match(pattern, input string) bool {
+	pIdx, sIdx := 0, 0
+	starIdx, matchIdx := -1, 0
+
+	for sIdx < len(input) {
+		if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
+			sIdx++
+			pIdx++
+		} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
+			starIdx = pIdx
+			matchIdx = sIdx
+			pIdx++
+		} else if starIdx != -1 {
+			pIdx = starIdx + 1
+			matchIdx++
+			sIdx = matchIdx
+		} else {
+			return false
+		}
+	}
+
+	for pIdx < len(pattern) && pattern[pIdx] == '*' {
+		pIdx++
+	}
+
+	return pIdx == len(pattern)
+}
+
 // Checks the resource to have arn prefix and not starting with /
 func isValidResource(rc string) (isValid bool, pattern string) {
 	if !strings.HasPrefix(rc, ResourceArnPrefix) {

auth/bucket_policy_resources_test.go

@@ -0,0 +1,182 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestUnmarshalJSON(t *testing.T) {
+	var r Resources
+
+	cases := []struct {
+		input    string
+		expected int
+		wantErr  bool
+	}{
+		{`"arn:aws:s3:::my-bucket/*"`, 1, false},
+		{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
+		{`""`, 0, true},
+		{`[]`, 0, true},
+		{`["invalid-bucket"]`, 0, true},
+	}
+
+	for _, tc := range cases {
+		r = Resources{}
+		err := json.Unmarshal([]byte(tc.input), &r)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+		if len(r) != tc.expected {
+			t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
+		}
+	}
+}
+
+func TestAdd(t *testing.T) {
+	r := Resources{}
+
+	cases := []struct {
+		input   string
+		wantErr bool
+	}{
+		{"arn:aws:s3:::valid-bucket/*", false},
+		{"arn:aws:s3:::valid-bucket/object", false},
+		{"invalid-bucket/*", true},
+		{"/invalid-start", true},
+	}
+
+	for _, tc := range cases {
+		err := r.Add(tc.input)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+	}
+}
+
+func TestContainsObjectPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, true},
+		{[]string{"arn:aws:s3:::my-bucket"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsObjectPattern() != tc.expected {
+			t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestContainsBucketPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsBucketPattern() != tc.expected {
+			t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestValidate(t *testing.T) {
+	cases := []struct {
+		resources []string
+		bucket    string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
+		{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
+		{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if (r.Validate(tc.bucket) == nil) != tc.expected {
+			t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
+		}
+	}
+}
+
+func TestFindMatch(t *testing.T) {
+	cases := []struct {
+		resources []string
+		input     string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
+		{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.FindMatch(tc.input) != tc.expected {
+			t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
+		}
+	}
+}
+
+func TestMatch(t *testing.T) {
+	r := Resources{}
+	cases := []struct {
+		pattern  string
+		input    string
+		expected bool
+	}{
+		{"my-bucket/*", "my-bucket/object", true},
+		{"my-bucket/?bject", "my-bucket/object", true},
+		{"my-bucket/*", "other-bucket/object", false},
+		{"*", "any-bucket/object", true},
+		{"my-bucket/*", "my-bucket/subdir/object", true},
+		{"my-bucket/*", "other-bucket", false},
+		{"my-bucket/*/*", "my-bucket/hello", false},
+		{"my-bucket/*/*", "my-bucket/hello/world", true},
+		{"foo/???/bar", "foo/qux/bar", true},
+		{"foo/???/bar", "foo/quxx/bar", false},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
+	}
+	for _, tc := range cases {
+		if r.Match(tc.pattern, tc.input) != tc.expected {
+			t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
+		}
+	}
+}

auth/iam.go

@@ -28,6 +28,19 @@ const (
 	RoleUserPlus Role = "userplus"
 )
 
+func (r Role) IsValid() bool {
+	switch r {
+	case RoleAdmin:
+		return true
+	case RoleUser:
+		return true
+	case RoleUserPlus:
+		return true
+	default:
+		return false
+	}
+}
+
 // Account is a gateway IAM account
 type Account struct {
 	Access  string `json:"access"`
@@ -37,6 +50,10 @@ type Account struct {
 	GroupID int    `json:"groupID"`
 }
 
+type ListUserAccountsResult struct {
+	Accounts []Account
+}
+
 // Mutable props, which could be changed when updating an IAM account
 type MutableProps struct {
 	Secret  *string `json:"secret"`
@@ -76,6 +93,7 @@ var (
 )
 
 type Opts struct {
+	RootAccount            Account
 	Dir                    string
 	LDAPServerURL          string
 	LDAPBindDN             string
@@ -106,6 +124,12 @@ type Opts struct {
 	CacheDisable           bool
 	CacheTTL               int
 	CachePrune             int
+	IpaHost                string
+	IpaVaultName           string
+	IpaUser                string
+	IpaPassword            string
+	IpaInsecure            bool
+	IpaDebug               bool
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -114,23 +138,26 @@ func New(o *Opts) (IAMService, error) {
 
 	switch {
 	case o.Dir != "":
-		svc, err = NewInternal(o.Dir)
+		svc, err = NewInternal(o.RootAccount, o.Dir)
 		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
-		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+		svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
 			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
 			o.LDAPGroupIdAtr, o.LDAPObjClasses)
 		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
 	case o.S3Endpoint != "":
-		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+		svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
 			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
 		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
 			o.S3Endpoint, o.S3Bucket)
 	case o.VaultEndpointURL != "":
-		svc, err = NewVaultIAMService(o.VaultEndpointURL, o.VaultSecretStoragePath,
+		svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
 			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
 			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
 		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
+	case o.IpaHost != "":
+		svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure, o.IpaDebug)
+		fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")

auth/iam_internal.go

@@ -40,7 +40,8 @@ type IAMServiceInternal struct {
 	// IAM service. All account updates should be sent to a single
 	// gateway instance if possible.
 	sync.RWMutex
-	dir string
+	dir     string
+	rootAcc Account
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -54,9 +55,10 @@ type iAMConfig struct {
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(dir string) (*IAMServiceInternal, error) {
+func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		dir: dir,
+		dir:     dir,
+		rootAcc: rootAcc,
 	}
 
 	err := i.initIAM()
@@ -70,6 +72,10 @@ func NewInternal(dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	if account.Access == s.rootAcc.Access {
+		return ErrUserExists
+	}
+
 	s.Lock()
 	defer s.Unlock()
 
@@ -97,6 +103,10 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
 	s.RLock()
 	defer s.RUnlock()
 

auth/iam_ipa.go

@@ -0,0 +1,446 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+const IpaVersion = "2.254"
+
+type IpaIAMService struct {
+	client          http.Client
+	id              int
+	version         string
+	host            string
+	vaultName       string
+	username        string
+	password        string
+	kraTransportKey *rsa.PublicKey
+	debug           bool
+	rootAcc         Account
+}
+
+var _ IAMService = &IpaIAMService{}
+
+func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
+
+	ipa := IpaIAMService{
+		id:        0,
+		version:   IpaVersion,
+		host:      host,
+		vaultName: vaultName,
+		username:  username,
+		password:  password,
+		debug:     debug,
+		rootAcc:   rootAcc,
+	}
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		// this should never happen
+		return nil, fmt.Errorf("cookie jar creation: %w", err)
+	}
+
+	mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
+	tr := &http.Transport{
+		TLSClientConfig: mTLSConfig,
+	}
+	ipa.client = http.Client{Jar: jar, Transport: tr}
+
+	err = ipa.login()
+	if err != nil {
+		return nil, fmt.Errorf("ipa login failed: %w", err)
+	}
+
+	req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
+	if err != nil {
+		return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
+	}
+	vaultConfig := struct {
+		Kra_Server_Server             []string
+		Transport_Cert                Base64EncodedWrapped
+		Wrapping_default_algorithm    string
+		Wrapping_supported_algorithms []string
+	}{}
+	err = ipa.rpc(req, &vaultConfig)
+	if err != nil {
+		return nil, fmt.Errorf("ipa vault config: %w", err)
+	}
+
+	cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
+	if err != nil {
+		return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
+	}
+
+	ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
+
+	isSupported := false
+	for _, algo := range vaultConfig.Wrapping_supported_algorithms {
+		if algo == "aes-128-cbc" {
+			isSupported = true
+			break
+		}
+	}
+
+	if !isSupported {
+		return nil,
+			fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
+				vaultConfig.Wrapping_supported_algorithms)
+	}
+	return &ipa, nil
+}
+
+func (ipa *IpaIAMService) CreateAccount(account Account) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ipa.rootAcc.Access {
+		return ipa.rootAcc, nil
+	}
+
+	req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa user_show: %w", err)
+	}
+
+	userResult := struct {
+		Gidnumber []string
+		Uidnumber []string
+	}{}
+
+	err = ipa.rpc(req, &userResult)
+	if err != nil {
+		return Account{}, err
+	}
+
+	uid, err := strconv.Atoi(userResult.Uidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
+	}
+	gid, err := strconv.Atoi(userResult.Gidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
+	}
+
+	account := Account{
+		Access:  access,
+		Role:    RoleUser,
+		UserID:  uid,
+		GroupID: gid,
+	}
+
+	session_key := make([]byte, 16)
+
+	_, err = rand.Read(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot generate session key: %w", err)
+	}
+
+	encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
+	}
+
+	req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
+		map[string]any{"username": access,
+			"session_key":   Base64EncodedWrapped(encryptedKey),
+			"wrapping_algo": "aes-128-cbc"})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
+	}
+
+	data := struct {
+		Vault_data Base64EncodedWrapped
+		Nonce      Base64EncodedWrapped
+	}{}
+
+	err = ipa.rpc(req, &data)
+	if err != nil {
+		return account, err
+	}
+
+	aes, err := aes.NewCipher(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
+	}
+	cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
+	cbc.CryptBlocks(data.Vault_data, data.Vault_data)
+	secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
+	}
+
+	secret := struct {
+		Data Base64Encoded
+	}{}
+	json.Unmarshal(secretUnpaddedJson, &secret)
+	account.Secret = string(secret.Data)
+
+	return account, nil
+}
+
+func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
+	return []Account{}, fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) Shutdown() error {
+	return nil
+}
+
+// Implementation
+
+func (ipa *IpaIAMService) login() error {
+	form := url.Values{}
+	form.Set("user", ipa.username)
+	form.Set("password", ipa.password)
+
+	req, err := http.NewRequest(
+		"POST",
+		fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
+		strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := ipa.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode == 401 {
+		return errors.New("cannot login to FreeIPA: invalid credentials")
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+type rpcRequest = string
+
+type rpcResponse struct {
+	Result    json.RawMessage
+	Principal string
+	Id        int
+	Version   string
+}
+
+func (p rpcResponse) String() string {
+	return string(p.Result)
+}
+
+var errRpc = errors.New("IPA RPC error")
+
+func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
+	err := ipa.login()
+	if err != nil {
+		return err
+	}
+
+	res, err := ipa.rpcInternal(req)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(res.Result, value)
+}
+
+func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
+	httpReq, err := http.NewRequest("POST",
+		fmt.Sprintf("%s/ipa/session/json", ipa.host),
+		strings.NewReader(req))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	ipa.log(fmt.Sprintf("%v", req))
+	httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	httpResp, err := ipa.client.Do(httpReq)
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	bytes, err := io.ReadAll(httpResp.Body)
+	ipa.log(string(bytes))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	result := struct {
+		Result struct {
+			Json    json.RawMessage `json:"result"`
+			Value   string          `json:"value"`
+			Summary any             `json:"summary"`
+		} `json:"result"`
+		Error     json.RawMessage `json:"error"`
+		Id        int             `json:"id"`
+		Principal string          `json:"principal"`
+		Version   string          `json:"version"`
+	}{}
+
+	err = json.Unmarshal(bytes, &result)
+	if err != nil {
+		return rpcResponse{}, err
+	}
+	if string(result.Error) != "null" {
+		return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
+	}
+
+	return rpcResponse{
+		Result:    result.Result.Json,
+		Principal: result.Principal,
+		Id:        result.Id,
+		Version:   result.Version,
+	}, nil
+}
+
+func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
+
+	id := ipa.id
+	ipa.id++
+
+	dict["version"] = ipa.version
+
+	jmethod, errMethod := json.Marshal(method)
+	jargs, errArgs := json.Marshal(args)
+	jdict, errDict := json.Marshal(dict)
+
+	err := errors.Join(errMethod, errArgs, errDict)
+	if err != nil {
+		return "", fmt.Errorf("ipa request invalid: %w", err)
+	}
+
+	request := map[string]interface{}{
+		"id":     id,
+		"method": json.RawMessage(jmethod),
+		"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
+	}
+
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	return string(requestJSON), nil
+}
+
+// pkcs7Unpad validates and unpads data from the given bytes slice.
+// The returned value will be 1 to n bytes smaller depending on the
+// amount of padding, where n is the block size.
+func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
+	if blocksize <= 0 {
+		return nil, errors.New("invalid blocksize")
+	}
+	if len(b) == 0 {
+		return nil, errors.New("invalid PKCS7 data (empty or not padded)")
+	}
+	if len(b)%blocksize != 0 {
+		return nil, errors.New("invalid padding on input")
+	}
+	c := b[len(b)-1]
+	n := int(c)
+	if n == 0 || n > len(b) {
+		return nil, errors.New("invalid padding on input")
+	}
+	for i := 0; i < n; i++ {
+		if b[len(b)-n+i] != c {
+			return nil, errors.New("invalid padding on input")
+		}
+	}
+	return b[:len(b)-n], nil
+}
+
+/*
+e.g.
+
+	"value" {
+		"__base64__": "aGVsbG93b3JsZAo="
+	 }
+*/
+type Base64EncodedWrapped []byte
+
+func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{}
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
+	return err
+}
+
+func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{Base64: base64.StdEncoding.EncodeToString(*b)}
+	return json.Marshal(intermediate)
+}
+
+/*
+e.g.
+
+	"value": "aGVsbG93b3JsZAo="
+*/
+type Base64Encoded []byte
+
+func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
+	var intermediate string
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate)
+	return err
+}
+
+func (ipa *IpaIAMService) log(msg string) {
+	if ipa.debug {
+		log.Println(msg)
+	}
+}

auth/iam_ldap.go

@@ -31,11 +31,12 @@ type LdapIAMService struct {
 	roleAtr    string
 	groupIdAtr string
 	userIdAtr  string
+	rootAcc    Account
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
+func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
 	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
 		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
@@ -58,10 +59,14 @@ func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userI
 		roleAtr:    roleAtr,
 		userIdAtr:  userIdAtr,
 		groupIdAtr: groupIdAtr,
+		rootAcc:    rootAcc,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
+	if ld.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
 	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
@@ -79,6 +84,9 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
 }
 
 func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ld.rootAcc.Access {
+		return ld.rootAcc, nil
+	}
 	searchRequest := ldap.NewSearchRequest(
 		ld.queryBase,
 		ldap.ScopeWholeSubtree,

auth/iam_s3_object.go

@@ -57,12 +57,13 @@ type IAMServiceS3 struct {
 	endpoint      string
 	sslSkipVerify bool
 	debug         bool
+	rootAcc       Account
 	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}
 
-func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
 	if access == "" {
 		return nil, fmt.Errorf("must provide s3 IAM service access key")
 	}
@@ -87,6 +88,7 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 		endpoint:      endpoint,
 		sslSkipVerify: sslSkipVerify,
 		debug:         debug,
+		rootAcc:       rootAcc,
 	}
 
 	cfg, err := i.getConfig()
@@ -106,6 +108,10 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 }
 
 func (s *IAMServiceS3) CreateAccount(account Account) error {
+	if s.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+
 	s.Lock()
 	defer s.Unlock()
 
@@ -124,6 +130,10 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 }
 
 func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
 	s.RLock()
 	defer s.RUnlock()
 
@@ -242,7 +252,7 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
 	})
 	if err != nil {
 		// if the error is object not exists,
-		// init empty accounts stuct and return that
+		// init empty accounts struct and return that
 		var nsk *types.NoSuchKey
 		if errors.As(err, &nsk) {
 			return iAMConfig{AccessAccounts: map[string]Account{}}, nil

auth/iam_single.go

@@ -15,7 +15,7 @@
 package auth
 
 import (
-	"errors"
+	"github.com/versity/versitygw/s3err"
 )
 
 // IAMServiceSingle manages the single tenant (root-only) IAM service
@@ -23,31 +23,29 @@ type IAMServiceSingle struct{}
 
 var _ IAMService = &IAMServiceSingle{}
 
-var ErrNotSupported = errors.New("method is not supported")
-
 // CreateAccount not valid in single tenant mode
 func (IAMServiceSingle) CreateAccount(account Account) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // GetUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	return Account{}, ErrNoSuchUser
+	return Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // UpdateUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
-	return ErrNotSupported
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // ListUserAccounts no accounts in single tenant mode
 func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
-	return []Account{}, nil
+	return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
 }
 
 // Shutdown graceful termination of service

auth/iam_vault.go

@@ -30,11 +30,12 @@ type VaultIAMService struct {
 	client            *vault.Client
 	reqOpts           []vault.RequestOption
 	secretStoragePath string
+	rootAcc           Account
 }
 
 var _ IAMService = &VaultIAMService{}
 
-func NewVaultIAMService(endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
+func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
 	opts := []vault.ClientOption{
 		vault.WithAddress(endpoint),
 		// set request timeout to 10 secs
@@ -46,7 +47,7 @@ func NewVaultIAMService(endpoint, secretStoragePath, mountPath, rootToken, roleI
 		tls.ServerCertificate.FromBytes = []byte(serverCert)
 		if clientCert != "" {
 			if clientCertKey == "" {
-				return nil, fmt.Errorf("client certificate and client certificate should both be specified")
+				return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
 			}
 
 			tls.ClientCertificate.FromBytes = []byte(clientCert)
@@ -100,10 +101,14 @@ func NewVaultIAMService(endpoint, secretStoragePath, mountPath, rootToken, roleI
 		client:            client,
 		reqOpts:           reqOpts,
 		secretStoragePath: secretStoragePath,
+		rootAcc:           rootAcc,
 	}, nil
 }
 
 func (vt *VaultIAMService) CreateAccount(account Account) error {
+	if vt.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
 		Data: map[string]any{
@@ -125,6 +130,9 @@ func (vt *VaultIAMService) CreateAccount(account Account) error {
 }
 
 func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
+	if vt.rootAcc.Access == access {
+		return vt.rootAcc, nil
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
 	cancel()

auth/object_lock.go

@@ -25,6 +25,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
 type BucketLockConfig struct {
@@ -92,12 +93,12 @@ func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfigur
 }
 
 func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
-	var retention types.ObjectLockRetention
+	var retention s3response.PutObjectRetentionInput
 	if err := xml.Unmarshal(input, &retention); err != nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
-	if retention.RetainUntilDate == nil || retention.RetainUntilDate.Before(time.Now()) {
+	if retention.RetainUntilDate.Before(time.Now()) {
 		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
 	}
 	switch retention.Mode {
@@ -135,7 +136,7 @@ func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
 	}
 }
 
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, bypass bool, be backend.Backend) error {
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
 	data, err := be.GetObjectLockConfiguration(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -171,8 +172,15 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 	}
 
 	for _, obj := range objects {
+		var key, versionId string
+		if obj.Key != nil {
+			key = *obj.Key
+		}
+		if obj.VersionId != nil {
+			versionId = *obj.VersionId
+		}
 		checkRetention := true
-		retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
+		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
 		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
 			continue
 		}
@@ -203,7 +211,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 							if err != nil {
 								return err
 							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
+							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
 							if err != nil {
 								return s3err.GetAPIError(s3err.ErrObjectLocked)
 							}
@@ -217,8 +225,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 
 		checkLegalHold := true
 
-		status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
+		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
 		if err != nil {
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+				continue
+			}
 			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
 				checkLegalHold = false
 			} else {
@@ -243,7 +254,7 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
 					if err != nil {
 						return err
 					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
+					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
 					if err != nil {
 						return s3err.GetAPIError(s3err.ErrObjectLocked)
 					}

backend/backend.go

@@ -18,9 +18,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 	"github.com/versity/versitygw/s3select"
@@ -32,40 +32,43 @@ type Backend interface {
 	Shutdown()
 
 	// bucket operations
-	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
 	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
 	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
-	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
-	PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
-	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	DeleteBucket(_ context.Context, bucket string) error
+	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
+	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
+	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
+	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
+	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
 
 	// multipart operations
-	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
-	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
-	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// standard object operations
-	PutObject(context.Context, *s3.PutObjectInput) (string, error)
+	PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
-	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
-	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
-	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
-	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
+	CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
 	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
-	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
+	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
 
 	// special case object operations
 	RestoreObject(context.Context, *s3.RestoreObjectInput) error
@@ -90,7 +93,7 @@ type Backend interface {
 	GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
 
 	// non AWS actions
-	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
 	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
@@ -105,7 +108,7 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -120,14 +123,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
 func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
+func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -138,9 +141,18 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
 func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -154,39 +166,39 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
 func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
 	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+	return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
+	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
-	return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+	return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -213,8 +225,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
 	}
 }
 
-func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
@@ -256,7 +268,7 @@ func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object,
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {

backend/common.go

@@ -18,7 +18,8 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"net/http"
+	"io"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -28,9 +29,13 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
-var (
-	// RFC3339TimeFormat RFC3339 time format
-	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+const (
+	// this is the media type for directories in AWS and Nextcloud
+	DirContentType     = "application/x-directory"
+	DefaultContentType = "binary/octet-stream"
+
+	// this is the minimum allowed size for mp parts
+	MinPartSize = 5 * 1024 * 1024
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -47,64 +52,187 @@ func (d ByObjectName) Len() int           { return len(d) }
 func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
 
-func GetStringPtr(s string) *string {
-	return &s
+func GetPtrFromString(str string) *string {
+	if str == "" {
+		return nil
+	}
+	return &str
+}
+
+func GetStringFromPtr(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
 }
 
 func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
 
+func TrimEtag(etag *string) *string {
+	if etag == nil {
+		return nil
+	}
+
+	return GetPtrFromString(strings.Trim(*etag, "\""))
+}
+
 var (
-	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidRange           = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
 )
 
-// ParseRange parses input range header and returns startoffset, length, and
-// error. If no endoffset specified, then length is set to -1.
-func ParseRange(size int64, acceptRange string) (int64, int64, error) {
+// ParseGetObjectRange parses input range header and returns startoffset, length, isValid
+// and error. If no endoffset specified, then length is set to the object size
+// for invalid inputs, it returns no error, but isValid=false
+// `InvalidRange` error is returnd, only if startoffset is greater than the object size
+func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
+	if acceptRange == "" {
+		return 0, size, false, nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) != 2 {
+		return 0, size, false, nil
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, size, false, nil
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) != 2 {
+		return 0, size, false, nil
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, size, false, nil
+	}
+
+	if startOffset >= size {
+		return 0, 0, false, errInvalidRange
+	}
+
+	if bRange[1] == "" {
+		return startOffset, size - startOffset, true, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, size, false, nil
+	}
+
+	if endOffset < startOffset {
+		return 0, size, false, nil
+	}
+
+	if endOffset >= size {
+		return startOffset, size - startOffset, true, nil
+	}
+
+	return startOffset, endOffset - startOffset + 1, true, nil
+}
+
+// ParseCopySourceRange parses input range header and returns startoffset, length
+// and error. If no endoffset specified, then length is set to the object size
+func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
 		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
 
-	if len(rangeKv) < 2 {
-		return 0, 0, errInvalidRange
+	if len(rangeKv) != 2 {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 1 || len(bRange) > 2 {
-		return 0, 0, errInvalidRange
+	if len(bRange) != 2 {
+		return 0, 0, errInvalidCopySourceRange
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
-	endOffset := int64(-1)
-	if len(bRange) == 1 || bRange[1] == "" {
-		return startOffset, endOffset, nil
+	if startOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
-	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
+	if bRange[1] == "" {
+		return startOffset, size - startOffset + 1, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
 	if err != nil {
-		return 0, 0, errInvalidRange
+		return 0, 0, errInvalidCopySourceRange
 	}
 
-	if endOffset <= startOffset {
-		return 0, 0, errInvalidRange
+	if endOffset < startOffset {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if endOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
-func CreateExceedingRangeErr(objSize int64) s3err.APIError {
-	return s3err.APIError{
-		Code:           "InvalidArgument",
-		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
-		HTTPStatusCode: http.StatusBadRequest,
+// ParseCopySource parses x-amz-copy-source header and returns source bucket,
+// source object, versionId, error respectively
+func ParseCopySource(copySourceHeader string) (string, string, string, error) {
+	if copySourceHeader[0] == '/' {
+		copySourceHeader = copySourceHeader[1:]
 	}
+
+	var copySource, versionId string
+	i := strings.LastIndex(copySourceHeader, "?versionId=")
+	if i == -1 {
+		copySource = copySourceHeader
+	} else {
+		copySource = copySourceHeader[:i]
+		versionId = copySourceHeader[i+11:]
+	}
+
+	srcBucket, srcObject, ok := strings.Cut(copySource, "/")
+	if !ok {
+		return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	return srcBucket, srcObject, versionId, nil
+}
+
+// ParseObjectTags parses the url encoded input string into
+// map[string]string key-value tag set
+func ParseObjectTags(t string) (map[string]string, error) {
+	if t == "" {
+		return nil, nil
+	}
+
+	tagging := make(map[string]string)
+
+	tagParts := strings.Split(t, "&")
+	for _, prt := range tagParts {
+		p := strings.Split(prt, "=")
+		if len(p) != 2 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+		}
+		if len(p[0]) > 128 || len(p[1]) > 256 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+		}
+		tagging[p[0]] = p[1]
+	}
+
+	return tagging, nil
 }
 
 func GetMultipartMD5(parts []types.CompletedPart) string {
@@ -112,8 +240,8 @@ func GetMultipartMD5(parts []types.CompletedPart) string {
 	for _, part := range parts {
 		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
 	}
-	s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
-	return s3MD5
+
+	return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
 }
 
 func getEtagBytes(etag string) []byte {
@@ -128,3 +256,16 @@ func md5String(data []byte) string {
 	sum := md5.Sum(data)
 	return hex.EncodeToString(sum[:])
 }
+
+type FileSectionReadCloser struct {
+	R io.Reader
+	F *os.File
+}
+
+func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
+	return f.R.Read(p)
+}
+
+func (f *FileSectionReadCloser) Close() error {
+	return f.F.Close()
+}

backend/meta/meta.go

@@ -14,17 +14,19 @@
 
 package meta
 
+import "os"
+
 // MetadataStorer defines the interface for managing metadata.
 // When object == "", the operation is on the bucket.
 type MetadataStorer interface {
 	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
 	// Returns the value of the attribute, or an error if the attribute does not exist.
-	RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
+	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
 
 	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
 	// If attribute already exists, new attribute should replace existing.
 	// Returns an error if the operation fails.
-	StoreAttribute(bucket, object, attribute string, value []byte) error
+	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
 
 	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
 	// Returns an error if the operation fails.

backend/meta/none.go

@@ -0,0 +1,54 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"os"
+)
+
+// NoMeta is a metadata storer that does not store metadata.
+// This can be useful for read only mounts where attempting to store metadata
+// would fail.
+type NoMeta struct{}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+// always returns ErrNoSuchKey
+func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
+	return nil, ErrNoSuchKey
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+// always returns nil without storing the attribute
+func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+// always returns nil without deleting the attribute
+func (NoMeta) DeleteAttribute(_, _, _ string) error {
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+// always returns an empty list of attributes
+func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
+	return []string{}, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+// always returns nil without deleting any attributes
+func (NoMeta) DeleteAttributes(bucket, object string) error {
+	return nil
+}

backend/meta/sidecar.go

@@ -0,0 +1,139 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// SideCar is a metadata storer that uses sidecar files to store metadata.
+type SideCar struct {
+	dir string
+}
+
+const (
+	sidecarmeta = "meta"
+)
+
+// NewSideCar creates a new SideCar metadata storer.
+func NewSideCar(dir string) (SideCar, error) {
+	fi, err := os.Lstat(dir)
+	if err != nil {
+		return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
+	}
+	if !fi.IsDir() {
+		return SideCar{}, fmt.Errorf("not a directory")
+	}
+
+	return SideCar{dir: dir}, nil
+}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	value, err := os.ReadFile(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, ErrNoSuchKey
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attribute: %v", err)
+	}
+
+	return value, nil
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	err := os.MkdirAll(metadir, 0777)
+	if err != nil {
+		return fmt.Errorf("failed to create metadata directory: %v", err)
+	}
+
+	attr := filepath.Join(metadir, attribute)
+	err = os.WriteFile(attr, value, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to write attribute: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	err := os.Remove(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return ErrNoSuchKey
+	}
+	if err != nil {
+		return fmt.Errorf("failed to remove attribute: %v", err)
+	}
+
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	ents, err := os.ReadDir(metadir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []string{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to list attributes: %v", err)
+	}
+
+	var attrs []string
+	for _, ent := range ents {
+		attrs = append(attrs, ent.Name())
+	}
+
+	return attrs, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+func (s SideCar) DeleteAttributes(bucket, object string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	err := os.RemoveAll(metadir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove attributes: %v", err)
+	}
+	return nil
+}

backend/meta/xattr.go

@@ -17,11 +17,13 @@ package meta
 import (
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
 
 	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/s3err"
 )
 
 const (
@@ -36,7 +38,15 @@ var (
 type XattrMeta struct{}
 
 // RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
+func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
+	if f != nil {
+		b, err := xattr.FGet(f, xattrPrefix+attribute)
+		if errors.Is(err, xattr.ENOATTR) {
+			return nil, ErrNoSuchKey
+		}
+		return b, err
+	}
+
 	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
 	if errors.Is(err, xattr.ENOATTR) {
 		return nil, ErrNoSuchKey
@@ -45,8 +55,20 @@ func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte,
 }
 
 // StoreAttribute stores the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
-	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
+	if f != nil {
+		err := xattr.FSet(f, xattrPrefix+attribute, value)
+		if errors.Is(err, syscall.EROFS) {
+			return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return err
+	}
+
+	err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
+	return err
 }
 
 // DeleteAttribute removes the value of a specific attribute for an object in a bucket.
@@ -55,6 +77,9 @@ func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
 	if errors.Is(err, xattr.ENOATTR) {
 		return ErrNoSuchKey
 	}
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
 	return err
 }
 

backend/mkdir.go

@@ -15,11 +15,6 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
-var (
-	// TODO: make this configurable
-	defaultDirPerm fs.FileMode = 0755
-)
-
 // MkdirAll is similar to os.MkdirAll but it will return
 // ErrObjectParentIsFile when appropriate
 // MkdirAll creates a directory named path,
@@ -32,7 +27,7 @@ var (
 // and returns nil.
 // Any directory created will be set to provided uid/gid ownership
 // if doChown is true.
-func MkdirAll(path string, uid, gid int, doChown bool) error {
+func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
 	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
 	dir, err := os.Stat(path)
 	if err == nil {
@@ -55,14 +50,14 @@ func MkdirAll(path string, uid, gid int, doChown bool) error {
 
 	if j > 1 {
 		// Create parent.
-		err = MkdirAll(path[:j-1], uid, gid, doChown)
+		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
 		if err != nil {
 			return err
 		}
 	}
 
 	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, defaultDirPerm)
+	err = os.Mkdir(path, dirPerm)
 	if err != nil {
 		// Handle arguments like "foo/." by
 		// double-checking that directory doesn't exist.

backend/posix/with_otmpfile.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 	"golang.org/x/sys/unix"
 )
 
@@ -43,6 +44,7 @@ type tmpfile struct {
 	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
 }
 
 var (
@@ -61,8 +63,12 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	// this is not supported.
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown)
+		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -108,6 +114,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
+		newDirPerm: p.newDirPerm,
 	}
 
 	// falloc is best effort, its fine if this fails
@@ -134,6 +141,9 @@ func (tmp *tmpfile) falloc() error {
 }
 
 func (tmp *tmpfile) link() error {
+	// make sure this is cleaned up in all error cases
+	defer tmp.f.Close()
+
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
 	// with any other simultaneous uploads. The final operation is to move the
@@ -148,7 +158,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -170,11 +180,21 @@ func (tmp *tmpfile) link() error {
 	}
 	defer dirf.Close()
 
-	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-	if err != nil {
-		return fmt.Errorf("link tmpfile (%q in %q): %w",
-			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
+	for {
+		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		if errors.Is(err, syscall.EEXIST) {
+			err := os.Remove(objPath)
+			if err != nil && !errors.Is(err, fs.ErrNotExist) {
+				return fmt.Errorf("remove stale path: %w", err)
+			}
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("link tmpfile (fd %q as %q): %w",
+				filepath.Base(tmp.f.Name()), objPath, err)
+		}
+		break
 	}
 
 	err = tmp.f.Close()

backend/posix/without_otmpfile.go

@@ -24,9 +24,11 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"syscall"
 
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
 )
 
 type tmpfile struct {
@@ -41,13 +43,19 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 
 	// Create a temp file for upload while in progress (see link comments below).
 	var err error
-	err = backend.MkdirAll(dir, uid, gid, doChown)
+	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
 		return nil, fmt.Errorf("create temp file: %w", err)
 	}
 

backend/scoutfs/scoutfs_compat.go

@@ -37,8 +37,10 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	metastore := meta.XattrMeta{}
 
 	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
-		ChownUID: opts.ChownUID,
-		ChownGID: opts.ChownGID,
+		ChownUID:    opts.ChownUID,
+		ChownGID:    opts.ChownGID,
+		BucketLinks: opts.BucketLinks,
+		NewDirPerm:  opts.NewDirPerm,
 	})
 	if err != nil {
 		return nil, err
@@ -50,13 +52,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:       p,
-		rootfd:      f,
-		rootdir:     rootdir,
-		meta:        metastore,
-		chownuid:    opts.ChownUID,
-		chowngid:    opts.ChownGID,
-		glaciermode: opts.GlacierMode,
+		Posix:            p,
+		rootfd:           f,
+		rootdir:          rootdir,
+		meta:             metastore,
+		chownuid:         opts.ChownUID,
+		chowngid:         opts.ChownGID,
+		glaciermode:      opts.GlacierMode,
+		newDirPerm:       opts.NewDirPerm,
+		disableNoArchive: opts.DisableNoArchive,
 	}, nil
 }
 
@@ -70,10 +74,10 @@ type tmpfile struct {
 	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
 }
 
 var (
-	// TODO: make this configurable
 	defaultFilePerm uint32 = 0644
 )
 
@@ -101,6 +105,7 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
+		newDirPerm: s.newDirPerm,
 	}
 
 	if doChown {
@@ -128,7 +133,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -173,6 +178,10 @@ func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}
+
 func moveData(from *os.File, to *os.File) error {
 	return scoutfs.MoveData(from, to)
 }

backend/scoutfs/scoutfs_incompat.go

@@ -28,9 +28,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
-type tmpfile struct {
-	f *os.File
-}
+type tmpfile struct{}
 
 var (
 	errNotSupported = errors.New("not supported")
@@ -56,6 +54,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return nil
+}
+
 func moveData(_, _ *os.File) error {
 	return errNotSupported
 }

backend/walk.go

@@ -15,32 +15,34 @@
 package backend
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/fs"
-	"os"
 	"sort"
 	"strings"
+	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3response"
 )
 
 type WalkResults struct {
 	CommonPrefixes []types.CommonPrefix
-	Objects        []types.Object
+	Objects        []s3response.Object
 	Truncated      bool
 	NextMarker     string
 }
 
-type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
+type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
 
 var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
-	var objects []types.Object
+	var objects []s3response.Object
 
 	var pastMarker bool
 	if marker == "" {
@@ -51,10 +53,21 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 	var newMarker string
 	var truncated bool
 
-	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+	root := "."
+	if strings.Contains(prefix, "/") {
+		idx := strings.LastIndex(prefix, "/")
+		if idx > 0 {
+			root = prefix[:idx]
+		}
+	}
+
+	err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
 		// Ignore the root directory
 		if path == "." {
 			return nil
@@ -71,6 +84,9 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			return fs.SkipAll
 		}
 
+		// After this point, return skipflag instead of nil
+		// so we can skip a directory without an early return
+		var skipflag error
 		if d.IsDir() {
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
@@ -80,44 +96,58 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			// building to match. So only skip if path isn't a prefix of prefix
 			// and prefix isn't a prefix of path.
 			if prefix != "" &&
-				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
-				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
+				!strings.HasPrefix(path+"/", prefix) &&
+				!strings.HasPrefix(prefix, path+"/") {
 				return fs.SkipDir
 			}
 
-			// TODO: can we do better here rather than a second readdir
-			// per directory?
-			ents, err := fs.ReadDir(fileSystem, path)
-			if err != nil {
-				return fmt.Errorf("readdir %q: %w", path, err)
-			}
-			if len(ents) == 0 {
-				dirobj, err := getObj(path, d)
-				if err == ErrSkipObj {
-					return nil
-				}
-				if err != nil {
-					return fmt.Errorf("directory to object %q: %w", path, err)
-				}
-				objects = append(objects, dirobj)
-			}
+			// Don't recurse into subdirectories which contain the delimiter
+			// after reaching the prefix
+			if delimiter != "" &&
+				strings.HasPrefix(path+"/", prefix) &&
+				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
+				skipflag = fs.SkipDir
+			} else {
+				if delimiter == "" {
+					dirobj, err := getObj(path+"/", d)
+					if err == ErrSkipObj {
+						return skipflag
+					}
+					if err != nil {
+						return fmt.Errorf("directory to object %q: %w", path, err)
+					}
+					objects = append(objects, dirobj)
 
-			return nil
+					return skipflag
+				}
+
+				// TODO: can we do better here rather than a second readdir
+				// per directory?
+				ents, err := fs.ReadDir(fileSystem, path)
+				if err != nil {
+					return fmt.Errorf("readdir %q: %w", path, err)
+				}
+
+				if len(ents) != 0 {
+					return skipflag
+				}
+			}
+			path += "/"
 		}
 
 		if !pastMarker {
 			if path == marker {
 				pastMarker = true
-				return nil
+				return skipflag
 			}
 			if path < marker {
-				return nil
+				return skipflag
 			}
 		}
 
 		// If object doesn't have prefix, don't include in results.
 		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return nil
+			return skipflag
 		}
 
 		if delimiter == "" {
@@ -125,7 +155,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			// prefix are included in results
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return nil
+				return skipflag
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -136,7 +166,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 				pastMax = true
 			}
 
-			return nil
+			return skipflag
 		}
 
 		// Since delimiter is specified, we only want results that
@@ -165,7 +195,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		if !found {
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return nil
+				return skipflag
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -174,20 +204,38 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			if (len(objects) + len(cpmap)) == int(max) {
 				pastMax = true
 			}
-			return nil
+			return skipflag
 		}
 
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
-		// delimiter at the end.
-		cpmap[prefix+before+delimiter] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
-			pastMax = true
+		// delimiter at the end when we add to the map.
+		cprefNoDelim := prefix + before
+		cpref := prefix + before + delimiter
+		if cpref == marker {
+			pastMarker = true
+			return skipflag
 		}
 
-		return nil
+		if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
+			// skip common prefixes that are before the marker
+			return skipflag
+		}
+
+		cpmap[cpref] = struct{}{}
+		if (len(objects) + len(cpmap)) == int(max) {
+			newMarker = cpref
+			truncated = true
+			return fs.SkipAll
+		}
+
+		return skipflag
 	})
 	if err != nil {
+		// suppress file not found caused by user's prefix
+		if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
+			return WalkResults{}, nil
+		}
 		return WalkResults{}, err
 	}
 
@@ -220,3 +268,213 @@ func contains(a string, strs []string) bool {
 	}
 	return false
 }
+
+type WalkVersioningResults struct {
+	CommonPrefixes      []types.CommonPrefix
+	ObjectVersions      []types.ObjectVersion
+	DelMarkers          []types.DeleteMarkerEntry
+	Truncated           bool
+	NextMarker          string
+	NextVersionIdMarker string
+}
+
+type ObjVersionFuncResult struct {
+	ObjectVersions      []types.ObjectVersion
+	DelMarkers          []types.DeleteMarkerEntry
+	NextVersionIdMarker string
+	Truncated           bool
+}
+
+type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
+
+// WalkVersions walks the supplied fs.FS and returns results compatible with
+// ListObjectVersions action response
+func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
+	cpmap := make(map[string]struct{})
+	var objects []types.ObjectVersion
+	var delMarkers []types.DeleteMarkerEntry
+
+	var pastMarker bool
+	if keyMarker == "" {
+		pastMarker = true
+	}
+	var nextMarker string
+	var nextVersionIdMarker string
+	var truncated bool
+
+	pastVersionIdMarker := versionIdMarker == ""
+
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
+
+		if !pastMarker {
+			if path == keyMarker {
+				pastMarker = true
+			}
+			if path < keyMarker {
+				return nil
+			}
+		}
+
+		if d.IsDir() {
+			// If prefix is defined and the directory does not match prefix,
+			// do not descend into the directory because nothing will
+			// match this prefix. Make sure to append the / at the end of
+			// directories since this is implied as a directory path name.
+			// If path is a prefix of prefix, then path could still be
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
+			if prefix != "" &&
+				!strings.HasPrefix(path+"/", prefix) &&
+				!strings.HasPrefix(prefix, path+"/") {
+				return fs.SkipDir
+			}
+
+			// Don't recurse into subdirectories when listing with delimiter.
+			if delimiter == "/" &&
+				prefix != path+"/" &&
+				strings.HasPrefix(path+"/", prefix) {
+				cpmap[path+"/"] = struct{}{}
+				return fs.SkipDir
+			}
+
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("directory to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+
+			return nil
+		}
+
+		// If object doesn't have prefix, don't include in results.
+		if prefix != "" && !strings.HasPrefix(path, prefix) {
+			return nil
+		}
+
+		if delimiter == "" {
+			// If no delimiter specified, then all files with matching
+			// prefix are included in results
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+
+			return nil
+		}
+
+		// Since delimiter is specified, we only want results that
+		// do not contain the delimiter beyond the prefix.  If the
+		// delimiter exists past the prefix, then the substring
+		// between the prefix and delimiter is part of common prefixes.
+		//
+		// For example:
+		// prefix = A/
+		// delimiter = /
+		// and objects:
+		// A/file
+		// A/B/file
+		// B/C
+		// would return:
+		// objects: A/file
+		// common prefix: A/B/
+		//
+		// Note: No objects are included past the common prefix since
+		// these are all rolled up into the common prefix.
+		// Note: The delimiter can be anything, so we have to operate on
+		// the full path without any assumptions on posix directory hierarchy
+		// here.  Usually the delimiter will be "/", but thats not required.
+		suffix := strings.TrimPrefix(path, prefix)
+		before, _, found := strings.Cut(suffix, delimiter)
+		if !found {
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+			return nil
+		}
+
+		// Common prefixes are a set, so should not have duplicates.
+		// These are abstractly a "directory", so need to include the
+		// delimiter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
+		if (len(objects) + len(cpmap)) == int(max) {
+			nextMarker = path
+			truncated = true
+
+			return fs.SkipAll
+		}
+
+		return nil
+	})
+	if err != nil {
+		return WalkVersioningResults{}, err
+	}
+
+	var commonPrefixStrings []string
+	for k := range cpmap {
+		commonPrefixStrings = append(commonPrefixStrings, k)
+	}
+	sort.Strings(commonPrefixStrings)
+	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
+	for _, cp := range commonPrefixStrings {
+		pfx := cp
+		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
+			Prefix: &pfx,
+		})
+	}
+
+	return WalkVersioningResults{
+		CommonPrefixes:      commonPrefixes,
+		ObjectVersions:      objects,
+		DelMarkers:          delMarkers,
+		Truncated:           truncated,
+		NextMarker:          nextMarker,
+		NextVersionIdMarker: nextVersionIdMarker,
+	}, nil
+}

backend/walk_test.go

@@ -15,36 +15,50 @@
 package backend_test
 
 import (
+	"context"
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
 	"io/fs"
+	"sync"
 	"testing"
 	"testing/fstest"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3response"
 )
 
 type walkTest struct {
-	fsys     fs.FS
-	expected backend.WalkResults
-	getobj   backend.GetObjFunc
+	fsys   fs.FS
+	getobj backend.GetObjFunc
+	cases  []testcase
 }
 
-func getObj(path string, d fs.DirEntry) (types.Object, error) {
+type testcase struct {
+	name      string
+	prefix    string
+	delimiter string
+	marker    string
+	maxObjs   int32
+	expected  backend.WalkResults
+}
+
+func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
 	if d.IsDir() {
 		etag := getMD5(path)
 
 		fi, err := d.Info()
 		if err != nil {
-			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
+		mtime := fi.ModTime()
 
-		return types.Object{
+		return s3response.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: backend.GetTimePtr(fi.ModTime()),
+			LastModified: &mtime,
 		}, nil
 	}
 
@@ -52,15 +66,16 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
 
 	fi, err := d.Info()
 	if err != nil {
-		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 	}
 
 	size := fi.Size()
+	mtime := fi.ModTime()
 
-	return types.Object{
+	return s3response.Object{
 		ETag:         &etag,
 		Key:          &path,
-		LastModified: backend.GetTimePtr(fi.ModTime()),
+		LastModified: &mtime,
 		Size:         &size,
 	}, nil
 }
@@ -82,50 +97,154 @@ func TestWalk(t *testing.T) {
 				"photos/2006/February/sample3.jpg": {},
 				"photos/2006/February/sample4.jpg": {},
 			},
-			expected: backend.WalkResults{
-				CommonPrefixes: []types.CommonPrefix{{
-					Prefix: backend.GetStringPtr("photos/"),
-				}},
-				Objects: []types.Object{{
-					Key: backend.GetStringPtr("sample.jpg"),
-				}},
-			},
 			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "aws example",
+					delimiter: "/",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photos/"),
+						}},
+						Objects: []s3response.Object{{
+							Key: backend.GetPtrFromString("sample.jpg"),
+						}},
+					},
+				},
+			},
 		},
 		{
 			// test case single dir/single file
 			fsys: fstest.MapFS{
 				"test/file": {},
 			},
-			expected: backend.WalkResults{
-				CommonPrefixes: []types.CommonPrefix{{
-					Prefix: backend.GetStringPtr("test/"),
-				}},
-				Objects: []types.Object{},
+			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "single dir single file",
+					delimiter: "/",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("test/"),
+						}},
+						Objects: []s3response.Object{},
+					},
+				},
+			},
+		},
+		{
+			// non-standard delimiter
+			fsys: fstest.MapFS{
+				"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
+				"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
+				"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
 			},
 			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "different delimiter 1",
+					delimiter: "|",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 2",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 3",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 4",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 5",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|6/Januar|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 6",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|6/Januar|y/sampl|",
+					expected: backend.WalkResults{
+						Objects: []s3response.Object{
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
+							},
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
+							},
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
+							},
+						},
+					},
+				},
+			},
 		},
 	}
 
 	for _, tt := range tests {
-		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
-		if err != nil {
-			t.Fatalf("walk: %v", err)
-		}
+		for _, tc := range tt.cases {
+			res, err := backend.Walk(context.Background(),
+				tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
+				tt.getobj, []string{})
+			if err != nil {
+				t.Errorf("tc.name: walk: %v", err)
+			}
 
-		compareResults(res, tt.expected, t)
+			compareResults(tc.name, res, tc.expected, t)
+		}
 	}
 }
 
-func compareResults(got, wanted backend.WalkResults, t *testing.T) {
+func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
 	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
-		t.Errorf("unexpected common prefix, got %v wanted %v",
+		t.Errorf("%v: unexpected common prefix, got %v wanted %v",
+			name,
 			printCommonPrefixes(got.CommonPrefixes),
 			printCommonPrefixes(wanted.CommonPrefixes))
 	}
 
 	if !compareObjects(got.Objects, wanted.Objects) {
-		t.Errorf("unexpected object, got %v wanted %v",
+		t.Errorf("%v: unexpected object, got %v wanted %v",
+			name,
 			printObjects(got.Objects),
 			printObjects(wanted.Objects))
 	}
@@ -168,7 +287,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
 	return res + "]"
 }
 
-func compareObjects(a, b []types.Object) bool {
+func compareObjects(a, b []s3response.Object) bool {
 	if len(a) == 0 && len(b) == 0 {
 		return true
 	}
@@ -184,7 +303,7 @@ func compareObjects(a, b []types.Object) bool {
 	return false
 }
 
-func containsObject(c types.Object, list []types.Object) bool {
+func containsObject(c s3response.Object, list []s3response.Object) bool {
 	for _, cp := range list {
 		if *c.Key == *cp.Key {
 			return true
@@ -193,14 +312,67 @@ func containsObject(c types.Object, list []types.Object) bool {
 	return false
 }
 
-func printObjects(list []types.Object) string {
+func printObjects(list []s3response.Object) string {
 	res := "["
 	for _, cp := range list {
-		if res == "[" {
-			res = res + *cp.Key
+		var key string
+		if cp.Key == nil {
+			key = "<nil>"
 		} else {
-			res = res + ", " + *cp.Key
+			key = *cp.Key
+		}
+		if res == "[" {
+			res = res + key
+		} else {
+			res = res + ", " + key
 		}
 	}
 	return res + "]"
 }
+
+type slowFS struct {
+	fstest.MapFS
+}
+
+const (
+	readDirPause = 100 * time.Millisecond
+
+	// walkTimeOut should be less than the tree traversal time
+	// which is the readdirPause time * the number of directories
+	walkTimeOut = 500 * time.Millisecond
+)
+
+func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
+	time.Sleep(readDirPause)
+	return s.MapFS.ReadDir(name)
+}
+
+func TestWalkStop(t *testing.T) {
+	s := &slowFS{MapFS: fstest.MapFS{
+		"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
+	}}
+
+	ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
+	defer cancel()
+
+	var err error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, err = backend.Walk(ctx, s, "", "/", "", 1000,
+			func(path string, d fs.DirEntry) (s3response.Object, error) {
+				return s3response.Object{}, nil
+			}, []string{})
+	}()
+
+	select {
+	case <-time.After(1 * time.Second):
+		t.Fatalf("walk is not terminated in time")
+	case <-ctx.Done():
+	}
+	wg.Wait()
+	if err != ctx.Err() {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}

cmd/versitygw/admin.go

@@ -19,7 +19,7 @@ import (
 	"crypto/sha256"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,6 +29,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3response"
@@ -37,6 +38,7 @@ import (
 var (
 	adminAccess   string
 	adminSecret   string
+	adminRegion   string
 	adminEndpoint string
 	allowInsecure bool
 )
@@ -171,6 +173,14 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminSecret,
 			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "admin s3 region string",
+				EnvVars:     []string{"ADMIN_REGION"},
+				Value:       "us-east-1",
+				Destination: &adminRegion,
+				Aliases:     []string{"r"},
+			},
 			&cli.StringFlag{
 				Name:        "endpoint-url",
 				Usage:       "admin apis endpoint url",
@@ -215,24 +225,24 @@ func createUser(ctx *cli.Context) error {
 		GroupID: groupID,
 	}
 
-	accJson, err := json.Marshal(acc)
+	accxml, err := xml.Marshal(acc)
 	if err != nil {
 		return fmt.Errorf("failed to parse user data: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(accJson)
+	hashedPayload := sha256.Sum256(accxml)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -251,11 +261,9 @@ func createUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -277,7 +285,7 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -296,11 +304,9 @@ func deleteUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -317,24 +323,24 @@ func updateUser(ctx *cli.Context) error {
 		props.GroupID = &groupId
 	}
 
-	propsJSON, err := json.Marshal(props)
+	propsxml, err := xml.Marshal(props)
 	if err != nil {
 		return fmt.Errorf("failed to parse user attributes: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(propsJSON)
+	hashedPayload := sha256.Sum256(propsxml)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -353,11 +359,9 @@ func updateUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
@@ -374,7 +378,7 @@ func listUsers(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -393,15 +397,15 @@ func listUsers(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var accs []auth.Account
-	if err := json.Unmarshal(body, &accs); err != nil {
+	var accs auth.ListUserAccountsResult
+	if err := xml.Unmarshal(body, &accs); err != nil {
 		return err
 	}
 
-	printAcctTable(accs)
+	printAcctTable(accs.Accounts)
 
 	return nil
 }
@@ -441,7 +445,7 @@ func changeBucketOwner(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -460,11 +464,9 @@ func changeBucketOwner(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Println(string(body))
-
 	return nil
 }
 
@@ -493,7 +495,7 @@ func listBuckets(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -512,15 +514,26 @@ func listBuckets(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var buckets []s3response.Bucket
-	if err := json.Unmarshal(body, &buckets); err != nil {
+	var result s3response.ListBucketsResult
+	if err := xml.Unmarshal(body, &result); err != nil {
 		return err
 	}
 
-	printBuckets(buckets)
+	printBuckets(result.Buckets)
 
 	return nil
 }
+
+func parseApiError(body []byte) error {
+	var apiErr smithy.GenericAPIError
+	err := xml.Unmarshal(body, &apiErr)
+	if err != nil {
+		apiErr.Code = "InternalServerError"
+		apiErr.Message = err.Error()
+	}
+
+	return &apiErr
+}

cmd/versitygw/gateway_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
@@ -57,7 +58,9 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
+	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
+		NewDirPerm: 0755,
+	})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}
@@ -75,6 +78,9 @@ func initPosix(ctx context.Context) {
 		}
 		wg.Done()
 	}()
+
+	// wait for server to start
+	time.Sleep(1 * time.Second)
 }
 
 func TestIntegration(t *testing.T) {

cmd/versitygw/main.go

@@ -19,9 +19,11 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
+	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
@@ -45,8 +47,8 @@ var (
 	natsURL, natsTopic                       string
 	eventWebhookURL                          string
 	eventConfigFilePath                      string
-	logWebhookURL                            string
-	accessLog                                string
+	logWebhookURL, accessLog                 string
+	adminLogFile                             string
 	healthPath                               string
 	debug                                    bool
 	pprof                                    string
@@ -72,6 +74,9 @@ var (
 	metricsService                           string
 	statsdServers                            string
 	dogstatsServers                          string
+	ipaHost, ipaVaultName                    string
+	ipaUser, ipaPassword                     string
+	ipaInsecure, ipaDebug                    bool
 )
 
 var (
@@ -112,10 +117,13 @@ func main() {
 
 func initApp() *cli.App {
 	return &cli.App{
-		Name:  "versitygw",
-		Usage: "Start S3 gateway service with specified backend storage.",
-		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
-to access the supported backend storage as if it was a native S3 service.`,
+		Usage: "Versity S3 Gateway",
+		Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.
+VersityGW is an open-source project licensed under the Apache 2.0 License. The
+source code is hosted on GitHub at https://github.com/versity/versitygw, and
+documentation can be found in the GitHub wiki.`,
+		Copyright: "Copyright (c) 2023-2024 Versity Software",
 		Action: func(ctx *cli.Context) error {
 			return ctx.App.Command("help").Run(ctx)
 		},
@@ -201,6 +209,7 @@ func initFlags() []cli.Flag {
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
+			Value:       false,
 			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
@@ -223,6 +232,12 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
 			Destination: &accessLog,
 		},
+		&cli.StringFlag{
+			Name:        "admin-access-log",
+			Usage:       "enable admin server access logging to specified file",
+			EnvVars:     []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
+			Destination: &adminLogFile,
+		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
@@ -495,6 +510,42 @@ func initFlags() []cli.Flag {
 			Aliases:     []string{"mds"},
 			Destination: &dogstatsServers,
 		},
+		&cli.StringFlag{
+			Name:        "ipa-host",
+			Usage:       "FreeIPA server url e.g. https://ipa.example.test",
+			EnvVars:     []string{"VGW_IPA_HOST"},
+			Destination: &ipaHost,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-vault-name",
+			Usage:       "A name of the user vault containing their secret",
+			EnvVars:     []string{"VGW_IPA_VAULT_NAME"},
+			Destination: &ipaVaultName,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-user",
+			Usage:       "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
+			EnvVars:     []string{"VGW_IPA_USER"},
+			Destination: &ipaUser,
+		},
+		&cli.StringFlag{
+			Name:        "ipa-password",
+			Usage:       "Password of the user used to connect to FreeIPA.",
+			EnvVars:     []string{"VGW_IPA_PASSWORD"},
+			Destination: &ipaPassword,
+		},
+		&cli.BoolFlag{
+			Name:        "ipa-insecure",
+			Usage:       "Verify TLS certificate of FreeIPA server. Default is 'true'.",
+			EnvVars:     []string{"VGW_IPA_INSECURE"},
+			Destination: &ipaInsecure,
+		},
+		&cli.BoolFlag{
+			Name:        "ipa-debug",
+			Usage:       "FreeIPA IAM debug output",
+			EnvVars:     []string{"VGW_IPA_DEBUG"},
+			Destination: &ipaDebug,
+		},
 	}
 }
 
@@ -512,10 +563,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	app := fiber.New(fiber.Config{
-		AppName:           "versitygw",
-		ServerHeader:      "VERSITYGW",
-		StreamRequestBody: true,
-		DisableKeepalive:  true,
+		AppName:               "versitygw",
+		ServerHeader:          "VERSITYGW",
+		StreamRequestBody:     true,
+		DisableKeepalive:      true,
+		Network:               fiber.NetworkTCP,
+		DisableStartupMessage: true,
 	})
 
 	var opts []s3api.Option
@@ -551,8 +604,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	admApp := fiber.New(fiber.Config{
-		AppName:      "versitygw",
-		ServerHeader: "VERSITYGW",
+		AppName:               "versitygw",
+		ServerHeader:          "VERSITYGW",
+		Network:               fiber.NetworkTCP,
+		DisableStartupMessage: true,
 	})
 
 	var admOpts []s3api.AdminOpt
@@ -573,6 +628,11 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
+		RootAccount: auth.Account{
+			Access: rootUserAccess,
+			Secret: rootUserSecret,
+			Role:   auth.RoleAdmin,
+		},
 		Dir:                    iamDir,
 		LDAPServerURL:          ldapURL,
 		LDAPBindDN:             ldapBindDN,
@@ -603,14 +663,21 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		CacheDisable:           iamCacheDisable,
 		CacheTTL:               iamCacheTTL,
 		CachePrune:             iamCachePrune,
+		IpaHost:                ipaHost,
+		IpaVaultName:           ipaVaultName,
+		IpaUser:                ipaUser,
+		IpaPassword:            ipaPassword,
+		IpaInsecure:            ipaInsecure,
+		IpaDebug:               ipaDebug,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)
 	}
 
-	logger, err := s3log.InitLogger(&s3log.LogConfig{
-		LogFile:    accessLog,
-		WebhookURL: logWebhookURL,
+	loggers, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:      accessLog,
+		WebhookURL:   logWebhookURL,
+		AdminLogFile: adminLogFile,
 	})
 	if err != nil {
 		return fmt.Errorf("setup logger: %w", err)
@@ -641,12 +708,16 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
 		Access: rootUserAccess,
 		Secret: rootUserSecret,
-	}, port, region, iam, logger, evSender, metricsManager, opts...)
+	}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
 	if err != nil {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
+
+	if !quiet {
+		printBanner(port, admPort, certFile != "", admCertFile != "")
+	}
 
 	c := make(chan error, 2)
 	go func() { c <- srv.Serve() }()
@@ -663,10 +734,17 @@ Loop:
 		case err = <-c:
 			break Loop
 		case <-sigHup:
-			if logger != nil {
-				err = logger.HangUp()
+			if loggers.S3Logger != nil {
+				err = loggers.S3Logger.HangUp()
 				if err != nil {
-					err = fmt.Errorf("HUP logger: %w", err)
+					err = fmt.Errorf("HUP s3 logger: %w", err)
+					break Loop
+				}
+			}
+			if loggers.AdminLogger != nil {
+				err = loggers.AdminLogger.HangUp()
+				if err != nil {
+					err = fmt.Errorf("HUP admin logger: %w", err)
 					break Loop
 				}
 			}
@@ -684,13 +762,22 @@ Loop:
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
-	if logger != nil {
-		err := logger.Shutdown()
+	if loggers.S3Logger != nil {
+		err := loggers.S3Logger.Shutdown()
 		if err != nil {
 			if saveErr == nil {
 				saveErr = err
 			}
-			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
+			fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
+		}
+	}
+	if loggers.AdminLogger != nil {
+		err := loggers.AdminLogger.Shutdown()
+		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
+			fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
 		}
 	}
 
@@ -710,3 +797,177 @@ Loop:
 
 	return saveErr
 }
+
+func printBanner(port, admPort string, ssl, admSsl bool) {
+	interfaces, err := getMatchingIPs(port)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
+		return
+	}
+
+	var admInterfaces []string
+	if admPort != "" {
+		admInterfaces, err = getMatchingIPs(admPort)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
+			return
+		}
+	}
+
+	title := "VersityGW"
+	version := fmt.Sprintf("Version %v, Build %v", Version, Build)
+	urls := []string{}
+
+	hst, prt, err := net.SplitHostPort(port)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
+		return
+	}
+
+	for _, ip := range interfaces {
+		url := fmt.Sprintf("http://%s:%s", ip, prt)
+		if ssl {
+			url = fmt.Sprintf("https://%s:%s", ip, prt)
+		}
+		urls = append(urls, url)
+	}
+
+	if hst == "" {
+		hst = "0.0.0.0"
+	}
+
+	boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
+
+	lines := []string{
+		centerText(title),
+		centerText(version),
+		centerText(boundHost),
+		centerText(""),
+	}
+
+	if len(admInterfaces) > 0 {
+		lines = append(lines,
+			leftText("S3 service listening on:"),
+		)
+	} else {
+		lines = append(lines,
+			leftText("Admin/S3 service listening on:"),
+		)
+	}
+
+	for _, url := range urls {
+		lines = append(lines, leftText("  "+url))
+	}
+
+	if len(admInterfaces) > 0 {
+		lines = append(lines,
+			centerText(""),
+			leftText("Admin service listening on:"),
+		)
+
+		_, prt, err := net.SplitHostPort(admPort)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
+			return
+		}
+
+		for _, ip := range admInterfaces {
+			url := fmt.Sprintf("http://%s:%s", ip, prt)
+			if admSsl {
+				url = fmt.Sprintf("https://%s:%s", ip, prt)
+			}
+			lines = append(lines, leftText("  "+url))
+		}
+	}
+
+	// Print the top border
+	fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
+
+	// Print each line
+	for _, line := range lines {
+		fmt.Printf("│%-*s│\n", columnWidth-2, line)
+	}
+
+	// Print the bottom border
+	fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
+}
+
+// getMatchingIPs returns all IP addresses for local system interfaces that
+// match the input address specification.
+func getMatchingIPs(spec string) ([]string, error) {
+	// Split the input spec into IP and port
+	host, _, err := net.SplitHostPort(spec)
+	if err != nil {
+		return nil, fmt.Errorf("parse address/port: %v", err)
+	}
+
+	// Handle cases where IP is omitted (e.g., ":1234")
+	if host == "" {
+		host = "0.0.0.0"
+	}
+
+	ipaddr, err := net.ResolveIPAddr("ip", host)
+	if err != nil {
+		return nil, err
+	}
+
+	parsedInputIP := ipaddr.IP
+
+	var result []string
+
+	// Get all network interfaces
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, iface := range interfaces {
+		// Get all addresses associated with the interface
+		addrs, err := iface.Addrs()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, addr := range addrs {
+			// Parse the address to get the IP part
+			ipAddr, _, err := net.ParseCIDR(addr.String())
+			if err != nil {
+				return nil, err
+			}
+
+			if ipAddr.IsLinkLocalUnicast() {
+				continue
+			}
+			if ipAddr.IsInterfaceLocalMulticast() {
+				continue
+			}
+			if ipAddr.IsLinkLocalMulticast() {
+				continue
+			}
+
+			// Check if the IP matches the input specification
+			if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
+				result = append(result, ipAddr.String())
+			}
+		}
+	}
+
+	return result, nil
+}
+
+const columnWidth = 70
+
+func centerText(text string) string {
+	padding := (columnWidth - 2 - len(text)) / 2
+	if padding < 0 {
+		padding = 0
+	}
+	return strings.Repeat(" ", padding) + text
+}
+
+func leftText(text string) string {
+	if len(text) > columnWidth-2 {
+		return text
+	}
+	return text + strings.Repeat(" ", columnWidth-2-len(text))
+}

cmd/versitygw/posix.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/meta"
@@ -24,6 +26,11 @@ import (
 
 var (
 	chownuid, chowngid bool
+	bucketlinks        bool
+	versioningDir      string
+	dirPerms           uint
+	sidecar            string
+	nometa             bool
 )
 
 func posixCommand() *cli.Command {
@@ -54,6 +61,38 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_CHOWN_GID"},
 				Destination: &chowngid,
 			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
+			&cli.StringFlag{
+				Name:        "versioning-dir",
+				Usage:       "the directory path to enable bucket versioning",
+				EnvVars:     []string{"VGW_VERSIONING_DIR"},
+				Destination: &versioningDir,
+			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
+			&cli.StringFlag{
+				Name:        "sidecar",
+				Usage:       "use provided sidecar directory to store metadata",
+				EnvVars:     []string{"VGW_META_SIDECAR"},
+				Destination: &sidecar,
+			},
+			&cli.BoolFlag{
+				Name:        "nometa",
+				Usage:       "disable metadata storage",
+				EnvVars:     []string{"VGW_META_NONE"},
+				Destination: &nometa,
+			},
 		},
 	}
 }
@@ -64,17 +103,45 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	gwroot := (ctx.Args().Get(0))
-	err := meta.XattrMeta{}.Test(gwroot)
-	if err != nil {
-		return fmt.Errorf("posix xattr check: %v", err)
+
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
 	}
 
-	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
-		ChownUID: chownuid,
-		ChownGID: chowngid,
-	})
+	if nometa && sidecar != "" {
+		return fmt.Errorf("cannot use both nometa and sidecar metadata")
+	}
+
+	opts := posix.PosixOpts{
+		ChownUID:      chownuid,
+		ChownGID:      chowngid,
+		BucketLinks:   bucketlinks,
+		VersioningDir: versioningDir,
+		NewDirPerm:    fs.FileMode(dirPerms),
+	}
+
+	var ms meta.MetadataStorer
+	switch {
+	case sidecar != "":
+		sc, err := meta.NewSideCar(sidecar)
+		if err != nil {
+			return fmt.Errorf("failed to init sidecar metadata: %w", err)
+		}
+		ms = sc
+		opts.SideCarDir = sidecar
+	case nometa:
+		ms = meta.NoMeta{}
+	default:
+		ms = meta.XattrMeta{}
+		err := meta.XattrMeta{}.Test(gwroot)
+		if err != nil {
+			return fmt.Errorf("xattr check failed: %w", err)
+		}
+	}
+
+	be, err := posix.New(gwroot, ms, opts)
 	if err != nil {
-		return fmt.Errorf("init posix: %v", err)
+		return fmt.Errorf("failed to init posix backend: %w", err)
 	}
 
 	return runGateway(ctx.Context, be)

cmd/versitygw/scoutfs.go

@@ -16,13 +16,16 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/scoutfs"
 )
 
 var (
-	glacier bool
+	glacier          bool
+	disableNoArchive bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -63,6 +66,26 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_CHOWN_GID"},
 				Destination: &chowngid,
 			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
+			&cli.BoolFlag{
+				Name:        "disable-noarchive",
+				Usage:       "disable setting noarchive for multipart part uploads",
+				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
+				Destination: &disableNoArchive,
+			},
 		},
 	}
 }
@@ -72,10 +95,17 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
 	var opts scoutfs.ScoutfsOpts
 	opts.GlacierMode = glacier
 	opts.ChownUID = chownuid
 	opts.ChownGID = chowngid
+	opts.BucketLinks = bucketlinks
+	opts.NewDirPerm = fs.FileMode(dirPerms)
+	opts.DisableNoArchive = disableNoArchive
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

cmd/versitygw/test.go

@@ -22,20 +22,23 @@ import (
 )
 
 var (
-	awsID           string
-	awsSecret       string
-	endpoint        string
-	prefix          string
-	dstBucket       string
-	partSize        int64
-	objSize         int64
-	concurrency     int
-	files           int
-	totalReqs       int
-	upload          bool
-	download        bool
-	pathStyle       bool
-	checksumDisable bool
+	awsID             string
+	awsSecret         string
+	endpoint          string
+	prefix            string
+	dstBucket         string
+	partSize          int64
+	objSize           int64
+	concurrency       int
+	files             int
+	totalReqs         int
+	upload            bool
+	download          bool
+	pathStyle         bool
+	checksumDisable   bool
+	versioningEnabled bool
+	azureTests        bool
+	tlsStatus         bool
 )
 
 func testCommand() *cli.Command {
@@ -77,6 +80,12 @@ func initTestFlags() []cli.Flag {
 			Aliases:     []string{"d"},
 			Destination: &debug,
 		},
+		&cli.BoolFlag{
+			Name:        "allow-insecure",
+			Usage:       "skip tls verification",
+			Aliases:     []string{"ai"},
+			Destination: &tlsStatus,
+		},
 	}
 }
 
@@ -87,11 +96,33 @@ func initTestCommands() []*cli.Command {
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+				&cli.BoolFlag{
+					Name:        "azure-test-mode",
+					Usage:       "Skips tests that are not supported by Azure",
+					Destination: &azureTests,
+					Aliases:     []string{"azure"},
+				},
+			},
 		},
 		{
 			Name:   "posix",
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test posix when versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		},
 		{
 			Name:   "iam",
@@ -187,6 +218,7 @@ func initTestCommands() []*cli.Command {
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
 					integration.WithPartSize(partSize),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -247,6 +279,7 @@ func initTestCommands() []*cli.Command {
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -272,10 +305,17 @@ func getAction(tf testFunc) func(*cli.Context) error {
 			integration.WithSecret(awsSecret),
 			integration.WithRegion(region),
 			integration.WithEndpoint(endpoint),
+			integration.WithTLSStatus(tlsStatus),
 		}
 		if debug {
 			opts = append(opts, integration.WithDebug())
 		}
+		if versioningEnabled {
+			opts = append(opts, integration.WithVersioningEnabled())
+		}
+		if azureTests {
+			opts = append(opts, integration.WithAzureMode())
+		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -303,15 +343,27 @@ func extractIntTests() (commands []*cli.Command) {
 					integration.WithSecret(awsSecret),
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
+					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
+				if versioningEnabled {
+					opts = append(opts, integration.WithVersioningEnabled())
+				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)
 				return err
 			},
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		})
 	}
 	return

docker-compose-bats.yml

@@ -1,35 +0,0 @@
-version: '3'
-
-services:
-  no_certs:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.nocerts
-  static_buckets:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.static
-  posix_backend:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.default
-  s3_backend:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.s3
-        - SECRETS_FILE=tests/.secrets.s3
-  direct:
-    build:
-      context: .
-      dockerfile: Dockerfile_test_bats
-      args:
-        - CONFIG_FILE=tests/.env.direct
-        - SECRETS_FILE=tests/.secrets.direct

extra/dashboard/README.md

@@ -0,0 +1,19 @@
+# Versity Gateway Dashboard
+
+This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana. 
+
+The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
+
+This implementation uses the influxql query language. 
+
+## Usage
+
+From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
+
+To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`. 
+
+The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
+
+The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with  username: `admin` and password: `admin`.
+
+To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

extra/dashboard/aws_env_setup.sh

@@ -0,0 +1,4 @@
+export AWS_SECRET_ACCESS_KEY=password
+export AWS_ACCESS_KEY_ID=user
+export AWS_ENDPOINT_URL=http://127.0.0.1:7070
+export AWS_REGION=us-east-1

extra/dashboard/docker-compose-metrics.yml

@@ -0,0 +1,64 @@
+services:
+  telegraf:
+    image: telegraf
+    container_name: telegraf
+    restart: always
+    volumes:
+    - ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      - influxdb
+    links:
+      - influxdb
+    ports:
+    - '8125:8125/udp'
+
+  influxdb:
+    image: influxdb
+    container_name: influxdb
+    restart: always
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
+      - DOCKER_INFLUXDB_INIT_ORG=myorg
+      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb_data:/var/lib/influxdb
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana-server
+    restart: always
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_INSTALL_PLUGINS=
+    links:
+      - influxdb
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./metrics-exploration/grafana_data:/var/lib/grafana
+      - ./metrics-exploration/provisioning:/etc/grafana/provisioning
+
+  versitygw:
+    image: versity/versitygw:latest
+    container_name: versitygw
+    ports:
+      - "7070:7070"
+    environment:
+      - ROOT_ACCESS_KEY=user
+      - ROOT_SECRET_KEY=password
+      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
+    depends_on:
+      - telegraf
+    command: >
+      posix /tmp/vgw
+
+volumes:
+  influxdb_data: {}

extra/dashboard/docker-compose.yml

@@ -0,0 +1,64 @@
+services:
+  telegraf:
+    image: telegraf
+    container_name: telegraf
+    restart: always
+    volumes:
+    - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      - influxdb
+    links:
+      - influxdb
+    ports:
+    - '8125:8125/udp'
+
+  influxdb:
+    image: influxdb
+    container_name: influxdb
+    restart: always
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
+      - DOCKER_INFLUXDB_INIT_ORG=myorg
+      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb_data:/var/lib/influxdb
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana-server
+    restart: always
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_INSTALL_PLUGINS=
+    links:
+      - influxdb
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./grafana_data:/var/lib/grafana
+      - ./provisioning:/etc/grafana/provisioning
+
+  versitygw:
+    image: versity/versitygw:latest
+    container_name: versitygw
+    ports:
+      - "7070:7070"
+    environment:
+      - ROOT_ACCESS_KEY=user
+      - ROOT_SECRET_KEY=password
+      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
+    depends_on:
+      - telegraf
+    command: >
+      posix /tmp/vgw
+
+volumes:
+  influxdb_data: {}

extra/dashboard/provisioning/dashboards/influxql.yaml

@@ -0,0 +1,25 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'influxql'
+    # <int> Org id. Default to 1
+    orgId: 1
+    # <string> name of the dashboard folder.
+    folder: 'influxql'
+    # <string> folder UID. will be automatically generated if not specified
+    folderUid: ''
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: true
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /etc/grafana/provisioning/dashboards/influxql
+      # <bool> use folder names from filesystem to create folders in Grafana
+      foldersFromFilesStructure: true
+

extra/dashboard/provisioning/datasources/influxdb.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+datasources:
+  - name: influxdb
+    type: influxdb
+    isDefault: true
+    access: proxy
+    url: http://influxdb:8086
+    jsonData:
+      dbName: 'metrics'
+      httpHeaderName1: 'Authorization'
+    secureJsonData:
+      httpHeaderValue1: 'Token my-super-secret-auth-token'

extra/dashboard/telegraf.conf

@@ -0,0 +1,34 @@
+[global_tags]
+
+[agent]
+  debug = true
+  quiet = false
+  interval = "60s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  hostname = "versitygw"
+  omit_hostname = false
+
+[[outputs.file]]
+  files = ["stdout"]
+
+[[outputs.influxdb_v2]]
+  urls = ["http://influxdb:8086"]
+  timeout = "5s"
+  token = "my-super-secret-auth-token"
+  organization = "myorg"
+  bucket = "metrics"
+
+[[inputs.statsd]]
+  protocol = "udp4"
+  service_address = ":8125"
+  percentiles = [90]
+  metric_separator = "_"
+  datadog_extensions = false
+  allowed_pending_messages = 10000
+  percentile_limit = 1000

extra/dashboard/test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+. ./aws_env_setup.sh
+
+aws s3 mb s3://test
+aws s3 cp docker-compose.yml s3://test/test.yaml

extra/example.conf

@@ -237,6 +237,8 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_LDAP_ACCESS_ATR=
 #VGW_IAM_LDAP_SECRET_ATR=
 #VGW_IAM_LDAP_ROLE_ATR=
+#VGW_IAM_LDAP_USER_ID_ATR=
+#VGW_IAM_LDAP_GROUP_ID_ATR=
 
 ###############
 # IAM caching #
@@ -305,6 +307,16 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_CHOWN_UID=false
 #VGW_CHOWN_GID=false
 
+# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
+# to directories at the top level gateway directory as buckets.
+#VGW_BUCKET_LINKS=false
+
+# The default permissions mode when creating new directories is 0755. Use
+# VGW_DIR_PERMS option to set a different mode for any new directory that the
+# gateway creates. This applies to buckets created through the gateway as well
+# as any parent directories automatically created with object uploads.
+#VGW_DIR_PERMS=0755
+
 ###########
 # scoutfs #
 ###########
@@ -336,6 +348,23 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_CHOWN_UID=false
 #VGW_CHOWN_GID=false
 
+# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
+# to directories at the top level gateway directory as buckets.
+#VGW_BUCKET_LINKS=false
+
+# The default permissions mode when creating new directories is 0755. Use
+# VGW_DIR_PERMS option to set a different mode for any new directory that the
+# gateway creates. This applies to buckets created through the gateway as well
+# as any parent directories automatically created with object uploads.
+#VGW_DIR_PERMS=0755
+
+# The default behavior of the gateway is to automatically set the noarchive
+# flag on the multipart upload parts while the multipart upload is in progress.
+# This is to prevent the parts from being archived since they are temporary
+# and will be deleted after the multipart upload is completed or aborted. The
+# VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
+#VGW_DISABLE_NOARCHIVE=false
+
 ######
 # s3 #
 ######

go.mod

@@ -1,81 +1,83 @@
 module github.com/versity/versitygw
 
-go 1.21.0
+go 1.23.0
+
+toolchain go1.23.6
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
-	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.30.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1
-	github.com/aws/smithy-go v1.20.2
-	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/gofiber/fiber/v2 v2.52.4
-	github.com/google/go-cmp v0.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
+	github.com/DataDog/datadog-go/v5 v5.6.0
+	github.com/aws/aws-sdk-go-v2 v1.36.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2
+	github.com/aws/smithy-go v1.22.3
+	github.com/go-ldap/ldap/v3 v3.4.10
+	github.com/gofiber/fiber/v2 v2.52.6
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.36.0
-	github.com/pkg/xattr v0.4.9
+	github.com/nats-io/nats.go v1.39.1
+	github.com/oklog/ulid/v2 v2.1.0
+	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
-	github.com/smira/go-statsd v1.3.3
-	github.com/urfave/cli/v2 v2.27.2
-	github.com/valyala/fasthttp v1.55.0
+	github.com/smira/go-statsd v1.3.4
+	github.com/urfave/cli/v2 v2.27.6
+	github.com/valyala/fasthttp v1.59.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sys v0.21.0
+	golang.org/x/sync v0.12.0
+	golang.org/x/sys v0.31.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.21.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.25.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.29.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nkeys v0.4.10 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.21
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.21
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.1
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.9
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.62
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 )

go.sum

@@ -1,83 +1,91 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2/go.mod h1:SqINnQ9lVVdRlyC8cd1lCI0SdX4n2paeABd2K8ggfnE=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
-github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.1 h1:8BKxhZZLX/WosEeoCvWysmKUscfa9v8LIPEEU0JjE2o=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
+github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
-github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.30.0 h1:6qAwtzlfcTtcL8NHtbDQAqgM5s6NDipQTkPxyH/6kAA=
-github.com/aws/aws-sdk-go-v2 v1.30.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
-github.com/aws/aws-sdk-go-v2/config v1.27.21 h1:yPX3pjGCe2hJsetlmGNB4Mngu7UPmvWPzzWCv1+boeM=
-github.com/aws/aws-sdk-go-v2/config v1.27.21/go.mod h1:4XtlEU6DzNai8RMbjSF5MgGZtYvrhBP/aKZcRtZAVdM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.21 h1:pjAqgzfgFhTv5grc7xPHtXCAaMapzmwA7aU+c/SZQGw=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.21/go.mod h1:nhK6PtBlfHTUDVmBLr1dg+WHCOCK+1Fu/WQyVHPsgNQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8 h1:FR+oWPFb/8qMVYMWN98bUZAGqPvLHiyqg1wqQGfUAXY=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8/go.mod h1:EgSKcHiuuakEIxJcKGzVNWh5srVAQ3jKaSrBGRYvM48=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.1 h1:D9VqWMuw7lJAX6d5eINfRQ/PkvtcJAK3Qmd6f6xEeUw=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.1/go.mod h1:ckvBx7codI4wzc5inOfDp5ZbK7TjMFa7eXwmLvXQrRk=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.12 h1:SJ04WXGTwnHlWIODtC5kJzKbeuHt+OUNOgKg7nfnUGw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.12/go.mod h1:FkpvXhA92gb3GE9LD6Og0pHHycTxW7xGpnEh5E7Opwo=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.12 h1:hb5KgeYfObi5MHkSSZMEudnIvX30iB+E21evI4r6BnQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.12/go.mod h1:CroKe/eWJdyfy9Vx4rljP5wTUjNJfb+fPz1uMYUhEGM=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 h1:DXFWyt7ymx/l1ygdyTTS0X923e+Q2wXIxConJzrgwc0=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12/go.mod h1:mVOr/LbvaNySK1/BTy4cBOCjhCNY2raWBwK4v+WR5J4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 h1:oWccitSnByVU74rQRHac4gLfDqjB6Z1YQGOY/dXKedI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14/go.mod h1:8SaZBlQdCLrc/2U3CEO48rYj9uR8qRsPRkmzwNM52pM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14 h1:zSDPny/pVnkqABXYRicYuPf9z2bTqfH13HT3v6UheIk=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14/go.mod h1:3TTcI5JSzda1nw/pkVC9dhgLre0SNBFj2lYS4GctXKI=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12 h1:tzha+v1SCEBpXWEuw6B/+jm4h5z8hZbTpXz0zRZqTnw=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12/go.mod h1:n+nt2qjHGoseWeLHt1vEr6ZRCCxIN2KcNpJxBcYQSwI=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1 h1:wsg9Z/vNnCmxWikfGIoOlnExtEU459cR+2d+iDJ8elo=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1/go.mod h1:8rDw3mVwmvIWWX/+LWY3PPIMZuwnQdJMCt0iVFVT3qw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.21.1 h1:sd0BsnAvLH8gsp2e3cbaIr+9D7T1xugueQ7V/zUAsS4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.21.1/go.mod h1:lcQG/MmxydijbeTOp04hIuJwXGWPZGI3bwdFDGRTv14=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.25.1 h1:1uEFNNskK/I1KoZ9Q8wJxMz5V9jyBlsiaNrM7vA3YUQ=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.25.1/go.mod h1:z0P8K+cBIsFXUr5rzo/psUeJ20XjPN0+Nn8067Nd+E4=
-github.com/aws/aws-sdk-go-v2/service/sts v1.29.1 h1:myX5CxqXE0QMZNja6FA1/FSE3Vu1rVmeUmpJMMzeZg0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.29.1/go.mod h1:N2mQiucsO0VwK9CYuS4/c2n6Smeh1v47Rz3dWCPFLdE=
-github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
-github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
+github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
+github.com/aws/aws-sdk-go-v2/config v1.29.9 h1:Kg+fAYNaJeGXp1vmjtidss8O2uXIsXwaRqsQJKXVr+0=
+github.com/aws/aws-sdk-go-v2/config v1.29.9/go.mod h1:oU3jj2O53kgOU4TXq/yipt6ryiooYjlkqqVaZk7gY/U=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBajbBWGqjNTCa79ocU=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66 h1:MTLivtC3s89de7Fe3P8rzML/8XPNRfuyJhlRTsCEt0k=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66/go.mod h1:NAuQ2s6gaFEsuTIb2+P5t6amB1w5MhvJFxppoezGWH0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2 h1:jIiopHEV22b4yQP2q36Y0OmwLbsxNWdWwfZRR5QRRO4=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 h1:8JdC7Gr9NROg1Rusk25IcZeTO59zLxsKgE0gkh5O6h0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.1/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 h1:KwuLovgQPcdjNMfFt9OhUd9a2OwcOKhxfvF4glTzLuA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
+github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
 github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
-github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
-github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
+github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
+github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
+github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
@@ -109,40 +117,42 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU=
-github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
-github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
-github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nats.go v1.39.1 h1:oTkfKBmz7W047vRxV762M67ZdXeOtUgvbBaNoQ+3PPk=
+github.com/nats-io/nats.go v1.39.1/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM=
+github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc=
+github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
+github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
+github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
-github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
-github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
+github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -153,8 +163,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
-github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
+github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
+github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
@@ -164,16 +174,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
-github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
+github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
-github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
-github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
-github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/valyala/fasthttp v1.59.0 h1:Qu0qYHfXvPk1mSLNqcFtEk6DpxgA26hy6bmydotDpRI=
+github.com/valyala/fasthttp v1.59.0/go.mod h1:GTxNb9Bc6r2a9D0TWNSPwDz78UxnTGBViY3xZNEqyYU=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -182,22 +190,29 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -207,15 +222,23 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -227,23 +250,27 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -252,23 +279,24 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

metrics/actions.go

@@ -24,51 +24,62 @@ var (
 )
 
 var (
-	ActionUndetected                 = "ActionUnDetected"
-	ActionAbortMultipartUpload       = "s3_AbortMultipartUpload"
-	ActionCompleteMultipartUpload    = "s3_CompleteMultipartUpload"
-	ActionCopyObject                 = "s3_CopyObject"
-	ActionCreateBucket               = "s3_CreateBucket"
-	ActionCreateMultipartUpload      = "s3_CreateMultipartUpload"
-	ActionDeleteBucket               = "s3_DeleteBucket"
-	ActionDeleteBucketPolicy         = "s3_DeleteBucketPolicy"
-	ActionDeleteBucketTagging        = "s3_DeleteBucketTagging"
-	ActionDeleteObject               = "s3_DeleteObject"
-	ActionDeleteObjectTagging        = "s3_DeleteObjectTagging"
-	ActionDeleteObjects              = "s3_DeleteObjects"
-	ActionGetBucketAcl               = "s3_GetBucketAcl"
-	ActionGetBucketPolicy            = "s3_GetBucketPolicy"
-	ActionGetBucketTagging           = "s3_GetBucketTagging"
-	ActionGetBucketVersioning        = "s3_GetBucketVersioning"
-	ActionGetObject                  = "s3_GetObject"
-	ActionGetObjectAcl               = "s3_GetObjectAcl"
-	ActionGetObjectAttributes        = "s3_GetObjectAttributes"
-	ActionGetObjectLegalHold         = "s3_GetObjectLegalHold"
-	ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
-	ActionGetObjectRetention         = "s3_GetObjectRetention"
-	ActionGetObjectTagging           = "s3_GetObjectTagging"
-	ActionHeadBucket                 = "s3_HeadBucket"
-	ActionHeadObject                 = "s3_HeadObject"
-	ActionListAllMyBuckets           = "s3_ListAllMyBuckets"
-	ActionListMultipartUploads       = "s3_ListMultipartUploads"
-	ActionListObjectVersions         = "s3_ListObjectVersions"
-	ActionListObjects                = "s3_ListObjects"
-	ActionListObjectsV2              = "s3_ListObjectsV2"
-	ActionListParts                  = "s3_ListParts"
-	ActionPutBucketAcl               = "s3_PutBucketAcl"
-	ActionPutBucketPolicy            = "s3_PutBucketPolicy"
-	ActionPutBucketTagging           = "s3_PutBucketTagging"
-	ActionPutBucketVersioning        = "s3_PutBucketVersioning"
-	ActionPutObject                  = "s3_PutObject"
-	ActionPutObjectAcl               = "s3_PutObjectAcl"
-	ActionPutObjectLegalHold         = "s3_PutObjectLegalHold"
-	ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
-	ActionPutObjectRetention         = "s3_PutObjectRetention"
-	ActionPutObjectTagging           = "s3_PutObjectTagging"
-	ActionRestoreObject              = "s3_RestoreObject"
-	ActionSelectObjectContent        = "s3_SelectObjectContent"
-	ActionUploadPart                 = "s3_UploadPart"
-	ActionUploadPartCopy             = "s3_UploadPartCopy"
+	ActionUndetected                    = "ActionUnDetected"
+	ActionAbortMultipartUpload          = "s3_AbortMultipartUpload"
+	ActionCompleteMultipartUpload       = "s3_CompleteMultipartUpload"
+	ActionCopyObject                    = "s3_CopyObject"
+	ActionCreateBucket                  = "s3_CreateBucket"
+	ActionCreateMultipartUpload         = "s3_CreateMultipartUpload"
+	ActionDeleteBucket                  = "s3_DeleteBucket"
+	ActionDeleteBucketPolicy            = "s3_DeleteBucketPolicy"
+	ActionDeleteBucketTagging           = "s3_DeleteBucketTagging"
+	ActionDeleteObject                  = "s3_DeleteObject"
+	ActionDeleteObjectTagging           = "s3_DeleteObjectTagging"
+	ActionDeleteObjects                 = "s3_DeleteObjects"
+	ActionGetBucketAcl                  = "s3_GetBucketAcl"
+	ActionGetBucketPolicy               = "s3_GetBucketPolicy"
+	ActionGetBucketTagging              = "s3_GetBucketTagging"
+	ActionGetBucketVersioning           = "s3_GetBucketVersioning"
+	ActionGetObject                     = "s3_GetObject"
+	ActionGetObjectAcl                  = "s3_GetObjectAcl"
+	ActionGetObjectAttributes           = "s3_GetObjectAttributes"
+	ActionGetObjectLegalHold            = "s3_GetObjectLegalHold"
+	ActionGetObjectLockConfiguration    = "s3_GetObjectLockConfiguration"
+	ActionGetObjectRetention            = "s3_GetObjectRetention"
+	ActionGetObjectTagging              = "s3_GetObjectTagging"
+	ActionHeadBucket                    = "s3_HeadBucket"
+	ActionHeadObject                    = "s3_HeadObject"
+	ActionListAllMyBuckets              = "s3_ListAllMyBuckets"
+	ActionListMultipartUploads          = "s3_ListMultipartUploads"
+	ActionListObjectVersions            = "s3_ListObjectVersions"
+	ActionListObjects                   = "s3_ListObjects"
+	ActionListObjectsV2                 = "s3_ListObjectsV2"
+	ActionListParts                     = "s3_ListParts"
+	ActionPutBucketAcl                  = "s3_PutBucketAcl"
+	ActionPutBucketPolicy               = "s3_PutBucketPolicy"
+	ActionPutBucketTagging              = "s3_PutBucketTagging"
+	ActionPutBucketVersioning           = "s3_PutBucketVersioning"
+	ActionPutObject                     = "s3_PutObject"
+	ActionPutObjectAcl                  = "s3_PutObjectAcl"
+	ActionPutObjectLegalHold            = "s3_PutObjectLegalHold"
+	ActionPutObjectLockConfiguration    = "s3_PutObjectLockConfiguration"
+	ActionPutObjectRetention            = "s3_PutObjectRetention"
+	ActionPutObjectTagging              = "s3_PutObjectTagging"
+	ActionRestoreObject                 = "s3_RestoreObject"
+	ActionSelectObjectContent           = "s3_SelectObjectContent"
+	ActionUploadPart                    = "s3_UploadPart"
+	ActionUploadPartCopy                = "s3_UploadPartCopy"
+	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
+	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
+	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
+
+	// Admin actions
+	ActionAdminCreateUser        = "admin_CreateUser"
+	ActionAdminUpdateUser        = "admin_UpdateUser"
+	ActionAdminDeleteUser        = "admin_DeleteUser"
+	ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
+	ActionAdminListUsers         = "admin_ListUsers"
+	ActionAdminListBuckets       = "admin_ListBuckets"
 )
 
 func init() {

runtests.sh

@@ -5,15 +5,27 @@ rm -rf /tmp/gw
 mkdir /tmp/gw
 rm -rf /tmp/covdata
 mkdir /tmp/covdata
+rm -rf /tmp/versioing.covdata
+mkdir /tmp/versioning.covdata
+rm -rf /tmp/versioningdir
+mkdir /tmp/versioningdir
 
-# run server in background
+# setup tls certificate and key
+ECHO "Generating TLS certificate and key in the cert.pem and key.pem files"
+
+openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
+openssl req -new -x509 -key key.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+
+ECHO "Running the sdk test over http"
+# run server in background not versioning-enabled
+# port: 7070(default)
 GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
 GW_PID=$!
 
-# wait a second for server to start up
 sleep 1
 
-# check if server is still running
+# check if gateway process is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -39,8 +51,110 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
 	exit 1
 fi
 
-# kill off server
 kill $GW_PID
+
+ECHO "Running the sdk test over https"
+
+# run server in background with TLS certificate
+# port: 7071(default)
+GOCOVERDIR=/tmp/https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7071 -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
+GW_HTTPS_PID=$!
+
+sleep 1
+
+# check if https gateway process is still running
+if ! kill -0 $GW_HTTPS_PID; then
+	echo "server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow; then
+	echo "full flow tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 posix; then
+	echo "posix tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+# iam tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 iam; then
+	echo "iam tests failed"
+	kill $GW_HTTPS_PID
+	exit 1
+fi
+
+kill $GW_HTTPS_PID
+
+
+ECHO "Running the sdk test over http against the versioning-enabled gateway"
+# run server in background versioning-enabled
+# port: 7072
+GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7072 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+GW_VS_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if versioning-enabled gateway process is still running
+if ! kill -0 $GW_VS_PID; then
+	echo "versioning-enabled server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs; then
+	echo "versioning-enabled full-flow tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 posix -vs; then
+	echo "versiongin-enabled posix tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_VS_PID
+
+ECHO "Running the sdk test over https against the versioning-enabled gateway"
+# run server in background versioning-enabled
+# port: 7073
+GOCOVERDIR=/tmp/versioning.https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7073 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+GW_VS_HTTPS_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if versioning-enabled gateway process is still running
+if ! kill -0 $GW_VS_HTTPS_PID; then
+	echo "versioning-enabled server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs; then
+	echo "versioning-enabled full-flow tests failed"
+	kill $GW_VS_HTTPS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 posix -vs; then
+	echo "versiongin-enabled posix tests failed"
+	kill $GW_VS_HTTPS_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_VS_HTTPS_PID
+
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-router.go

@@ -19,12 +19,13 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminRouter struct{}
 
-func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
-	controller := controllers.NewAdminController(iam, be)
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
+	controller := controllers.NewAdminController(iam, be, logger)
 
 	// CreateUser admin api
 	app.Patch("/create-user", controller.CreateUser)

s3api/admin-server.go

@@ -22,6 +22,7 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminServer struct {
@@ -32,7 +33,7 @@ type S3AdminServer struct {
 	cert    *tls.Certificate
 }
 
-func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
 	server := &S3AdminServer{
 		app:     app,
 		backend: be,
@@ -46,13 +47,16 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 
 	// Logging middlewares
 	app.Use(logger.New())
-	app.Use(middlewares.DecodeURL(nil, nil))
+	app.Use(middlewares.DecodeURL(l, nil))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyV4Signature(root, iam, nil, nil, region, false))
-	app.Use(middlewares.VerifyMD5Body(nil))
+	app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(l))
 
-	server.router.Init(app, be, iam)
+	// Admin role checker
+	app.Use(middlewares.IsAdmin(l))
+
+	server.router.Init(app, be, iam, l)
 
 	return server
 }

s3api/controllers/admin.go

@@ -16,146 +16,188 @@ package controllers
 
 import (
 	"encoding/json"
+	"encoding/xml"
 	"fmt"
+	"net/http"
 	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+	"github.com/versity/versitygw/s3response"
 )
 
 type AdminController struct {
 	iam auth.IAMService
 	be  backend.Backend
+	l   s3log.AuditLogger
 }
 
-func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
-	return AdminController{iam: iam, be: be}
+func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
+	return AdminController{iam: iam, be: be, l: l}
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
-	}
 	var usr auth.Account
-	err := json.Unmarshal(ctx.Body(), &usr)
+	err := xml.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("failed to parse request body: %w", err).Error())
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
+			})
 	}
 
-	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return ctx.Status(fiber.StatusBadRequest).SendString("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
+	if !usr.Role.IsValid() {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
+			})
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		status := fiber.StatusInternalServerError
-		msg := fmt.Errorf("failed to create user: %w", err).Error()
-
-		if strings.Contains(msg, "user already exists") {
-			status = fiber.StatusConflict
+		if strings.Contains(err.Error(), "user already exists") {
+			err = s3err.GetAPIError(s3err.ErrAdminUserExists)
 		}
 
-		return ctx.Status(status).SendString(msg)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
+			})
 	}
 
-	return ctx.Status(fiber.StatusCreated).SendString("The user has been created successfully")
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminCreateUser,
+			Status: http.StatusCreated,
+		})
 }
 
 func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
-	}
-
 	access := ctx.Query("access")
 	if access == "" {
-		return ctx.Status(fiber.StatusBadRequest).SendString("missing user access parameter")
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
 	}
 
 	var props auth.MutableProps
-	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
-		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("invalid request body %w", err).Error())
+	if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
 	}
 
 	err := c.iam.UpdateUserAccount(access, props)
 	if err != nil {
-		status := fiber.StatusInternalServerError
-		msg := fmt.Errorf("failed to update user account: %w", err).Error()
-
-		if strings.Contains(msg, "user not found") {
-			status = fiber.StatusNotFound
+		if strings.Contains(err.Error(), "user not found") {
+			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
 		}
 
-		return ctx.Status(status).SendString(msg)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
 	}
 
-	return ctx.SendString("the user has been updated successfully")
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminUpdateUser,
+		})
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
-	}
 
 	err := c.iam.DeleteUserAccount(access)
-	if err != nil {
-		return err
-	}
-
-	return ctx.SendString("The user has been deleted successfully")
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminDeleteUser,
+		})
 }
 
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
-	}
 	accs, err := c.iam.ListUserAccounts()
-	if err != nil {
-		return err
-	}
-
-	return ctx.JSON(accs)
+	return SendXMLResponse(ctx,
+		auth.ListUserAccountsResult{
+			Accounts: accs,
+		}, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListUsers,
+		})
 }
 
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
-	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
 
 	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
 	if err != nil {
-		return err
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
+			})
 	}
 	if len(accs) > 0 {
-		return ctx.Status(fiber.StatusNotFound).SendString("user specified as the new bucket owner does not exist")
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
+			})
 	}
 
-	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
+	acl := auth.ACL{
+		Owner: owner,
+		Grantees: []auth.Grantee{
+			{
+				Permission: auth.PermissionFullControl,
+				Access:     owner,
+				Type:       types.TypeCanonicalUser,
+			},
+		},
+	}
+
+	aclParsed, err := json.Marshal(acl)
 	if err != nil {
-		return err
+		return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
+			})
 	}
 
-	return ctx.SendString("Bucket owner has been updated successfully")
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminChangeBucketOwner,
+		})
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
-	}
-
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
-	if err != nil {
-		return err
-	}
-
-	return ctx.JSON(buckets)
+	return SendXMLResponse(ctx,
+		s3response.ListBucketsResult{
+			Buckets: buckets,
+		}, err, &MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListBuckets,
+		})
 }

s3api/controllers/admin_test.go

@@ -15,12 +15,11 @@
 package controllers
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
@@ -43,33 +42,26 @@ func TestAdminController_CreateUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/create-user", adminController.CreateUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	usr := auth.Account{
-		Access: "access",
-		Secret: "secret",
-		Role:   "invalid role",
-	}
-
-	user, _ := json.Marshal(&usr)
-
-	usr.Role = "admin"
-
-	succUsr, _ := json.Marshal(&usr)
-
-	appErr.Patch("/create-user", adminController.CreateUser)
+	succUser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>admin</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+	invuser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>invalid_role</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
 
 	tests := []struct {
 		name       string
@@ -79,31 +71,31 @@ func TestAdminController_CreateUser(t *testing.T) {
 		statusCode int
 	}{
 		{
-			name: "Admin-create-user-success",
+			name: "Admin-create-user-malformed-body",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
-			},
-			wantErr:    false,
-			statusCode: 201,
-		},
-		{
-			name: "Admin-create-user-invalid-user-role",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
 			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
-			app:  appErr,
+			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
 			},
 			wantErr:    false,
-			statusCode: 403,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-create-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
+			},
+			wantErr:    false,
+			statusCode: 201,
 		},
 	}
 	for _, tt := range tests {
@@ -134,24 +126,8 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/update-user", adminController.UpdateUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appErr.Patch("/update-user", adminController.UpdateUser)
-
-	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
-
 	adminControllerErr := AdminController{
 		iam: &IAMServiceMock{
 			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
@@ -162,13 +138,16 @@ func TestAdminController_UpdateUser(t *testing.T) {
 
 	appNotFound := fiber.New()
 
-	appNotFound.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
 
+	succUser := `
+		<Account>
+			<Secret>secret</Secret>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -180,7 +159,7 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			name: "Admin-update-user-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
 			statusCode: 200,
@@ -189,10 +168,10 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			name: "Admin-update-user-missing-access",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
-			statusCode: 400,
+			statusCode: 404,
 		},
 		{
 			name: "Admin-update-user-invalid-request-body",
@@ -203,20 +182,11 @@ func TestAdminController_UpdateUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "Admin-update-user-invalid-requester-role",
-			app:  appErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Admin-update-user-not-found",
 			app:  appNotFound,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
 			},
 			wantErr:    false,
 			statusCode: 404,
@@ -250,22 +220,8 @@ func TestAdminController_DeleteUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/delete-user", adminController.DeleteUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appErr.Patch("/delete-user", adminController.DeleteUser)
-
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -282,15 +238,6 @@ func TestAdminController_DeleteUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
-		{
-			name: "Admin-delete-user-invalid-requester-role",
-			app:  appErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -327,30 +274,9 @@ func TestAdminController_ListUsers(t *testing.T) {
 	}
 
 	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appErr.Patch("/list-users", adminControllerErr.ListUsers)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-users", adminController.ListUsers)
-
 	appSucc := fiber.New()
-
-	appSucc.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
@@ -360,15 +286,6 @@ func TestAdminController_ListUsers(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
-		{
-			name: "Admin-list-users-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Admin-list-users-iam-error",
 			app:  appErr,
@@ -407,7 +324,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 	adminController := AdminController{
 		be: &BackendMock{
-			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
+			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
 				return nil
 			},
 		},
@@ -435,39 +352,12 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
-
 	appIamErr := fiber.New()
-
-	appIamErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
 
 	appIamNoSuchUser := fiber.New()
-
-	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
@@ -477,15 +367,6 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
-		{
-			name: "Change-bucket-owner-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
 			app:  appIamErr,
@@ -540,23 +421,8 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
-
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -564,15 +430,6 @@ func TestAdminController_ListBuckets(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
-		{
-			name: "List-buckets-incorrect-role",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
-			},
-			wantErr:    false,
-			statusCode: 403,
-		},
 		{
 			name: "List-buckets-success",
 			app:  app,

s3api/controllers/backend_moq_test.go

@@ -7,9 +7,9 @@ import (
 	"bufio"
 	"context"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3response"
-	"io"
 	"sync"
 )
 
@@ -26,31 +26,34 @@ var _ backend.Backend = &BackendMock{}
 //			AbortMultipartUploadFunc: func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error {
 //				panic("mock out the AbortMultipartUpload method")
 //			},
-//			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, newOwner string) error {
+//			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
 //				panic("mock out the ChangeBucketOwner method")
 //			},
 //			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 //				panic("mock out the CompleteMultipartUpload method")
 //			},
-//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+//			CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 //				panic("mock out the CopyObject method")
 //			},
 //			CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
 //				panic("mock out the CreateBucket method")
 //			},
-//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+//			CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 //				panic("mock out the CreateMultipartUpload method")
 //			},
-//			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
+//			DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketOwnershipControls method")
+//			},
 //			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketPolicy method")
 //			},
 //			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketTagging method")
 //			},
-//			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
+//			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
 //				panic("mock out the DeleteObject method")
 //			},
 //			DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error {
@@ -62,22 +65,25 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+//				panic("mock out the GetBucketOwnershipControls method")
+//			},
 //			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 //				panic("mock out the GetBucketPolicy method")
 //			},
 //			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 //				panic("mock out the GetBucketTagging method")
 //			},
-//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
 //				panic("mock out the GetBucketVersioning method")
 //			},
-//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+//			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 //				panic("mock out the GetObject method")
 //			},
 //			GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 //				panic("mock out the GetObjectAcl method")
 //			},
-//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+//			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 //				panic("mock out the GetObjectAttributes method")
 //			},
 //			GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error) {
@@ -98,7 +104,7 @@ var _ backend.Backend = &BackendMock{}
 //			HeadObjectFunc: func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 //				panic("mock out the HeadObject method")
 //			},
-//			ListBucketsFunc: func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+//			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 //				panic("mock out the ListBuckets method")
 //			},
 //			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
@@ -107,13 +113,13 @@ var _ backend.Backend = &BackendMock{}
 //			ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 //				panic("mock out the ListMultipartUploads method")
 //			},
-//			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+//			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
 //				panic("mock out the ListObjectVersions method")
 //			},
-//			ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+//			ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 //				panic("mock out the ListObjects method")
 //			},
-//			ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+//			ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 //				panic("mock out the ListObjectsV2 method")
 //			},
 //			ListPartsFunc: func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error) {
@@ -122,16 +128,19 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+//				panic("mock out the PutBucketOwnershipControls method")
+//			},
 //			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
 //				panic("mock out the PutBucketPolicy method")
 //			},
 //			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
 //				panic("mock out the PutBucketTagging method")
 //			},
-//			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+//			PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 //				panic("mock out the PutBucketVersioning method")
 //			},
-//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
+//			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 //				panic("mock out the PutObject method")
 //			},
 //			PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error {
@@ -161,10 +170,10 @@ var _ backend.Backend = &BackendMock{}
 //			StringFunc: func() string {
 //				panic("mock out the String method")
 //			},
-//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+//			UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 //				panic("mock out the UploadPart method")
 //			},
-//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+//			UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 //				panic("mock out the UploadPartCopy method")
 //			},
 //		}
@@ -178,22 +187,25 @@ type BackendMock struct {
 	AbortMultipartUploadFunc func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error
 
 	// ChangeBucketOwnerFunc mocks the ChangeBucketOwner method.
-	ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, newOwner string) error
+	ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, acl []byte) error
 
 	// CompleteMultipartUploadFunc mocks the CompleteMultipartUpload method.
 	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 
 	// CopyObjectFunc mocks the CopyObject method.
-	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
 
 	// CreateBucketFunc mocks the CreateBucket method.
 	CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error
 
 	// CreateMultipartUploadFunc mocks the CreateMultipartUpload method.
-	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 
 	// DeleteBucketFunc mocks the DeleteBucket method.
-	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
+	DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error
+
+	// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
+	DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
 
 	// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
 	DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
@@ -202,7 +214,7 @@ type BackendMock struct {
 	DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
 
 	// DeleteObjectFunc mocks the DeleteObject method.
-	DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error
+	DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
 
 	// DeleteObjectTaggingFunc mocks the DeleteObjectTagging method.
 	DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error
@@ -213,6 +225,9 @@ type BackendMock struct {
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketOwnershipControlsFunc mocks the GetBucketOwnershipControls method.
+	GetBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error)
+
 	// GetBucketPolicyFunc mocks the GetBucketPolicy method.
 	GetBucketPolicyFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
 
@@ -220,16 +235,16 @@ type BackendMock struct {
 	GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
 
 	// GetBucketVersioningFunc mocks the GetBucketVersioning method.
-	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
 
 	// GetObjectFunc mocks the GetObject method.
-	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 
 	// GetObjectAclFunc mocks the GetObjectAcl method.
 	GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 
 	// GetObjectAttributesFunc mocks the GetObjectAttributes method.
-	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
+	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
 
 	// GetObjectLegalHoldFunc mocks the GetObjectLegalHold method.
 	GetObjectLegalHoldFunc func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error)
@@ -250,7 +265,7 @@ type BackendMock struct {
 	HeadObjectFunc func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 
 	// ListBucketsFunc mocks the ListBuckets method.
-	ListBucketsFunc func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	ListBucketsFunc func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
 
 	// ListBucketsAndOwnersFunc mocks the ListBucketsAndOwners method.
 	ListBucketsAndOwnersFunc func(contextMoqParam context.Context) ([]s3response.Bucket, error)
@@ -259,13 +274,13 @@ type BackendMock struct {
 	ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 
 	// ListObjectVersionsFunc mocks the ListObjectVersions method.
-	ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
+	ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
 
 	// ListObjectsFunc mocks the ListObjects method.
-	ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 
 	// ListObjectsV2Func mocks the ListObjectsV2 method.
-	ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
 
 	// ListPartsFunc mocks the ListParts method.
 	ListPartsFunc func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error)
@@ -273,6 +288,9 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketOwnershipControlsFunc mocks the PutBucketOwnershipControls method.
+	PutBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error
+
 	// PutBucketPolicyFunc mocks the PutBucketPolicy method.
 	PutBucketPolicyFunc func(contextMoqParam context.Context, bucket string, policy []byte) error
 
@@ -280,10 +298,10 @@ type BackendMock struct {
 	PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error
 
 	// PutBucketVersioningFunc mocks the PutBucketVersioning method.
-	PutBucketVersioningFunc func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error
+	PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error
 
 	// PutObjectFunc mocks the PutObject method.
-	PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error)
+	PutObjectFunc func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error)
 
 	// PutObjectAclFunc mocks the PutObjectAcl method.
 	PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error
@@ -313,10 +331,10 @@ type BackendMock struct {
 	StringFunc func() string
 
 	// UploadPartFunc mocks the UploadPart method.
-	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error)
+	UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error)
 
 	// UploadPartCopyFunc mocks the UploadPartCopy method.
-	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
 
 	// calls tracks calls to the methods.
 	calls struct {
@@ -333,8 +351,8 @@ type BackendMock struct {
 			ContextMoqParam context.Context
 			// Bucket is the bucket argument value.
 			Bucket string
-			// NewOwner is the newOwner argument value.
-			NewOwner string
+			// ACL is the acl argument value.
+			ACL []byte
 		}
 		// CompleteMultipartUpload holds details about calls to the CompleteMultipartUpload method.
 		CompleteMultipartUpload []struct {
@@ -348,7 +366,7 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// CopyObjectInput is the copyObjectInput argument value.
-			CopyObjectInput *s3.CopyObjectInput
+			CopyObjectInput s3response.CopyObjectInput
 		}
 		// CreateBucket holds details about calls to the CreateBucket method.
 		CreateBucket []struct {
@@ -364,14 +382,21 @@ type BackendMock struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// CreateMultipartUploadInput is the createMultipartUploadInput argument value.
-			CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+			CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 		}
 		// DeleteBucket holds details about calls to the DeleteBucket method.
 		DeleteBucket []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// DeleteBucketInput is the deleteBucketInput argument value.
-			DeleteBucketInput *s3.DeleteBucketInput
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
+		// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
+		DeleteBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
 		}
 		// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
 		DeleteBucketPolicy []struct {
@@ -417,6 +442,13 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketOwnershipControls holds details about calls to the GetBucketOwnershipControls method.
+		GetBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetBucketPolicy holds details about calls to the GetBucketPolicy method.
 		GetBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -444,8 +476,6 @@ type BackendMock struct {
 			ContextMoqParam context.Context
 			// GetObjectInput is the getObjectInput argument value.
 			GetObjectInput *s3.GetObjectInput
-			// Writer is the writer argument value.
-			Writer io.Writer
 		}
 		// GetObjectAcl holds details about calls to the GetObjectAcl method.
 		GetObjectAcl []struct {
@@ -517,10 +547,8 @@ type BackendMock struct {
 		ListBuckets []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// Owner is the owner argument value.
-			Owner string
-			// IsAdmin is the isAdmin argument value.
-			IsAdmin bool
+			// ListBucketsInput is the listBucketsInput argument value.
+			ListBucketsInput s3response.ListBucketsInput
 		}
 		// ListBucketsAndOwners holds details about calls to the ListBucketsAndOwners method.
 		ListBucketsAndOwners []struct {
@@ -571,6 +599,15 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketOwnershipControls holds details about calls to the PutBucketOwnershipControls method.
+		PutBucketOwnershipControls []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Ownership is the ownership argument value.
+			Ownership types.ObjectOwnership
+		}
 		// PutBucketPolicy holds details about calls to the PutBucketPolicy method.
 		PutBucketPolicy []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -593,15 +630,17 @@ type BackendMock struct {
 		PutBucketVersioning []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
-			// PutBucketVersioningInput is the putBucketVersioningInput argument value.
-			PutBucketVersioningInput *s3.PutBucketVersioningInput
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Status is the status argument value.
+			Status types.BucketVersioningStatus
 		}
 		// PutObject holds details about calls to the PutObject method.
 		PutObject []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// PutObjectInput is the putObjectInput argument value.
-			PutObjectInput *s3.PutObjectInput
+			PutObjectInput s3response.PutObjectInput
 		}
 		// PutObjectAcl holds details about calls to the PutObjectAcl method.
 		PutObjectAcl []struct {
@@ -693,54 +732,57 @@ type BackendMock struct {
 			UploadPartCopyInput *s3.UploadPartCopyInput
 		}
 	}
-	lockAbortMultipartUpload       sync.RWMutex
-	lockChangeBucketOwner          sync.RWMutex
-	lockCompleteMultipartUpload    sync.RWMutex
-	lockCopyObject                 sync.RWMutex
-	lockCreateBucket               sync.RWMutex
-	lockCreateMultipartUpload      sync.RWMutex
-	lockDeleteBucket               sync.RWMutex
-	lockDeleteBucketPolicy         sync.RWMutex
-	lockDeleteBucketTagging        sync.RWMutex
-	lockDeleteObject               sync.RWMutex
-	lockDeleteObjectTagging        sync.RWMutex
-	lockDeleteObjects              sync.RWMutex
-	lockGetBucketAcl               sync.RWMutex
-	lockGetBucketPolicy            sync.RWMutex
-	lockGetBucketTagging           sync.RWMutex
-	lockGetBucketVersioning        sync.RWMutex
-	lockGetObject                  sync.RWMutex
-	lockGetObjectAcl               sync.RWMutex
-	lockGetObjectAttributes        sync.RWMutex
-	lockGetObjectLegalHold         sync.RWMutex
-	lockGetObjectLockConfiguration sync.RWMutex
-	lockGetObjectRetention         sync.RWMutex
-	lockGetObjectTagging           sync.RWMutex
-	lockHeadBucket                 sync.RWMutex
-	lockHeadObject                 sync.RWMutex
-	lockListBuckets                sync.RWMutex
-	lockListBucketsAndOwners       sync.RWMutex
-	lockListMultipartUploads       sync.RWMutex
-	lockListObjectVersions         sync.RWMutex
-	lockListObjects                sync.RWMutex
-	lockListObjectsV2              sync.RWMutex
-	lockListParts                  sync.RWMutex
-	lockPutBucketAcl               sync.RWMutex
-	lockPutBucketPolicy            sync.RWMutex
-	lockPutBucketTagging           sync.RWMutex
-	lockPutBucketVersioning        sync.RWMutex
-	lockPutObject                  sync.RWMutex
-	lockPutObjectAcl               sync.RWMutex
-	lockPutObjectLegalHold         sync.RWMutex
-	lockPutObjectLockConfiguration sync.RWMutex
-	lockPutObjectRetention         sync.RWMutex
-	lockPutObjectTagging           sync.RWMutex
-	lockRestoreObject              sync.RWMutex
-	lockSelectObjectContent        sync.RWMutex
-	lockShutdown                   sync.RWMutex
-	lockString                     sync.RWMutex
-	lockUploadPart                 sync.RWMutex
-	lockUploadPartCopy             sync.RWMutex
+	lockAbortMultipartUpload          sync.RWMutex
+	lockChangeBucketOwner             sync.RWMutex
+	lockCompleteMultipartUpload       sync.RWMutex
+	lockCopyObject                    sync.RWMutex
+	lockCreateBucket                  sync.RWMutex
+	lockCreateMultipartUpload         sync.RWMutex
+	lockDeleteBucket                  sync.RWMutex
+	lockDeleteBucketOwnershipControls sync.RWMutex
+	lockDeleteBucketPolicy            sync.RWMutex
+	lockDeleteBucketTagging           sync.RWMutex
+	lockDeleteObject                  sync.RWMutex
+	lockDeleteObjectTagging           sync.RWMutex
+	lockDeleteObjects                 sync.RWMutex
+	lockGetBucketAcl                  sync.RWMutex
+	lockGetBucketOwnershipControls    sync.RWMutex
+	lockGetBucketPolicy               sync.RWMutex
+	lockGetBucketTagging              sync.RWMutex
+	lockGetBucketVersioning           sync.RWMutex
+	lockGetObject                     sync.RWMutex
+	lockGetObjectAcl                  sync.RWMutex
+	lockGetObjectAttributes           sync.RWMutex
+	lockGetObjectLegalHold            sync.RWMutex
+	lockGetObjectLockConfiguration    sync.RWMutex
+	lockGetObjectRetention            sync.RWMutex
+	lockGetObjectTagging              sync.RWMutex
+	lockHeadBucket                    sync.RWMutex
+	lockHeadObject                    sync.RWMutex
+	lockListBuckets                   sync.RWMutex
+	lockListBucketsAndOwners          sync.RWMutex
+	lockListMultipartUploads          sync.RWMutex
+	lockListObjectVersions            sync.RWMutex
+	lockListObjects                   sync.RWMutex
+	lockListObjectsV2                 sync.RWMutex
+	lockListParts                     sync.RWMutex
+	lockPutBucketAcl                  sync.RWMutex
+	lockPutBucketOwnershipControls    sync.RWMutex
+	lockPutBucketPolicy               sync.RWMutex
+	lockPutBucketTagging              sync.RWMutex
+	lockPutBucketVersioning           sync.RWMutex
+	lockPutObject                     sync.RWMutex
+	lockPutObjectAcl                  sync.RWMutex
+	lockPutObjectLegalHold            sync.RWMutex
+	lockPutObjectLockConfiguration    sync.RWMutex
+	lockPutObjectRetention            sync.RWMutex
+	lockPutObjectTagging              sync.RWMutex
+	lockRestoreObject                 sync.RWMutex
+	lockSelectObjectContent           sync.RWMutex
+	lockShutdown                      sync.RWMutex
+	lockString                        sync.RWMutex
+	lockUploadPart                    sync.RWMutex
+	lockUploadPartCopy                sync.RWMutex
 }
 
 // AbortMultipartUpload calls AbortMultipartUploadFunc.
@@ -780,23 +822,23 @@ func (mock *BackendMock) AbortMultipartUploadCalls() []struct {
 }
 
 // ChangeBucketOwner calls ChangeBucketOwnerFunc.
-func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, bucket string, newOwner string) error {
+func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, bucket string, acl []byte) error {
 	if mock.ChangeBucketOwnerFunc == nil {
 		panic("BackendMock.ChangeBucketOwnerFunc: method is nil but Backend.ChangeBucketOwner was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		Bucket          string
-		NewOwner        string
+		ACL             []byte
 	}{
 		ContextMoqParam: contextMoqParam,
 		Bucket:          bucket,
-		NewOwner:        newOwner,
+		ACL:             acl,
 	}
 	mock.lockChangeBucketOwner.Lock()
 	mock.calls.ChangeBucketOwner = append(mock.calls.ChangeBucketOwner, callInfo)
 	mock.lockChangeBucketOwner.Unlock()
-	return mock.ChangeBucketOwnerFunc(contextMoqParam, bucket, newOwner)
+	return mock.ChangeBucketOwnerFunc(contextMoqParam, bucket, acl)
 }
 
 // ChangeBucketOwnerCalls gets all the calls that were made to ChangeBucketOwner.
@@ -806,12 +848,12 @@ func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, buck
 func (mock *BackendMock) ChangeBucketOwnerCalls() []struct {
 	ContextMoqParam context.Context
 	Bucket          string
-	NewOwner        string
+	ACL             []byte
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		Bucket          string
-		NewOwner        string
+		ACL             []byte
 	}
 	mock.lockChangeBucketOwner.RLock()
 	calls = mock.calls.ChangeBucketOwner
@@ -856,13 +898,13 @@ func (mock *BackendMock) CompleteMultipartUploadCalls() []struct {
 }
 
 // CopyObject calls CopyObjectFunc.
-func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	if mock.CopyObjectFunc == nil {
 		panic("BackendMock.CopyObjectFunc: method is nil but Backend.CopyObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
-		CopyObjectInput *s3.CopyObjectInput
+		CopyObjectInput s3response.CopyObjectInput
 	}{
 		ContextMoqParam: contextMoqParam,
 		CopyObjectInput: copyObjectInput,
@@ -879,11 +921,11 @@ func (mock *BackendMock) CopyObject(contextMoqParam context.Context, copyObjectI
 //	len(mockedBackend.CopyObjectCalls())
 func (mock *BackendMock) CopyObjectCalls() []struct {
 	ContextMoqParam context.Context
-	CopyObjectInput *s3.CopyObjectInput
+	CopyObjectInput s3response.CopyObjectInput
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
-		CopyObjectInput *s3.CopyObjectInput
+		CopyObjectInput s3response.CopyObjectInput
 	}
 	mock.lockCopyObject.RLock()
 	calls = mock.calls.CopyObject
@@ -932,13 +974,13 @@ func (mock *BackendMock) CreateBucketCalls() []struct {
 }
 
 // CreateMultipartUpload calls CreateMultipartUploadFunc.
-func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	if mock.CreateMultipartUploadFunc == nil {
 		panic("BackendMock.CreateMultipartUploadFunc: method is nil but Backend.CreateMultipartUpload was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam            context.Context
-		CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+		CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 	}{
 		ContextMoqParam:            contextMoqParam,
 		CreateMultipartUploadInput: createMultipartUploadInput,
@@ -955,11 +997,11 @@ func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context,
 //	len(mockedBackend.CreateMultipartUploadCalls())
 func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
 	ContextMoqParam            context.Context
-	CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+	CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 } {
 	var calls []struct {
 		ContextMoqParam            context.Context
-		CreateMultipartUploadInput *s3.CreateMultipartUploadInput
+		CreateMultipartUploadInput s3response.CreateMultipartUploadInput
 	}
 	mock.lockCreateMultipartUpload.RLock()
 	calls = mock.calls.CreateMultipartUpload
@@ -968,21 +1010,21 @@ func (mock *BackendMock) CreateMultipartUploadCalls() []struct {
 }
 
 // DeleteBucket calls DeleteBucketFunc.
-func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
+func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketFunc == nil {
 		panic("BackendMock.DeleteBucketFunc: method is nil but Backend.DeleteBucket was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam   context.Context
-		DeleteBucketInput *s3.DeleteBucketInput
+		ContextMoqParam context.Context
+		Bucket          string
 	}{
-		ContextMoqParam:   contextMoqParam,
-		DeleteBucketInput: deleteBucketInput,
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
 	}
 	mock.lockDeleteBucket.Lock()
 	mock.calls.DeleteBucket = append(mock.calls.DeleteBucket, callInfo)
 	mock.lockDeleteBucket.Unlock()
-	return mock.DeleteBucketFunc(contextMoqParam, deleteBucketInput)
+	return mock.DeleteBucketFunc(contextMoqParam, bucket)
 }
 
 // DeleteBucketCalls gets all the calls that were made to DeleteBucket.
@@ -990,12 +1032,12 @@ func (mock *BackendMock) DeleteBucket(contextMoqParam context.Context, deleteBuc
 //
 //	len(mockedBackend.DeleteBucketCalls())
 func (mock *BackendMock) DeleteBucketCalls() []struct {
-	ContextMoqParam   context.Context
-	DeleteBucketInput *s3.DeleteBucketInput
+	ContextMoqParam context.Context
+	Bucket          string
 } {
 	var calls []struct {
-		ContextMoqParam   context.Context
-		DeleteBucketInput *s3.DeleteBucketInput
+		ContextMoqParam context.Context
+		Bucket          string
 	}
 	mock.lockDeleteBucket.RLock()
 	calls = mock.calls.DeleteBucket
@@ -1003,6 +1045,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketOwnershipControls calls DeleteBucketOwnershipControlsFunc.
+func (mock *BackendMock) DeleteBucketOwnershipControls(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.DeleteBucketOwnershipControlsFunc: method is nil but Backend.DeleteBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketOwnershipControls.Lock()
+	mock.calls.DeleteBucketOwnershipControls = append(mock.calls.DeleteBucketOwnershipControls, callInfo)
+	mock.lockDeleteBucketOwnershipControls.Unlock()
+	return mock.DeleteBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketOwnershipControlsCalls gets all the calls that were made to DeleteBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketOwnershipControlsCalls())
+func (mock *BackendMock) DeleteBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketOwnershipControls.RLock()
+	calls = mock.calls.DeleteBucketOwnershipControls
+	mock.lockDeleteBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // DeleteBucketPolicy calls DeleteBucketPolicyFunc.
 func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketPolicyFunc == nil {
@@ -1076,7 +1154,7 @@ func (mock *BackendMock) DeleteBucketTaggingCalls() []struct {
 }
 
 // DeleteObject calls DeleteObjectFunc.
-func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
+func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
 	if mock.DeleteObjectFunc == nil {
 		panic("BackendMock.DeleteObjectFunc: method is nil but Backend.DeleteObject was just called")
 	}
@@ -1223,6 +1301,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketOwnershipControls calls GetBucketOwnershipControlsFunc.
+func (mock *BackendMock) GetBucketOwnershipControls(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+	if mock.GetBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.GetBucketOwnershipControlsFunc: method is nil but Backend.GetBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketOwnershipControls.Lock()
+	mock.calls.GetBucketOwnershipControls = append(mock.calls.GetBucketOwnershipControls, callInfo)
+	mock.lockGetBucketOwnershipControls.Unlock()
+	return mock.GetBucketOwnershipControlsFunc(contextMoqParam, bucket)
+}
+
+// GetBucketOwnershipControlsCalls gets all the calls that were made to GetBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketOwnershipControlsCalls())
+func (mock *BackendMock) GetBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketOwnershipControls.RLock()
+	calls = mock.calls.GetBucketOwnershipControls
+	mock.lockGetBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // GetBucketPolicy calls GetBucketPolicyFunc.
 func (mock *BackendMock) GetBucketPolicy(contextMoqParam context.Context, bucket string) ([]byte, error) {
 	if mock.GetBucketPolicyFunc == nil {
@@ -1296,7 +1410,7 @@ func (mock *BackendMock) GetBucketTaggingCalls() []struct {
 }
 
 // GetBucketVersioning calls GetBucketVersioningFunc.
-func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
 	if mock.GetBucketVersioningFunc == nil {
 		panic("BackendMock.GetBucketVersioningFunc: method is nil but Backend.GetBucketVersioning was just called")
 	}
@@ -1332,23 +1446,21 @@ func (mock *BackendMock) GetBucketVersioningCalls() []struct {
 }
 
 // GetObject calls GetObjectFunc.
-func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	if mock.GetObjectFunc == nil {
 		panic("BackendMock.GetObjectFunc: method is nil but Backend.GetObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}{
 		ContextMoqParam: contextMoqParam,
 		GetObjectInput:  getObjectInput,
-		Writer:          writer,
 	}
 	mock.lockGetObject.Lock()
 	mock.calls.GetObject = append(mock.calls.GetObject, callInfo)
 	mock.lockGetObject.Unlock()
-	return mock.GetObjectFunc(contextMoqParam, getObjectInput, writer)
+	return mock.GetObjectFunc(contextMoqParam, getObjectInput)
 }
 
 // GetObjectCalls gets all the calls that were made to GetObject.
@@ -1358,12 +1470,10 @@ func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInp
 func (mock *BackendMock) GetObjectCalls() []struct {
 	ContextMoqParam context.Context
 	GetObjectInput  *s3.GetObjectInput
-	Writer          io.Writer
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		GetObjectInput  *s3.GetObjectInput
-		Writer          io.Writer
 	}
 	mock.lockGetObject.RLock()
 	calls = mock.calls.GetObject
@@ -1408,7 +1518,7 @@ func (mock *BackendMock) GetObjectAclCalls() []struct {
 }
 
 // GetObjectAttributes calls GetObjectAttributesFunc.
-func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
+func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	if mock.GetObjectAttributesFunc == nil {
 		panic("BackendMock.GetObjectAttributesFunc: method is nil but Backend.GetObjectAttributes was just called")
 	}
@@ -1680,23 +1790,21 @@ func (mock *BackendMock) HeadObjectCalls() []struct {
 }
 
 // ListBuckets calls ListBucketsFunc.
-func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 	if mock.ListBucketsFunc == nil {
 		panic("BackendMock.ListBucketsFunc: method is nil but Backend.ListBuckets was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam context.Context
-		Owner           string
-		IsAdmin         bool
+		ContextMoqParam  context.Context
+		ListBucketsInput s3response.ListBucketsInput
 	}{
-		ContextMoqParam: contextMoqParam,
-		Owner:           owner,
-		IsAdmin:         isAdmin,
+		ContextMoqParam:  contextMoqParam,
+		ListBucketsInput: listBucketsInput,
 	}
 	mock.lockListBuckets.Lock()
 	mock.calls.ListBuckets = append(mock.calls.ListBuckets, callInfo)
 	mock.lockListBuckets.Unlock()
-	return mock.ListBucketsFunc(contextMoqParam, owner, isAdmin)
+	return mock.ListBucketsFunc(contextMoqParam, listBucketsInput)
 }
 
 // ListBucketsCalls gets all the calls that were made to ListBuckets.
@@ -1704,14 +1812,12 @@ func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner stri
 //
 //	len(mockedBackend.ListBucketsCalls())
 func (mock *BackendMock) ListBucketsCalls() []struct {
-	ContextMoqParam context.Context
-	Owner           string
-	IsAdmin         bool
+	ContextMoqParam  context.Context
+	ListBucketsInput s3response.ListBucketsInput
 } {
 	var calls []struct {
-		ContextMoqParam context.Context
-		Owner           string
-		IsAdmin         bool
+		ContextMoqParam  context.Context
+		ListBucketsInput s3response.ListBucketsInput
 	}
 	mock.lockListBuckets.RLock()
 	calls = mock.calls.ListBuckets
@@ -1788,7 +1894,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct {
 }
 
 // ListObjectVersions calls ListObjectVersionsFunc.
-func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
 	if mock.ListObjectVersionsFunc == nil {
 		panic("BackendMock.ListObjectVersionsFunc: method is nil but Backend.ListObjectVersions was just called")
 	}
@@ -1824,7 +1930,7 @@ func (mock *BackendMock) ListObjectVersionsCalls() []struct {
 }
 
 // ListObjects calls ListObjectsFunc.
-func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	if mock.ListObjectsFunc == nil {
 		panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called")
 	}
@@ -1860,7 +1966,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct {
 }
 
 // ListObjectsV2 calls ListObjectsV2Func.
-func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 	if mock.ListObjectsV2Func == nil {
 		panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called")
 	}
@@ -1971,6 +2077,46 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketOwnershipControls calls PutBucketOwnershipControlsFunc.
+func (mock *BackendMock) PutBucketOwnershipControls(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+	if mock.PutBucketOwnershipControlsFunc == nil {
+		panic("BackendMock.PutBucketOwnershipControlsFunc: method is nil but Backend.PutBucketOwnershipControls was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Ownership:       ownership,
+	}
+	mock.lockPutBucketOwnershipControls.Lock()
+	mock.calls.PutBucketOwnershipControls = append(mock.calls.PutBucketOwnershipControls, callInfo)
+	mock.lockPutBucketOwnershipControls.Unlock()
+	return mock.PutBucketOwnershipControlsFunc(contextMoqParam, bucket, ownership)
+}
+
+// PutBucketOwnershipControlsCalls gets all the calls that were made to PutBucketOwnershipControls.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketOwnershipControlsCalls())
+func (mock *BackendMock) PutBucketOwnershipControlsCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Ownership       types.ObjectOwnership
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Ownership       types.ObjectOwnership
+	}
+	mock.lockPutBucketOwnershipControls.RLock()
+	calls = mock.calls.PutBucketOwnershipControls
+	mock.lockPutBucketOwnershipControls.RUnlock()
+	return calls
+}
+
 // PutBucketPolicy calls PutBucketPolicyFunc.
 func (mock *BackendMock) PutBucketPolicy(contextMoqParam context.Context, bucket string, policy []byte) error {
 	if mock.PutBucketPolicyFunc == nil {
@@ -2052,21 +2198,23 @@ func (mock *BackendMock) PutBucketTaggingCalls() []struct {
 }
 
 // PutBucketVersioning calls PutBucketVersioningFunc.
-func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 	if mock.PutBucketVersioningFunc == nil {
 		panic("BackendMock.PutBucketVersioningFunc: method is nil but Backend.PutBucketVersioning was just called")
 	}
 	callInfo := struct {
-		ContextMoqParam          context.Context
-		PutBucketVersioningInput *s3.PutBucketVersioningInput
+		ContextMoqParam context.Context
+		Bucket          string
+		Status          types.BucketVersioningStatus
 	}{
-		ContextMoqParam:          contextMoqParam,
-		PutBucketVersioningInput: putBucketVersioningInput,
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Status:          status,
 	}
 	mock.lockPutBucketVersioning.Lock()
 	mock.calls.PutBucketVersioning = append(mock.calls.PutBucketVersioning, callInfo)
 	mock.lockPutBucketVersioning.Unlock()
-	return mock.PutBucketVersioningFunc(contextMoqParam, putBucketVersioningInput)
+	return mock.PutBucketVersioningFunc(contextMoqParam, bucket, status)
 }
 
 // PutBucketVersioningCalls gets all the calls that were made to PutBucketVersioning.
@@ -2074,12 +2222,14 @@ func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, pu
 //
 //	len(mockedBackend.PutBucketVersioningCalls())
 func (mock *BackendMock) PutBucketVersioningCalls() []struct {
-	ContextMoqParam          context.Context
-	PutBucketVersioningInput *s3.PutBucketVersioningInput
+	ContextMoqParam context.Context
+	Bucket          string
+	Status          types.BucketVersioningStatus
 } {
 	var calls []struct {
-		ContextMoqParam          context.Context
-		PutBucketVersioningInput *s3.PutBucketVersioningInput
+		ContextMoqParam context.Context
+		Bucket          string
+		Status          types.BucketVersioningStatus
 	}
 	mock.lockPutBucketVersioning.RLock()
 	calls = mock.calls.PutBucketVersioning
@@ -2088,13 +2238,13 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct {
 }
 
 // PutObject calls PutObjectFunc.
-func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
+func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
 	if mock.PutObjectFunc == nil {
 		panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
-		PutObjectInput  *s3.PutObjectInput
+		PutObjectInput  s3response.PutObjectInput
 	}{
 		ContextMoqParam: contextMoqParam,
 		PutObjectInput:  putObjectInput,
@@ -2111,11 +2261,11 @@ func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInp
 //	len(mockedBackend.PutObjectCalls())
 func (mock *BackendMock) PutObjectCalls() []struct {
 	ContextMoqParam context.Context
-	PutObjectInput  *s3.PutObjectInput
+	PutObjectInput  s3response.PutObjectInput
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
-		PutObjectInput  *s3.PutObjectInput
+		PutObjectInput  s3response.PutObjectInput
 	}
 	mock.lockPutObject.RLock()
 	calls = mock.calls.PutObject
@@ -2470,7 +2620,7 @@ func (mock *BackendMock) StringCalls() []struct {
 }
 
 // UploadPart calls UploadPartFunc.
-func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
+func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
 	if mock.UploadPartFunc == nil {
 		panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called")
 	}
@@ -2506,7 +2656,7 @@ func (mock *BackendMock) UploadPartCalls() []struct {
 }
 
 // UploadPartCopy calls UploadPartCopyFunc.
-func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
 	if mock.UploadPartCopyFunc == nil {
 		panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called")
 	}

s3api/controllers/base_test.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -93,7 +92,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, nil
 			},
 		},
@@ -110,7 +109,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr := fiber.New()
 	s3ApiControllerErr := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
 			},
 		},
@@ -188,10 +187,10 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 				return &s3.GetObjectAclOutput{}, nil
 			},
-			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
-				return s3response.GetObjectAttributesResult{}, nil
+			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+				return s3response.GetObjectAttributesResponse{}, nil
 			},
-			GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+			GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
 					Metadata:        map[string]string{"hello": "world"},
 					ContentType:     getPtr("application/xml"),
@@ -233,6 +232,9 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
+	invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
+	invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -330,6 +332,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Get-actions-get-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invalidChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Get-actions-get-object-success",
 			app:  app,
@@ -374,20 +385,20 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 				return s3response.ListMultipartUploadsResult{}, nil
 			},
-			ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-				return &s3.ListObjectsV2Output{}, nil
+			ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+				return s3response.ListObjectsV2Result{}, nil
 			},
-			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-				return &s3.ListObjectsOutput{}, nil
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+				return s3response.ListObjectsResult{}, nil
 			},
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return map[string]string{}, nil
 			},
-			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-				return &s3.GetBucketVersioningOutput{}, nil
+			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+				return s3response.GetBucketVersioningOutput{}, nil
 			},
-			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
-				return &s3.ListObjectVersionsOutput{}, nil
+			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+				return s3response.ListVersionsResult{}, nil
 			},
 			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 				return []byte{}, nil
@@ -395,6 +406,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 				return objectLockResult, nil
 			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerEnforced, nil
+			},
 		},
 	}
 
@@ -414,8 +428,8 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+				return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 			},
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -448,6 +462,15 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 404,
 		},
+		{
+			name: "Get-bucket-ownership-control-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-bucket-tagging-success",
 			app:  app,
@@ -562,7 +585,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	app := fiber.New()
 
 	// Mock valid acl
-	acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
+	acl := auth.ACL{Owner: "valid access"}
 	acldata, err := json.Marshal(acl)
 	if err != nil {
 		t.Errorf("Failed to parse the params: %v", err.Error())
@@ -636,6 +659,22 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</ObjectLockConfiguration>
 	`
 
+	ownershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
+	invalidOwnershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>invalid_value</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -650,7 +689,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
 				return nil
 			},
-			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+			PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 				return nil
 			},
 			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
@@ -659,6 +698,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
 				return nil
 			},
+			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+				return nil
+			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerPreferred, nil
+			},
 		},
 	}
 	// Mock ctx.Locals
@@ -691,6 +736,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
 	errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
+	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -714,6 +762,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
 			},
 			wantErr:    false,
+			statusCode: 204,
+		},
+		{
+			name: "Put-bucket-ownership-controls-invalid-ownership",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
+			},
+			wantErr:    false,
 			statusCode: 200,
 		},
 		{
@@ -804,7 +870,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 				req: incorrectBucketOwner,
 			},
 			wantErr:    false,
-			statusCode: 403,
+			statusCode: 400,
 		},
 		{
 			name: "Put-bucket-acl-success",
@@ -816,7 +882,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 200,
 		},
 		{
-			name: "Put-bucket-invalid-bucket-name",
+			name: "Create-bucket-invalid-acl-ownership-combination",
+			app:  app,
+			args: args{
+				req: invAclOwnershipReq,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Create-bucket-invalid-bucket-name",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
@@ -825,7 +900,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 400,
 		},
 		{
-			name: "Put-bucket-success",
+			name: "Create-bucket-success",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
@@ -878,12 +953,12 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	</Tagging>
 	`
 
-	retentionBody := `
-	<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-		<Mode>GOVERNANCE</Mode>
-		<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
-	</Retention>
-	`
+	//retentionBody := `
+	//<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	//	<Mode>GOVERNANCE</Mode>
+	//	<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
+	//</Retention>
+	//`
 
 	legalHoldBody := `
 	<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
@@ -900,22 +975,22 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
 				return nil
 			},
-			CopyObjectFunc: func(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+			CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 				return &s3.CopyObjectOutput{
 					CopyObjectResult: &types.CopyObjectResult{},
 				}, nil
 			},
-			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
-				return "ETag", nil
+			PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
+				return s3response.PutObjectOutput{}, nil
 			},
-			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
-				return "hello", nil
+			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+				return &s3.UploadPartOutput{}, nil
 			},
 			PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
 				return nil
 			},
-			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-				return s3response.CopyObjectResult{}, nil
+			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+				return s3response.CopyPartResult{}, nil
 			},
 			PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
 				return nil
@@ -949,6 +1024,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
 
+	// CopyObject invalid checksum algorithm
+	cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
+	cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	// PutObjectAcl success
 	aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
 	aclReq.Header.Set("X-Amz-Acl", "private")
@@ -970,6 +1050,40 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	// PutObject invalid checksum algorithm
+	invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
+	// PutObject invalid base64 checksum
+	invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
+
+	// PutObject invalid crc32
+	invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
+
+	// PutObject invalid crc32c
+	invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
+
+	// PutObject invalid sha1
+	invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
+
+	// PutObject invalid sha256
+	invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
+
+	// PutObject multiple checksum headers
+	mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
+	mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
+	// PutObject checksum algorithm and header mismatch
+	checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
+	checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -1013,15 +1127,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
-		{
-			name: "put-object-retention-success",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
+		//{
+		//	name: "put-object-retention-success",
+		//	app:  app,
+		//	args: args{
+		//		req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
+		//	},
+		//	wantErr:    false,
+		//	statusCode: 200,
+		//},
 		{
 			name: "put-legal-hold-invalid-request",
 			app:  app,
@@ -1112,6 +1226,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Copy-object-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: cpyInvChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Copy-object-success",
 			app:  app,
@@ -1154,12 +1277,18 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
+			DeleteBucketFunc: func(_ context.Context, bucket string) error {
 				return nil
 			},
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 				return nil
 			},
+			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
+			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
 		},
 	}
 
@@ -1198,6 +1327,23 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			wantErr:    false,
 			statusCode: 204,
 		},
+		{
+			name: "Delete-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		}, {
+			name: "Delete-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -1297,8 +1443,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
-				return nil
+			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+				return &s3.DeleteObjectOutput{}, nil
 			},
 			AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
 				return nil
@@ -1328,8 +1474,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 		GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 			return acldata, nil
 		},
-		DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
-			return s3err.GetAPIError(7)
+		DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		},
 		GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 			return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
@@ -1542,7 +1688,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 				return acldata, nil
 			},
 			HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-				return nil, s3err.GetAPIError(42)
+				return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 			},
 		},
 	}
@@ -1556,6 +1702,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	})
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
+	invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
+	invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -1572,6 +1721,15 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Head-object-invalid-checksum-mode",
+			app:  app,
+			args: args{
+				req: invChecksumMode,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Head-object-error",
 			app:  appErr,
@@ -1611,8 +1769,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 				return &s3.CompleteMultipartUploadOutput{}, nil
 			},
-			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-				return &s3.CreateMultipartUploadOutput{}, nil
+			CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+				return s3response.InitiateMultipartUploadResult{}, nil
 			},
 			SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
 				return func(w *bufio.Writer) {}
@@ -1627,6 +1785,19 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 		</SelectObjectContentRequest>
 	`
 
+	completMpBody := `
+		<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Part>
+				<ETag>etag</ETag>
+				<PartNumber>1</PartNumber>
+			</Part>
+		</CompleteMultipartUpload>
+	`
+
+	completMpEmptyBody := `
+		<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></CompleteMultipartUpload>
+	`
+
 	app.Use(func(ctx *fiber.Ctx) error {
 		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
@@ -1636,6 +1807,9 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	})
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
+	invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
+	invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
+
 	tests := []struct {
 		name       string
 		app        *fiber.App
@@ -1679,15 +1853,33 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 400,
 		},
+		{
+			name: "Complete-multipart-upload-empty-parts",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpEmptyBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Complete-multipart-upload-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(`<root><key>body</key></root>`)),
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpBody)),
 			},
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Create-multipart-upload-invalid-checksum-algorithm",
+			app:  app,
+			args: args{
+				req: invChecksumAlgo,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Create-multipart-upload-success",
 			app:  app,

s3api/middlewares/acl-parser.go

@@ -50,7 +50,8 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			!ctx.Request().URI().QueryArgs().Has("tagging") &&
 			!ctx.Request().URI().QueryArgs().Has("versioning") &&
 			!ctx.Request().URI().QueryArgs().Has("policy") &&
-			!ctx.Request().URI().QueryArgs().Has("object-lock") {
+			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
+			!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
@@ -73,6 +74,11 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 		}
 
+		// if owner is not set, set default owner to root account
+		if parsedAcl.Owner == "" {
+			parsedAcl.Owner = ctx.Locals("rootAccess").(string)
+		}
+
 		ctx.Locals("parsedAcl", parsedAcl)
 		return ctx.Next()
 	}

s3api/middlewares/admin.go

@@ -0,0 +1,59 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		acct := ctx.Locals("account").(auth.Account)
+		if acct.Role != auth.RoleAdmin {
+			path := ctx.Path()
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
+				&controllers.MetaOpts{
+					Logger: logger,
+					Action: detectAction(path),
+				})
+		}
+
+		return ctx.Next()
+	}
+}
+
+func detectAction(path string) (action string) {
+	if strings.Contains(path, "create-user") {
+		action = metrics.ActionAdminCreateUser
+	} else if strings.Contains(path, "update-user") {
+		action = metrics.ActionAdminUpdateUser
+	} else if strings.Contains(path, "delete-user") {
+		action = metrics.ActionAdminDeleteUser
+	} else if strings.Contains(path, "list-user") {
+		action = metrics.ActionAdminListUsers
+	} else if strings.Contains(path, "list-buckets") {
+		action = metrics.ActionAdminListBuckets
+	} else if strings.Contains(path, "change-bucket-owner") {
+		action = metrics.ActionAdminChangeBucketOwner
+	}
+	return action
+}

s3api/middlewares/authentication.go

@@ -76,6 +76,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
+		ctx.Locals("rootAccess", root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
@@ -108,6 +109,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return sendResponse(ctx, err, logger, mm)
 		}
 
+		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if utils.IsBigDataAction(ctx) {
 			// for streaming PUT actions, authorization is deferred
 			// until end of stream due to need to get length and
@@ -115,10 +117,24 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
 				return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
 			})
+
+			// wrap the io.Reader with ChunkReader if x-amz-content-sha256
+			// provide chunk encoding value
+			if utils.IsStreamingPayload(hashPayload) {
+				var err error
+				wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+					var cr io.Reader
+					cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate)
+					return cr
+				})
+				if err != nil {
+					return sendResponse(ctx, err, logger, mm)
+				}
+			}
+
 			return ctx.Next()
 		}
 
-		hashPayload := ctx.Get("X-Amz-Content-Sha256")
 		if !utils.IsSpecialPayload(hashPayload) {
 			// Calculate the hash of the request payload
 			hashedPayload := sha256.Sum256(ctx.Body())

s3api/middlewares/chunk.go

@@ -1,62 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package middlewares
-
-import (
-	"io"
-	"time"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/metrics"
-	"github.com/versity/versitygw/s3api/utils"
-	"github.com/versity/versitygw/s3log"
-)
-
-// ProcessChunkedBody initializes the chunked upload stream if the
-// request appears to be a chunked upload
-func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
-	return func(ctx *fiber.Ctx) error {
-		decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
-		if decodedLength == "" {
-			return ctx.Next()
-		}
-		// TODO: validate content length
-
-		authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
-		if err != nil {
-			return sendResponse(ctx, err, logger, mm)
-		}
-
-		acct := ctx.Locals("account").(auth.Account)
-		amzdate := ctx.Get("X-Amz-Date")
-		date, _ := time.Parse(iso8601Format, amzdate)
-
-		if utils.IsBigDataAction(ctx) {
-			var err error
-			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
-				var cr *utils.ChunkReader
-				cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
-				return cr
-			})
-			if err != nil {
-				return sendResponse(ctx, err, logger, mm)
-			}
-			return ctx.Next()
-		}
-
-		return ctx.Next()
-	}
-}

s3api/middlewares/md5.go

@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
 		}
 
 		sum := md5.Sum(ctx.Body())
-		calculatedSum := utils.Md5SumString(sum[:])
+		calculatedSum := utils.Base64SumString(sum[:])
 
 		if incomingSum != calculatedSum {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

s3api/middlewares/presign-auth.go

@@ -43,6 +43,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
+		ctx.Locals("rootAccess", root.Access)
+
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
 			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)

s3api/middlewares/url-decoder.go

@@ -26,12 +26,11 @@ import (
 
 func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		reqURL := ctx.Request().URI().String()
-		decoded, err := url.Parse(reqURL)
+		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
 		if err != nil {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}
-		ctx.Path(decoded.Path)
+		ctx.Path(unescp)
 		return ctx.Next()
 	}
 }

s3api/router.go

@@ -20,6 +20,7 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
 )
@@ -28,29 +29,29 @@ type S3ApiRouter struct {
 	WithAdmSrv bool
 }
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
 	s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
 
 	if sa.WithAdmSrv {
-		adminController := controllers.NewAdminController(iam, be)
+		adminController := controllers.NewAdminController(iam, be, aLogger)
 
 		// CreateUser admin api
-		app.Patch("/create-user", adminController.CreateUser)
+		app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
 
 		// DeleteUsers admin api
-		app.Patch("/delete-user", adminController.DeleteUser)
+		app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
 
 		// UpdateUser admin api
-		app.Patch("update-user", adminController.UpdateUser)
+		app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
 
 		// ListUsers admin api
-		app.Patch("/list-users", adminController.ListUsers)
+		app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
 
 		// ChangeBucketOwner admin api
-		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+		app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
 
 		// ListBucketsAndOwners admin api
-		app.Patch("/list-buckets", adminController.ListBuckets)
+		app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
 	}
 
 	// ListBuckets action

s3api/router_test.go

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, false, false)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false)
 		})
 	}
 }

s3api/server.go

@@ -47,6 +47,7 @@ func New(
 	port, region string,
 	iam auth.IAMService,
 	l s3log.AuditLogger,
+	adminLogger s3log.AuditLogger,
 	evs s3event.S3EventSender,
 	mm *metrics.Manager,
 	opts ...Option,
@@ -64,7 +65,9 @@ func New(
 
 	// Logging middlewares
 	if !server.quiet {
-		app.Use(logger.New())
+		app.Use(logger.New(logger.Config{
+			Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
+		}))
 	}
 	// Set up health endpoint if specified
 	if server.health != "" {
@@ -78,11 +81,10 @@ func New(
 	// Authentication middlewares
 	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
-	app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l, server.readonly))
 
-	server.router.Init(app, be, iam, l, evs, mm, server.debug, server.readonly)
+	server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly)
 
 	return server, nil
 }

s3api/server_test.go

@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
-				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil)
+				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
 				return

s3api/utils/auth-reader.go

@@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de
 	var hr *HashReader
 	hashPayload := ctx.Get("X-Amz-Content-Sha256")
 	if !IsSpecialPayload(hashPayload) {
-		hr, _ = NewHashReader(r, "", HashTypeSha256)
+		hr, _ = NewHashReader(r, "", HashTypeSha256Hex)
 	} else {
 		hr, _ = NewHashReader(r, "", HashTypeNone)
 	}
@@ -260,19 +260,3 @@ func removeSpace(str string) string {
 	}
 	return b.String()
 }
-
-var (
-	specialValues = map[string]bool{
-		"UNSIGNED-PAYLOAD":                                 true,
-		"STREAMING-UNSIGNED-PAYLOAD-TRAILER":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER":       true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD":         true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
-	}
-)
-
-// IsSpecialPayload checks for streaming/unsigned authorization types
-func IsSpecialPayload(str string) bool {
-	return specialValues[str]
-}

s3api/utils/chunk-reader.go

@@ -15,260 +15,98 @@
 package utils
 
 import (
-	"bytes"
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
 	"fmt"
-	"hash"
 	"io"
-	"math"
-	"strconv"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/s3err"
 )
 
-// chunked uploads described in:
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+type payloadType string
 
 const (
-	chunkHdrStr       = ";chunk-signature="
-	chunkHdrDelim     = "\r\n"
-	zeroLenSig        = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-	awsV4             = "AWS4"
-	awsS3Service      = "s3"
-	awsV4Request      = "aws4_request"
-	streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
+	payloadTypeUnsigned                 payloadType = "UNSIGNED-PAYLOAD"
+	payloadTypeStreamingUnsignedTrailer payloadType = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+	payloadTypeStreamingSigned          payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	payloadTypeStreamingSignedTrailer   payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
+	payloadTypeStreamingEcdsa           payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
+	payloadTypeStreamingEcdsaTrailer    payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
 )
 
-// ChunkReader reads from chunked upload request body, and returns
-// object data stream
-type ChunkReader struct {
-	r                io.Reader
-	signingKey       []byte
-	prevSig          string
-	parsedSig        string
-	currentChunkSize int64
-	chunkDataLeft    int64
-	trailerExpected  int
-	stash            []byte
-	chunkHash        hash.Hash
-	strToSignPrefix  string
-	skipcheck        bool
-}
-
-// NewChunkReader reads from request body io.Reader and parses out the
-// chunk metadata in stream. The headers are validated for proper signatures.
-// Reading from the chunk reader will read only the object data stream
-// without the chunk headers/trailers.
-func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (*ChunkReader, error) {
-	return &ChunkReader{
-		r:          r,
-		signingKey: getSigningKey(secret, region, date),
-		// the authdata.Signature is validated in the auth-reader,
-		// so we can use that here without any other checks
-		prevSig:         authdata.Signature,
-		chunkHash:       sha256.New(),
-		strToSignPrefix: getStringToSignPrefix(date, region),
-	}, nil
-}
-
-// Read satisfies the io.Reader for this type
-func (cr *ChunkReader) Read(p []byte) (int, error) {
-	n, err := cr.r.Read(p)
-	if err != nil && err != io.EOF {
-		return n, err
-	}
-
-	if cr.chunkDataLeft < int64(n) {
-		chunkSize := cr.chunkDataLeft
-		if chunkSize > 0 {
-			cr.chunkHash.Write(p[:chunkSize])
-		}
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		n += int(chunkSize)
-		return n, err
-	}
-
-	cr.chunkDataLeft -= int64(n)
-	cr.chunkHash.Write(p[:n])
-	return n, err
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
-// This part is the same for all chunks,
-// only the previous signature and hash of current chunk changes
-func getStringToSignPrefix(date time.Time, region string) string {
-	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
-		date.Format("20060102"),
-		region,
-		awsS3Service,
-		awsV4Request)
-
-	return fmt.Sprintf("%s\n%s\n%s",
-		streamPayloadAlgo,
-		date.Format("20060102T150405Z"),
-		credentialScope)
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
-// signature For each chunk, you calculate the signature using the following
-// string to sign. For the first chunk, you use the seed-signature as the
-// previous signature.
-func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
-	return fmt.Sprintf("%s\n%s\n%s\n%s",
-		prefix,
-		prevSig,
-		zeroLenSig,
-		hex.EncodeToString(chunkHash))
-}
-
-// The provided p should have all of the previous chunk data and trailer
-// consumed already. The positioning here is expected that p[0] starts the
-// new chunk size with the ";chunk-signature=" following. The only exception
-// is if we started consuming the trailer, but hit the end of the read buffer.
-// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
-// finish consuming the final trailer bytes.
-// This parses the chunk metadata in situ without allocating an extra buffer.
-// It will just read and validate the chunk metadata and then move the
-// following chunk data to overwrite the metadata in the provided buffer.
-func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
-	n := len(p)
-
-	if !cr.skipcheck && cr.parsedSig != "" {
-		chunkhash := cr.chunkHash.Sum(nil)
-		cr.chunkHash.Reset()
-
-		sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
-		cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
-
-		if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
-		}
-	}
-
-	if cr.trailerExpected != 0 {
-		if len(p) < len(chunkHdrDelim) {
-			// This is the special case where we need to consume the
-			// trailer, but instead hit the end of the buffer. The
-			// subsequent call will finish consuming the trailer.
-			cr.chunkDataLeft = 0
-			cr.trailerExpected -= len(p)
-			cr.skipcheck = true
-			return 0, nil
-		}
-		// move data up to remove trailer
-		copy(p, p[cr.trailerExpected:])
-		n -= cr.trailerExpected
-	}
-
-	cr.skipcheck = false
-
-	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
-	cr.currentChunkSize = chunkSize
-	cr.parsedSig = sig
-	if err == errskipHeader {
-		cr.chunkDataLeft = 0
-		return 0, nil
-	}
-	if err != nil {
-		return 0, err
-	}
-	if chunkSize == 0 {
-		return 0, io.EOF
-	}
-
-	cr.trailerExpected = len(chunkHdrDelim)
-
-	// move data up to remove chunk header
-	copy(p, p[bufOffset:n])
-	n -= bufOffset
-
-	// if remaining buffer larger than chunk data,
-	// parse next header in buffer
-	if int64(n) > chunkSize {
-		cr.chunkDataLeft = 0
-		cr.chunkHash.Write(p[:chunkSize])
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		if (chunkSize + int64(n)) > math.MaxInt {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
-		}
-		return n + int(chunkSize), err
-	}
-
-	cr.chunkDataLeft = chunkSize - int64(n)
-	cr.chunkHash.Write(p[:n])
-
-	return n, nil
-}
-
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
-// Task 3: Calculate Signature
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
-func getSigningKey(secret, region string, date time.Time) []byte {
-	dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
-	dateRegionKey := hmac256(dateKey, []byte(region))
-	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
-	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
-	return signingKey
-}
-
-func hmac256(key []byte, data []byte) []byte {
-	hash := hmac.New(sha256.New, key)
-	hash.Write(data)
-	return hash.Sum(nil)
-}
-
 var (
-	errInvalidChunkFormat = errors.New("invalid chunk header format")
-	errskipHeader         = errors.New("skip to next header")
+	specialValues = map[payloadType]bool{
+		payloadTypeUnsigned:                 true,
+		payloadTypeStreamingUnsignedTrailer: true,
+		payloadTypeStreamingSigned:          true,
+		payloadTypeStreamingSignedTrailer:   true,
+		payloadTypeStreamingEcdsa:           true,
+		payloadTypeStreamingEcdsaTrailer:    true,
+	}
 )
 
+func (pt payloadType) isValid() bool {
+	return pt == payloadTypeUnsigned ||
+		pt == payloadTypeStreamingUnsignedTrailer ||
+		pt == payloadTypeStreamingSigned ||
+		pt == payloadTypeStreamingSignedTrailer ||
+		pt == payloadTypeStreamingEcdsa ||
+		pt == payloadTypeStreamingEcdsaTrailer
+}
+
+type checksumType string
+
 const (
-	maxHeaderSize = 1024
+	checksumTypeCrc32     checksumType = "x-amz-checksum-crc32"
+	checksumTypeCrc32c    checksumType = "x-amz-checksum-crc32c"
+	checksumTypeSha1      checksumType = "x-amz-checksum-sha1"
+	checksumTypeSha256    checksumType = "x-amz-checksum-sha256"
+	checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
 )
 
-// Theis returns the chunk payload size, signature, data start offset, and
-// error if any. See the AWS documentation for the chunk header format. The
-// header[0] byte is expected to be the first byte of the chunk size here.
-func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
-	stashLen := len(cr.stash)
-	if cr.stash != nil {
-		tmp := make([]byte, maxHeaderSize)
-		copy(tmp, cr.stash)
-		copy(tmp[len(cr.stash):], header)
-		header = tmp
-		cr.stash = nil
-	}
-
-	semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
-	if semicolonIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
-	}
-
-	sigIndex := semicolonIndex + len(chunkHdrStr)
-	sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
-	if sigEndIndex == -1 {
-		cr.stash = make([]byte, len(header))
-		copy(cr.stash, header)
-		cr.trailerExpected = 0
-		return 0, "", 0, errskipHeader
-	}
-
-	chunkSizeBytes := header[:semicolonIndex]
-	chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
-	if err != nil {
-		return 0, "", 0, errInvalidChunkFormat
-	}
-
-	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
-	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
-
-	return chunkSize, signature, dataStartOffset - stashLen, nil
+func (c checksumType) isValid() bool {
+	return c == checksumTypeCrc32 ||
+		c == checksumTypeCrc32c ||
+		c == checksumTypeSha1 ||
+		c == checksumTypeSha256 ||
+		c == checksumTypeCrc64nvme
+}
+
+// IsSpecialPayload checks for special authorization types
+func IsSpecialPayload(str string) bool {
+	return specialValues[payloadType(str)]
+}
+
+// IsChunkEncoding checks for streaming/unsigned authorization types
+func IsStreamingPayload(str string) bool {
+	pt := payloadType(str)
+	return pt == payloadTypeStreamingUnsignedTrailer ||
+		pt == payloadTypeStreamingSigned ||
+		pt == payloadTypeStreamingSignedTrailer
+}
+
+func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
+	decContLength := ctx.Get("X-Amz-Decoded-Content-Length")
+	if decContLength == "" {
+		return nil, s3err.GetAPIError(s3err.ErrMissingDecodedContentLength)
+	}
+	contentSha256 := payloadType(ctx.Get("X-Amz-Content-Sha256"))
+	if !contentSha256.isValid() {
+		//TODO: Add proper APIError
+		return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
+	}
+
+	checksumType := checksumType(ctx.Get("X-Amz-Trailer"))
+	if checksumType != "" && !checksumType.isValid() {
+		//TODO: Add proper APIError
+		return nil, fmt.Errorf("invalid X-Amz-Trailer: %v", checksumType)
+	}
+
+	switch contentSha256 {
+	case payloadTypeStreamingUnsignedTrailer:
+		return NewUnsignedChunkReader(r, checksumType)
+		//TODO: Add other chunk readers
+	}
+
+	return NewSignedChunkReader(r, authdata, region, secret, date)
 }

s3api/utils/csum-reader.go

@@ -16,13 +16,19 @@ package utils
 
 import (
 	"crypto/md5"
+	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"hash"
+	"hash/crc32"
+	"hash/crc64"
 	"io"
+	"math/bits"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -31,11 +37,21 @@ type HashType string
 
 const (
 	// HashTypeMd5 generates MD5 checksum for the data stream
-	HashTypeMd5 = "md5"
-	// HashTypeSha256 generates SHA256 checksum for the data stream
-	HashTypeSha256 = "sha256"
+	HashTypeMd5 HashType = "md5"
+	// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
+	HashTypeSha256 HashType = "sha256"
+	// HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream
+	HashTypeSha256Hex HashType = "sha256-hex"
+	// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
+	HashTypeSha1 HashType = "sha1"
+	// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
+	HashTypeCRC32 HashType = "crc32"
+	// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
+	HashTypeCRC32C HashType = "crc32c"
+	// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
+	HashTypeCRC64NVME HashType = "crc64nvme"
 	// HashTypeNone is a no-op checksum for the data stream
-	HashTypeNone = "none"
+	HashTypeNone HashType = "none"
 )
 
 // HashReader is an io.Reader that calculates the checksum
@@ -62,8 +78,18 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
 	switch ht {
 	case HashTypeMd5:
 		hash = md5.New()
+	case HashTypeSha256Hex:
+		hash = sha256.New()
 	case HashTypeSha256:
 		hash = sha256.New()
+	case HashTypeSha1:
+		hash = sha1.New()
+	case HashTypeCRC32:
+		hash = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeCRC64NVME:
+		hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
 	case HashTypeNone:
 		hash = noop{}
 	default:
@@ -88,15 +114,40 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	if errors.Is(readerr, io.EOF) && hr.sum != "" {
 		switch hr.hashType {
 		case HashTypeMd5:
-			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
 			}
-		case HashTypeSha256:
-			sum := hex.EncodeToString(hr.hash.Sum(nil))
+		case HashTypeSha256Hex:
+			sum := hr.Sum()
 			if sum != hr.sum {
 				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
 			}
+		case HashTypeCRC32:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32)
+			}
+		case HashTypeCRC32C:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c)
+			}
+		case HashTypeSha1:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1)
+			}
+		case HashTypeSha256:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
+			}
+		case HashTypeCRC64NVME:
+			sum := hr.Sum()
+			if sum != hr.sum {
+				return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
+			}
 		default:
 			return n, errInvalidHashType
 		}
@@ -104,20 +155,38 @@ func (hr *HashReader) Read(p []byte) (int, error) {
 	return n, readerr
 }
 
+func (hr *HashReader) SetReader(r io.Reader) {
+	hr.r = r
+}
+
 // Sum returns the checksum hash of the data read so far
 func (hr *HashReader) Sum() string {
 	switch hr.hashType {
 	case HashTypeMd5:
-		return Md5SumString(hr.hash.Sum(nil))
-	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256Hex:
 		return hex.EncodeToString(hr.hash.Sum(nil))
+	case HashTypeCRC32:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC32C:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha1:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeSha256:
+		return Base64SumString(hr.hash.Sum(nil))
+	case HashTypeCRC64NVME:
+		return Base64SumString(hr.hash.Sum(nil))
 	default:
 		return ""
 	}
 }
 
+func (hr *HashReader) Type() HashType {
+	return hr.hashType
+}
+
 // Md5SumString converts the hash bytes to the string checksum value
-func Md5SumString(b []byte) string {
+func Base64SumString(b []byte) string {
 	return base64.StdEncoding.EncodeToString(b)
 }
 
@@ -128,3 +197,59 @@ func (n noop) Sum(b []byte) []byte         { return []byte{} }
 func (n noop) Reset()                      {}
 func (n noop) Size() int                   { return 0 }
 func (n noop) BlockSize() int              { return 1 }
+
+// NewCompositeChecksumReader initializes a composite checksum
+// processor, which decodes and validates the provided
+// checksums and returns the final checksum based on
+// the previous processings.
+//
+// The supported checksum types are:
+// - CRC32
+// - CRC32C
+// - SHA1
+// - SHA256
+func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
+	var hasher hash.Hash
+	switch ht {
+	case HashTypeSha256:
+		hasher = sha256.New()
+	case HashTypeSha1:
+		hasher = sha1.New()
+	case HashTypeCRC32:
+		hasher = crc32.NewIEEE()
+	case HashTypeCRC32C:
+		hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case HashTypeNone:
+		hasher = noop{}
+	default:
+		return nil, errInvalidHashType
+	}
+
+	return &CompositeChecksumReader{
+		hasher: hasher,
+	}, nil
+}
+
+type CompositeChecksumReader struct {
+	hasher hash.Hash
+}
+
+// Decodes and writes the checksum in the hasher
+func (ccr *CompositeChecksumReader) Process(checksum string) error {
+	data, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		return fmt.Errorf("base64 decode: %w", err)
+	}
+
+	_, err = ccr.hasher.Write(data)
+	if err != nil {
+		return fmt.Errorf("hash write: %w", err)
+	}
+
+	return nil
+}
+
+// Returns the base64 encoded composite checksum
+func (ccr *CompositeChecksumReader) Sum() string {
+	return Base64SumString(ccr.hasher.Sum(nil))
+}

s3api/utils/signed-chunk-reader.go

@@ -0,0 +1,276 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"math"
+	"strconv"
+	"time"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+// chunked uploads described in:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+
+const (
+	chunkHdrStr       = ";chunk-signature="
+	chunkHdrDelim     = "\r\n"
+	zeroLenSig        = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	awsV4             = "AWS4"
+	awsS3Service      = "s3"
+	awsV4Request      = "aws4_request"
+	streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
+)
+
+// ChunkReader reads from chunked upload request body, and returns
+// object data stream
+type ChunkReader struct {
+	r                io.Reader
+	signingKey       []byte
+	prevSig          string
+	parsedSig        string
+	currentChunkSize int64
+	chunkDataLeft    int64
+	trailerExpected  int
+	stash            []byte
+	chunkHash        hash.Hash
+	strToSignPrefix  string
+	skipcheck        bool
+}
+
+// NewChunkReader reads from request body io.Reader and parses out the
+// chunk metadata in stream. The headers are validated for proper signatures.
+// Reading from the chunk reader will read only the object data stream
+// without the chunk headers/trailers.
+func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
+	return &ChunkReader{
+		r:          r,
+		signingKey: getSigningKey(secret, region, date),
+		// the authdata.Signature is validated in the auth-reader,
+		// so we can use that here without any other checks
+		prevSig:         authdata.Signature,
+		chunkHash:       sha256.New(),
+		strToSignPrefix: getStringToSignPrefix(date, region),
+	}, nil
+}
+
+// Read satisfies the io.Reader for this type
+func (cr *ChunkReader) Read(p []byte) (int, error) {
+	n, err := cr.r.Read(p)
+	if err != nil && err != io.EOF {
+		return 0, err
+	}
+
+	if cr.chunkDataLeft < int64(n) {
+		chunkSize := cr.chunkDataLeft
+		if chunkSize > 0 {
+			cr.chunkHash.Write(p[:chunkSize])
+		}
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		n += int(chunkSize)
+		return n, err
+	}
+
+	cr.chunkDataLeft -= int64(n)
+	cr.chunkHash.Write(p[:n])
+	return n, err
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// This part is the same for all chunks,
+// only the previous signature and hash of current chunk changes
+func getStringToSignPrefix(date time.Time, region string) string {
+	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
+		date.Format("20060102"),
+		region,
+		awsS3Service,
+		awsV4Request)
+
+	return fmt.Sprintf("%s\n%s\n%s",
+		streamPayloadAlgo,
+		date.Format("20060102T150405Z"),
+		credentialScope)
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// signature For each chunk, you calculate the signature using the following
+// string to sign. For the first chunk, you use the seed-signature as the
+// previous signature.
+func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
+	return fmt.Sprintf("%s\n%s\n%s\n%s",
+		prefix,
+		prevSig,
+		zeroLenSig,
+		hex.EncodeToString(chunkHash))
+}
+
+// The provided p should have all of the previous chunk data and trailer
+// consumed already. The positioning here is expected that p[0] starts the
+// new chunk size with the ";chunk-signature=" following. The only exception
+// is if we started consuming the trailer, but hit the end of the read buffer.
+// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
+// finish consuming the final trailer bytes.
+// This parses the chunk metadata in situ without allocating an extra buffer.
+// It will just read and validate the chunk metadata and then move the
+// following chunk data to overwrite the metadata in the provided buffer.
+func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
+	n := len(p)
+
+	if !cr.skipcheck && cr.parsedSig != "" {
+		chunkhash := cr.chunkHash.Sum(nil)
+		cr.chunkHash.Reset()
+
+		sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
+		cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
+
+		if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
+	}
+
+	if cr.trailerExpected != 0 {
+		if len(p) < len(chunkHdrDelim) {
+			// This is the special case where we need to consume the
+			// trailer, but instead hit the end of the buffer. The
+			// subsequent call will finish consuming the trailer.
+			cr.chunkDataLeft = 0
+			cr.trailerExpected -= len(p)
+			cr.skipcheck = true
+			return 0, nil
+		}
+		// move data up to remove trailer
+		copy(p, p[cr.trailerExpected:])
+		n -= cr.trailerExpected
+	}
+
+	cr.skipcheck = false
+
+	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
+	cr.currentChunkSize = chunkSize
+	cr.parsedSig = sig
+	if err == errskipHeader {
+		cr.chunkDataLeft = 0
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	if chunkSize == 0 {
+		return 0, io.EOF
+	}
+
+	cr.trailerExpected = len(chunkHdrDelim)
+
+	// move data up to remove chunk header
+	copy(p, p[bufOffset:n])
+	n -= bufOffset
+
+	// if remaining buffer larger than chunk data,
+	// parse next header in buffer
+	if int64(n) > chunkSize {
+		cr.chunkDataLeft = 0
+		cr.chunkHash.Write(p[:chunkSize])
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		if (chunkSize + int64(n)) > math.MaxInt {
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
+		return n + int(chunkSize), err
+	}
+
+	cr.chunkDataLeft = chunkSize - int64(n)
+	cr.chunkHash.Write(p[:n])
+
+	return n, nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+// Task 3: Calculate Signature
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
+func getSigningKey(secret, region string, date time.Time) []byte {
+	dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
+	dateRegionKey := hmac256(dateKey, []byte(region))
+	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
+	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
+	return signingKey
+}
+
+func hmac256(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+var (
+	errInvalidChunkFormat = errors.New("invalid chunk header format")
+	errskipHeader         = errors.New("skip to next header")
+)
+
+const (
+	maxHeaderSize = 1024
+)
+
+// This returns the chunk payload size, signature, data start offset, and
+// error if any. See the AWS documentation for the chunk header format. The
+// header[0] byte is expected to be the first byte of the chunk size here.
+func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
+	stashLen := len(cr.stash)
+	if stashLen > maxHeaderSize {
+		return 0, "", 0, errInvalidChunkFormat
+	}
+	if cr.stash != nil {
+		tmp := make([]byte, maxHeaderSize)
+		copy(tmp, cr.stash)
+		copy(tmp[len(cr.stash):], header)
+		header = tmp
+		cr.stash = nil
+	}
+
+	semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
+	if semicolonIndex == -1 {
+		cr.stash = make([]byte, len(header))
+		copy(cr.stash, header)
+		cr.trailerExpected = 0
+		return 0, "", 0, errskipHeader
+	}
+
+	sigIndex := semicolonIndex + len(chunkHdrStr)
+	sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
+	if sigEndIndex == -1 {
+		cr.stash = make([]byte, len(header))
+		copy(cr.stash, header)
+		cr.trailerExpected = 0
+		return 0, "", 0, errskipHeader
+	}
+
+	chunkSizeBytes := header[:semicolonIndex]
+	chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
+	if err != nil {
+		return 0, "", 0, errInvalidChunkFormat
+	}
+
+	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
+	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
+
+	return chunkSize, signature, dataStartOffset - stashLen, nil
+}

s3api/utils/unsigned-chunk-reader.go

@@ -0,0 +1,235 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"hash"
+	"hash/crc32"
+	"hash/crc64"
+	"io"
+	"math/bits"
+	"strconv"
+	"strings"
+)
+
+var (
+	trailerDelim         = []byte{'\n', '\r', '\n'}
+	errMalformedEncoding = errors.New("malformed chunk encoding")
+)
+
+type UnsignedChunkReader struct {
+	reader           *bufio.Reader
+	checksumType     checksumType
+	expectedChecksum string
+	hasher           hash.Hash
+	stash            []byte
+	chunkCounter     int
+	offset           int
+}
+
+func NewUnsignedChunkReader(r io.Reader, ct checksumType) (*UnsignedChunkReader, error) {
+	hasher, err := getHasher(ct)
+	if err != nil {
+		return nil, err
+	}
+	return &UnsignedChunkReader{
+		reader:       bufio.NewReader(r),
+		checksumType: ct,
+		stash:        make([]byte, 0),
+		hasher:       hasher,
+		chunkCounter: 1,
+	}, nil
+}
+
+func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
+	// First read any stashed data
+	if len(ucr.stash) != 0 {
+		n := copy(p, ucr.stash)
+		ucr.offset += n
+
+		if n < len(ucr.stash) {
+			ucr.stash = ucr.stash[n:]
+			ucr.offset = 0
+			return n, nil
+		}
+	}
+
+	for {
+		// Read the chunk size
+		chunkSize, err := ucr.extractChunkSize()
+		if err != nil {
+			return 0, err
+		}
+
+		if chunkSize == 0 {
+			// Stop reading parsing payloads as 0 sized chunk is reached
+			break
+		}
+		rdr := io.TeeReader(ucr.reader, ucr.hasher)
+		payload := make([]byte, chunkSize)
+		// Read and cache the payload
+		_, err = io.ReadFull(rdr, payload)
+		if err != nil {
+			return 0, err
+		}
+
+		// Skip the trailing "\r\n"
+		if err := ucr.readAndSkip('\r', '\n'); err != nil {
+			return 0, err
+		}
+
+		// Copy the payload into the io.Reader buffer
+		n := copy(p[ucr.offset:], payload)
+		ucr.offset += n
+		ucr.chunkCounter++
+
+		if int64(n) < chunkSize {
+			// stash the remaining data
+			ucr.stash = payload[n:]
+			dataRead := ucr.offset
+			ucr.offset = 0
+			return dataRead, nil
+		}
+	}
+
+	// Read and validate trailers
+	if err := ucr.readTrailer(); err != nil {
+		return 0, err
+	}
+
+	return ucr.offset, io.EOF
+}
+
+// Reads and validates the bytes provided from the underlying io.Reader
+func (ucr *UnsignedChunkReader) readAndSkip(data ...byte) error {
+	for _, d := range data {
+		b, err := ucr.reader.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+
+		if b != d {
+			return errMalformedEncoding
+		}
+	}
+
+	return nil
+}
+
+// Extracts the chunk size from the payload
+func (ucr *UnsignedChunkReader) extractChunkSize() (int64, error) {
+	line, err := ucr.reader.ReadString('\n')
+	if err != nil {
+		return 0, errMalformedEncoding
+	}
+	line = strings.TrimSpace(line)
+
+	chunkSize, err := strconv.ParseInt(line, 16, 64)
+	if err != nil {
+		return 0, errMalformedEncoding
+	}
+
+	return chunkSize, nil
+}
+
+// Reads and validates the trailer at the end
+func (ucr *UnsignedChunkReader) readTrailer() error {
+	var trailerBuffer bytes.Buffer
+
+	for {
+		v, err := ucr.reader.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+		if v != '\r' {
+			trailerBuffer.WriteByte(v)
+			continue
+		}
+		var tmp [3]byte
+		_, err = io.ReadFull(ucr.reader, tmp[:])
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+		if !bytes.Equal(tmp[:], trailerDelim) {
+			return errMalformedEncoding
+		}
+		break
+	}
+
+	// Parse the trailer
+	trailerHeader := trailerBuffer.String()
+	trailerHeader = strings.TrimSpace(trailerHeader)
+	trailerHeaderParts := strings.Split(trailerHeader, ":")
+	if len(trailerHeaderParts) != 2 {
+		return errMalformedEncoding
+	}
+
+	if trailerHeaderParts[0] != string(ucr.checksumType) {
+		//TODO: handle the error
+		return errMalformedEncoding
+	}
+
+	ucr.expectedChecksum = trailerHeaderParts[1]
+
+	// Validate checksum
+	return ucr.validateChecksum()
+}
+
+// Validates the trailing checksum sent at the end
+func (ucr *UnsignedChunkReader) validateChecksum() error {
+	csum := ucr.hasher.Sum(nil)
+	checksum := base64.StdEncoding.EncodeToString(csum)
+
+	if checksum != ucr.expectedChecksum {
+		return fmt.Errorf("actual checksum: %v, expected checksum: %v", checksum, ucr.expectedChecksum)
+	}
+
+	return nil
+}
+
+// Retruns the hash calculator based on the hash type provided
+func getHasher(ct checksumType) (hash.Hash, error) {
+	switch ct {
+	case checksumTypeCrc32:
+		return crc32.NewIEEE(), nil
+	case checksumTypeCrc32c:
+		return crc32.New(crc32.MakeTable(crc32.Castagnoli)), nil
+	case checksumTypeCrc64nvme:
+		table := crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))
+		return crc64.New(table), nil
+	case checksumTypeSha1:
+		return sha1.New(), nil
+	case checksumTypeSha256:
+		return sha256.New(), nil
+	default:
+		return nil, errors.New("unsupported checksum type")
+	}
+}

s3api/utils/utils.go

@@ -16,6 +16,7 @@ package utils
 
 import (
 	"bytes"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -39,6 +40,10 @@ var (
 	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
+const (
+	upperhex = "0123456789ABCDEF"
+)
+
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
 	headers.DisableNormalizing()
@@ -64,7 +69,9 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		body = bytes.NewReader(req.Body())
 	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
+	escapedURI := escapeOriginalURI(ctx)
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -172,9 +179,15 @@ func ParseUint(str string) (int32, error) {
 	if str == "" {
 		return 1000, nil
 	}
-	num, err := strconv.ParseUint(str, 10, 16)
+	num, err := strconv.ParseInt(str, 10, 32)
 	if err != nil {
-		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+		return 1000, fmt.Errorf("invalid int: %w", err)
+	}
+	if num < 0 {
+		return 1000, fmt.Errorf("negative uint: %v", num)
+	}
+	if num > 1000 {
+		num = 1000
 	}
 	return int32(num), nil
 }
@@ -190,6 +203,13 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
 	}
 }
 
+// Streams the response body by chunks
+func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser, bodysize int) {
+	// SetBodyStream will call Close() on the reader when the stream is done
+	// since rdr is a ReadCloser
+	ctx.Context().SetBodyStream(rdr, bodysize)
+}
+
 func IsValidBucketName(bucket string) bool {
 	if len(bucket) < 3 || len(bucket) > 63 {
 		return false
@@ -248,35 +268,53 @@ func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
 	return
 }
 
-func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResult) s3response.GetObjectAttributesResult {
-	if _, ok := attrs[types.ObjectAttributesEtag]; !ok {
+func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResponse) s3response.GetObjectAttributesResponse {
+	// These properties shouldn't appear in the final response body
+	output.LastModified = nil
+	output.VersionId = nil
+	output.DeleteMarker = nil
+
+	if _, ok := attrs[s3response.ObjectAttributesEtag]; !ok {
 		output.ETag = nil
 	}
-	if _, ok := attrs[types.ObjectAttributesObjectParts]; !ok {
+	if _, ok := attrs[s3response.ObjectAttributesObjectParts]; !ok {
 		output.ObjectParts = nil
 	}
-	if _, ok := attrs[types.ObjectAttributesObjectSize]; !ok {
+	if _, ok := attrs[s3response.ObjectAttributesObjectSize]; !ok {
 		output.ObjectSize = nil
 	}
-	if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
-		output.StorageClass = nil
+	if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
+		output.StorageClass = ""
+	}
+	if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok {
+		output.Checksum = nil
 	}
 
 	return output
 }
 
-func ParseObjectAttributes(ctx *fiber.Ctx) map[types.ObjectAttributes]struct{} {
-	attrs := map[types.ObjectAttributes]struct{}{}
+func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
+	attrs := map[s3response.ObjectAttributes]struct{}{}
+	var err error
 	ctx.Request().Header.VisitAll(func(key, value []byte) {
 		if string(key) == "X-Amz-Object-Attributes" {
 			oattrs := strings.Split(string(value), ",")
 			for _, a := range oattrs {
-				attrs[types.ObjectAttributes(a)] = struct{}{}
+				attr := s3response.ObjectAttributes(a)
+				if !attr.IsValid() {
+					err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
+					break
+				}
+				attrs[attr] = struct{}{}
 			}
 		}
 	})
 
-	return attrs
+	if len(attrs) == 0 {
+		return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
+	}
+
+	return attrs, err
 }
 
 type objLockCfg struct {
@@ -311,13 +349,13 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 	if objLockMode != "" &&
 		objLockMode != types.ObjectLockModeCompliance &&
 		objLockMode != types.ObjectLockModeGovernance {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return nil, s3err.GetAPIError(s3err.ErrInvalidObjectLockMode)
 	}
 
 	legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)
 
 	if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return nil, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus)
 	}
 
 	return &objLockCfg{
@@ -326,3 +364,270 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
 		LegalHoldStatus: legalHold,
 	}, nil
 }
+
+func IsValidOwnership(val types.ObjectOwnership) bool {
+	switch val {
+	case types.ObjectOwnershipBucketOwnerEnforced:
+		return true
+	case types.ObjectOwnershipBucketOwnerPreferred:
+		return true
+	case types.ObjectOwnershipObjectWriter:
+		return true
+	default:
+		return false
+	}
+}
+
+func escapeOriginalURI(ctx *fiber.Ctx) string {
+	path := ctx.Path()
+
+	// Escape the URI original path
+	escapedURI := escapePath(path)
+
+	// Add the URI query params
+	query := string(ctx.Request().URI().QueryArgs().QueryString())
+	if query != "" {
+		escapedURI = escapedURI + "?" + query
+	}
+
+	return escapedURI
+}
+
+// Escapes the path string
+// Most of the parts copied from std url
+func escapePath(s string) string {
+	hexCount := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			hexCount++
+		}
+	}
+
+	if hexCount == 0 {
+		return s
+	}
+
+	var buf [64]byte
+	var t []byte
+
+	required := len(s) + 2*hexCount
+	if required <= len(buf) {
+		t = buf[:required]
+	} else {
+		t = make([]byte, required)
+	}
+
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case shouldEscape(c):
+			t[j] = '%'
+			t[j+1] = upperhex[c>>4]
+			t[j+2] = upperhex[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+
+	return string(t)
+}
+
+// Checks if the character needs to be escaped
+func shouldEscape(c byte) bool {
+	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '_', '.', '~', '/':
+		return false
+	}
+
+	return true
+}
+
+func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) {
+	sdkAlgorithm := types.ChecksumAlgorithm(strings.ToUpper(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")))
+
+	err := IsChecksumAlgorithmValid(sdkAlgorithm)
+	if err != nil {
+		return "", nil, err
+	}
+
+	checksums := map[types.ChecksumAlgorithm]string{}
+
+	var hdrErr error
+	// Parse and validate checksum headers
+	ctx.Request().Header.VisitAll(func(key, value []byte) {
+		// Skip `X-Amz-Checksum-Type` as it's a special header
+		if hdrErr != nil || !strings.HasPrefix(string(key), "X-Amz-Checksum-") || string(key) == "X-Amz-Checksum-Type" {
+			return
+		}
+
+		algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(key), "X-Amz-Checksum-")))
+		err := IsChecksumAlgorithmValid(algo)
+		if err != nil {
+			hdrErr = s3err.GetAPIError(s3err.ErrInvalidChecksumHeader)
+			return
+		}
+
+		checksums[algo] = string(value)
+	})
+
+	if hdrErr != nil {
+		return sdkAlgorithm, nil, hdrErr
+	}
+
+	headerCtr := 0
+	for al, val := range checksums {
+		if val != "" && !IsValidChecksum(val, al) {
+			return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al))))
+		}
+		// If any other checksum value is provided,
+		// rather than x-amz-sdk-checksum-algorithm
+		if sdkAlgorithm != "" && sdkAlgorithm != al && val != "" {
+			return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
+		}
+		if val != "" {
+			sdkAlgorithm = al
+			headerCtr++
+		}
+
+		if headerCtr > 1 {
+			return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
+		}
+	}
+
+	return sdkAlgorithm, checksums, nil
+}
+
+var checksumLengths = map[types.ChecksumAlgorithm]int{
+	types.ChecksumAlgorithmCrc32:     4,
+	types.ChecksumAlgorithmCrc32c:    4,
+	types.ChecksumAlgorithmCrc64nvme: 8,
+	types.ChecksumAlgorithmSha1:      20,
+	types.ChecksumAlgorithmSha256:    32,
+}
+
+func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
+	decoded, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		return false
+	}
+
+	expectedLength, exists := checksumLengths[algorithm]
+	if !exists {
+		return false
+	}
+
+	return len(decoded) == expectedLength
+}
+
+func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
+	alg = types.ChecksumAlgorithm(strings.ToUpper(string(alg)))
+	if alg != "" &&
+		alg != types.ChecksumAlgorithmCrc32 &&
+		alg != types.ChecksumAlgorithmCrc32c &&
+		alg != types.ChecksumAlgorithmSha1 &&
+		alg != types.ChecksumAlgorithmSha256 &&
+		alg != types.ChecksumAlgorithmCrc64nvme {
+		return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
+	}
+
+	return nil
+}
+
+// Validates the provided checksum type
+func IsChecksumTypeValid(t types.ChecksumType) error {
+	if t != "" &&
+		t != types.ChecksumTypeComposite &&
+		t != types.ChecksumTypeFullObject {
+		return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")
+	}
+	return nil
+}
+
+type checksumTypeSchema map[types.ChecksumType]struct{}
+type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema
+
+// A table defining the checksum algorithm/type support
+var checksumMap checksumSchema = checksumSchema{
+	types.ChecksumAlgorithmCrc32: checksumTypeSchema{
+		types.ChecksumTypeComposite:  struct{}{},
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	types.ChecksumAlgorithmCrc32c: checksumTypeSchema{
+		types.ChecksumTypeComposite:  struct{}{},
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	types.ChecksumAlgorithmSha1: checksumTypeSchema{
+		types.ChecksumTypeComposite: struct{}{},
+		"":                          struct{}{},
+	},
+	types.ChecksumAlgorithmSha256: checksumTypeSchema{
+		types.ChecksumTypeComposite: struct{}{},
+		"":                          struct{}{},
+	},
+	types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{
+		types.ChecksumTypeFullObject: struct{}{},
+		"":                           struct{}{},
+	},
+	// Both could be empty
+	"": checksumTypeSchema{
+		"": struct{}{},
+	},
+}
+
+// Checks if checksum type and algorithm are supported together
+func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType) error {
+	typeSchema := checksumMap[algo]
+	_, ok := typeSchema[t]
+	if !ok {
+		return s3err.GetChecksumSchemaMismatchErr(algo, t)
+	}
+
+	return nil
+}
+
+// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers
+func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) {
+	algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
+	if err := IsChecksumAlgorithmValid(algo); err != nil {
+		return "", "", err
+	}
+
+	chType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
+	if err := IsChecksumTypeValid(chType); err != nil {
+		return "", "", err
+	}
+
+	// Verify if checksum algorithm is provided, if
+	// checksum type is specified
+	if chType != "" && algo == "" {
+		return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)
+	}
+
+	// Verify if the checksum type is supported for
+	// the provided checksum algorithm
+	if err := checkChecksumTypeAndAlgo(algo, chType); err != nil {
+		return algo, chType, err
+	}
+
+	// x-amz-checksum-type defaults to COMPOSITE
+	// if x-amz-checksum-algorithm is set except
+	// for the CRC64NVME algorithm: it defaults to FULL_OBJECT
+	if algo != "" && chType == "" {
+		if algo == types.ChecksumAlgorithmCrc64nvme {
+			chType = types.ChecksumTypeFullObject
+		} else {
+			chType = types.ChecksumTypeComposite
+		}
+	}
+
+	return algo, chType, nil
+}

s3api/utils/utils_test.go

@@ -19,10 +19,12 @@ import (
 	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3response"
 )
 
@@ -266,6 +268,14 @@ func TestParseUint(t *testing.T) {
 			want:    23,
 			wantErr: false,
 		},
+		{
+			name: "Parse-uint-greater-than-1000",
+			args: args{
+				str: "25000000",
+			},
+			want:    1000,
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -283,47 +293,68 @@ func TestParseUint(t *testing.T) {
 
 func TestFilterObjectAttributes(t *testing.T) {
 	type args struct {
-		attrs  map[types.ObjectAttributes]struct{}
-		output s3response.GetObjectAttributesResult
+		attrs  map[s3response.ObjectAttributes]struct{}
+		output s3response.GetObjectAttributesResponse
 	}
+
 	etag, objSize := "etag", int64(3222)
+	delMarker := true
+
 	tests := []struct {
 		name string
 		args args
-		want s3response.GetObjectAttributesResult
+		want s3response.GetObjectAttributesResponse
 	}{
 		{
 			name: "keep only ETag",
 			args: args{
-				attrs: map[types.ObjectAttributes]struct{}{
-					types.ObjectAttributesEtag: {},
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag: {},
 				},
-				output: s3response.GetObjectAttributesResult{
+				output: s3response.GetObjectAttributesResponse{
 					ObjectSize: &objSize,
 					ETag:       &etag,
 				},
 			},
-			want: s3response.GetObjectAttributesResult{ETag: &etag},
+			want: s3response.GetObjectAttributesResponse{ETag: &etag},
 		},
 		{
 			name: "keep multiple props",
 			args: args{
-				attrs: map[types.ObjectAttributes]struct{}{
-					types.ObjectAttributesEtag:         {},
-					types.ObjectAttributesObjectSize:   {},
-					types.ObjectAttributesStorageClass: {},
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag:         {},
+					s3response.ObjectAttributesObjectSize:   {},
+					s3response.ObjectAttributesStorageClass: {},
 				},
-				output: s3response.GetObjectAttributesResult{
+				output: s3response.GetObjectAttributesResponse{
 					ObjectSize:  &objSize,
 					ETag:        &etag,
 					ObjectParts: &s3response.ObjectParts{},
 					VersionId:   &etag,
 				},
 			},
-			want: s3response.GetObjectAttributesResult{
+			want: s3response.GetObjectAttributesResponse{
 				ETag:       &etag,
 				ObjectSize: &objSize,
-				VersionId:  &etag,
+			},
+		},
+		{
+			name: "make sure LastModified, DeleteMarker and VersionId are removed",
+			args: args{
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag: {},
+				},
+				output: s3response.GetObjectAttributesResponse{
+					ObjectSize:   &objSize,
+					ETag:         &etag,
+					ObjectParts:  &s3response.ObjectParts{},
+					VersionId:    &etag,
+					LastModified: backend.GetTimePtr(time.Now()),
+					DeleteMarker: &delMarker,
+				},
+			},
+			want: s3response.GetObjectAttributesResponse{
+				ETag: &etag,
 			},
 		},
 	}
@@ -335,3 +366,494 @@ func TestFilterObjectAttributes(t *testing.T) {
 		})
 	}
 }
+
+func TestIsValidOwnership(t *testing.T) {
+	type args struct {
+		val types.ObjectOwnership
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "valid-BucketOwnerEnforced",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerEnforced,
+			},
+			want: true,
+		},
+		{
+			name: "valid-BucketOwnerPreferred",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerPreferred,
+			},
+			want: true,
+		},
+		{
+			name: "valid-ObjectWriter",
+			args: args{
+				val: types.ObjectOwnershipObjectWriter,
+			},
+			want: true,
+		},
+		{
+			name: "invalid_value",
+			args: args{
+				val: types.ObjectOwnership("invalid_value"),
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidOwnership(tt.args.val); got != tt.want {
+				t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_shouldEscape(t *testing.T) {
+	type args struct {
+		c byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "shouldn't-escape-alphanum",
+			args: args{
+				c: 'h',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-char",
+			args: args{
+				c: '_',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-number",
+			args: args{
+				c: '0',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-path-separator",
+			args: args{
+				c: '/',
+			},
+			want: false,
+		},
+		{
+			name: "should-escape-special-char-1",
+			args: args{
+				c: '&',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-2",
+			args: args{
+				c: '*',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-3",
+			args: args{
+				c: '(',
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldEscape(tt.args.c); got != tt.want {
+				t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_escapePath(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "empty-string",
+			args: args{
+				s: "",
+			},
+			want: "",
+		},
+		{
+			name: "alphanum-path",
+			args: args{
+				s: "/test-bucket/test-key",
+			},
+			want: "/test-bucket/test-key",
+		},
+		{
+			name: "path-with-unescapable-chars",
+			args: args{
+				s: "/test~bucket/test.key",
+			},
+			want: "/test~bucket/test.key",
+		},
+		{
+			name: "path-with-escapable-chars",
+			args: args{
+				s: "/bucket-*(/test=key&",
+			},
+			want: "/bucket-%2A%28/test%3Dkey%26",
+		},
+		{
+			name: "path-with-space",
+			args: args{
+				s: "/test-bucket/my key",
+			},
+			want: "/test-bucket/my%20key",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := escapePath(tt.args.s); got != tt.want {
+				t.Errorf("escapePath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsChecksumAlgorithmValid(t *testing.T) {
+	type args struct {
+		alg types.ChecksumAlgorithm
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "empty",
+			args: args{
+				alg: "",
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc32",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc32,
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc32c",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc32c,
+			},
+			wantErr: false,
+		},
+		{
+			name: "sha1",
+			args: args{
+				alg: types.ChecksumAlgorithmSha1,
+			},
+			wantErr: false,
+		},
+		{
+			name: "sha256",
+			args: args{
+				alg: types.ChecksumAlgorithmSha256,
+			},
+			wantErr: false,
+		},
+		{
+			name: "crc64nvme",
+			args: args{
+				alg: types.ChecksumAlgorithmCrc64nvme,
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid",
+			args: args{
+				alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := IsChecksumAlgorithmValid(tt.args.alg); (err != nil) != tt.wantErr {
+				t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestIsValidChecksum(t *testing.T) {
+	type args struct {
+		checksum  string
+		algorithm types.ChecksumAlgorithm
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "invalid-base64",
+			args: args{
+				checksum:  "invalid_base64_string",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: false,
+		},
+		{
+			name: "invalid-crc32",
+			args: args{
+				checksum:  "YXNkZmFzZGZhc2Rm",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: false,
+		},
+		{
+			name: "valid-crc32",
+			args: args{
+				checksum:  "ww2FVQ==",
+				algorithm: types.ChecksumAlgorithmCrc32,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-crc32c",
+			args: args{
+				checksum:  "Zmdoa2doZmtnZmhr",
+				algorithm: types.ChecksumAlgorithmCrc32c,
+			},
+			want: false,
+		},
+		{
+			name: "valid-crc32c",
+			args: args{
+				checksum:  "DOsb4w==",
+				algorithm: types.ChecksumAlgorithmCrc32c,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-sha1",
+			args: args{
+				checksum:  "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm",
+				algorithm: types.ChecksumAlgorithmSha1,
+			},
+			want: false,
+		},
+		{
+			name: "valid-sha1",
+			args: args{
+				checksum:  "L4q6V59Zcwn12wyLIytoE2c1ugk=",
+				algorithm: types.ChecksumAlgorithmSha1,
+			},
+			want: true,
+		},
+		{
+			name: "invalid-sha256",
+			args: args{
+				checksum:  "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==",
+				algorithm: types.ChecksumAlgorithmSha256,
+			},
+			want: false,
+		},
+		{
+			name: "valid-sha256",
+			args: args{
+				checksum:  "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
+				algorithm: types.ChecksumAlgorithmSha256,
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm); got != tt.want {
+				t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsChecksumTypeValid(t *testing.T) {
+	type args struct {
+		t types.ChecksumType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "valid_FULL_OBJECT",
+			args: args{
+				t: types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "valid_COMPOSITE",
+			args: args{
+				t: types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid",
+			args: args{
+				t: types.ChecksumType("invalid_checksum_type"),
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := IsChecksumTypeValid(tt.args.t); (err != nil) != tt.wantErr {
+				t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_checkChecksumTypeAndAlgo(t *testing.T) {
+	type args struct {
+		algo types.ChecksumAlgorithm
+		t    types.ChecksumType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "full_object-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-crc32c",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32c,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-sha1",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-sha256",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-crc64nvme",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc64nvme,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "full_object-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeFullObject,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc32",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc32c",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc32c,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-sha1",
+			args: args{
+				algo: types.ChecksumAlgorithmSha1,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-sha256",
+			args: args{
+				algo: types.ChecksumAlgorithmSha256,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: false,
+		},
+		{
+			name: "composite-crc64nvme",
+			args: args{
+				algo: types.ChecksumAlgorithmCrc64nvme,
+				t:    types.ChecksumTypeComposite,
+			},
+			wantErr: true,
+		},
+		{
+			name: "composite-empty",
+			args: args{
+				t: types.ChecksumTypeComposite,
+			},
+			wantErr: true,
+		},
+		{
+			name: "full_object-empty",
+			args: args{
+				t: types.ChecksumTypeFullObject,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t); (err != nil) != tt.wantErr {
+				t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -17,7 +17,11 @@ package s3err
 import (
 	"bytes"
 	"encoding/xml"
+	"fmt"
 	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 // APIError structure
@@ -57,6 +61,7 @@ const (
 	ErrAccessDenied
 	ErrMethodNotAllowed
 	ErrBucketNotEmpty
+	ErrVersionedBucketNotEmpty
 	ErrBucketAlreadyExists
 	ErrBucketAlreadyOwnedByYou
 	ErrNoSuchBucket
@@ -65,14 +70,20 @@ const (
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidMaxKeys
+	ErrInvalidMaxBuckets
 	ErrInvalidMaxUploads
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
+	ErrInvalidObjectAttributes
 	ErrInvalidPart
+	ErrEmptyParts
 	ErrInvalidPartNumber
+	ErrInvalidPartOrder
+	ErrInvalidCompleteMpPartNumber
 	ErrInternalError
 	ErrInvalidCopyDest
 	ErrInvalidCopySource
+	ErrInvalidCopySourceRange
 	ErrInvalidTag
 	ErrAuthHeaderEmpty
 	ErrSignatureVersionNotSupported
@@ -102,6 +113,7 @@ const (
 	ErrSignatureTerminationStr
 	ErrSignatureIncorrService
 	ErrContentSHA256Mismatch
+	ErrMissingDecodedContentLength
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
 	ErrMissingDateHeader
@@ -119,16 +131,47 @@ const (
 	ErrObjectLocked
 	ErrPastObjectLockRetainDate
 	ErrObjectLockInvalidRetentionPeriod
+	ErrInvalidLegalHoldStatus
+	ErrInvalidObjectLockMode
 	ErrNoSuchBucketPolicy
 	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
+	ErrObjectAttributesInvalidHeader
 	ErrRequestTimeTooSkewed
+	ErrInvalidBucketAclWithObjectOwnership
+	ErrBothCannedAndHeaderGrants
+	ErrOwnershipControlsNotFound
+	ErrAclNotSupported
+	ErrMalformedACL
+	ErrUnexpectedContent
+	ErrMissingSecurityHeader
+	ErrInvalidMetadataDirective
+	ErrInvalidTaggingDirective
+	ErrKeyTooLong
+	ErrInvalidVersionId
+	ErrNoSuchVersion
+	ErrSuspendedVersioningNotAllowed
+	ErrMultipleChecksumHeaders
+	ErrInvalidChecksumAlgorithm
+	ErrInvalidChecksumPart
+	ErrChecksumTypeWithAlgo
+	ErrInvalidChecksumHeader
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
+	ErrDirectoryNotEmpty
 	ErrQuotaExceeded
+	ErrVersioningNotConfigured
+
+	// Admin api errors
+	ErrAdminAccessDenied
+	ErrAdminUserNotFound
+	ErrAdminUserExists
+	ErrAdminInvalidUserRole
+	ErrAdminMissingUserAcess
+	ErrAdminMethodNotSupported
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -144,7 +187,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
-		Description:    "The bucket you tried to delete is not empty",
+		Description:    "The bucket you tried to delete is not empty.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrVersionedBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty. You must delete all versions in the bucket.",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrBucketAlreadyExists: {
@@ -167,19 +215,24 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The Content-Md5 you specified is not valid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidMaxBuckets: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-buckets must be an integer between 1 and 10000.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidMaxUploads: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxKeys: {
 		Code:           "InvalidArgument",
-		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartNumberMarker: {
@@ -187,9 +240,14 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Argument partNumberMarker must be an integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidObjectAttributes: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid attribute name specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchBucket: {
 		Code:           "NoSuchBucket",
-		Description:    "The specified bucket does not exist",
+		Description:    "The specified bucket does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchKey: {
@@ -212,9 +270,24 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrEmptyParts: {
+		Code:           "InvalidRequest",
+		Description:    "You must specify at least one part",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidPartNumber: {
 		Code:           "InvalidArgument",
-		Description:    "Part number must be an integer between 1 and 10000, inclusive",
+		Description:    "Part number must be an integer between 1 and 10000, inclusive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartOrder: {
+		Code:           "InvalidPartOrder",
+		Description:    "The list of parts was not in ascending order. Parts must be ordered by part number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCompleteMpPartNumber: {
+		Code:           "InvalidArgument",
+		Description:    "PartNumber must be >= 1",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidCopyDest: {
@@ -227,6 +300,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidCopySourceRange: {
+		Code:           "InvalidArgument",
+		Description:    "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidTag: {
 		Code:           "InvalidArgument",
 		Description:    "The Tag value you have provided is invalid",
@@ -259,7 +337,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
-		Description:    "Invalid according to Policy: Policy Condition failed",
+		Description:    "Invalid according to Policy: Policy Condition failed.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrEntityTooSmall: {
@@ -294,7 +372,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrMalformedPresignedDate: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
+		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSignHeadersTag: {
@@ -309,7 +387,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
-		Description:    "There were headers present in the request which were not signed",
+		Description:    "There were headers present in the request which were not signed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidQueryParams: {
@@ -324,32 +402,32 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrExpiredPresignRequest: {
 		Code:           "AccessDenied",
-		Description:    "Request has expired",
+		Description:    "Request has expired.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMalformedExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires should be a number",
+		Description:    "X-Amz-Expires should be a number.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNegativeExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires must be non-negative",
+		Description:    "X-Amz-Expires must be non-negative.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
+		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
-		Description:    "The access key ID you provided does not exist in our records.",
+		Description:    "The AWS Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrRequestNotReadyYet: {
 		Code:           "AccessDenied",
-		Description:    "Request is not valid yet",
+		Description:    "Request is not valid yet.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureDoesNotMatch: {
@@ -359,17 +437,17 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrSignatureDateDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
+		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureTerminationStr: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'",
+		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureIncorrService: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Credential should be scoped to correct service: s3",
+		Description:    "Credential should be scoped to correct service: s3.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrContentSHA256Mismatch: {
@@ -377,34 +455,39 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingDecodedContentLength: {
+		Code:           "MissingContentLength",
+		Description:    "You must provide the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
 	ErrMissingDateHeader: {
 		Code:           "AccessDenied",
-		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidRequest: {
 		Code:           "InvalidRequest",
-		Description:    "Invalid Request",
+		Description:    "Invalid Request.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAuthNotSetup: {
 		Code:           "InvalidRequest",
-		Description:    "Signed request requires setting up SeaweedFS S3 authentication",
+		Description:    "Signed request requires setting up SeaweedFS S3 authentication.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
-		Description:    "A header you provided implies functionality that is not implemented",
+		Description:    "A header you provided implies functionality that is not implemented.",
 		HTTPStatusCode: http.StatusNotImplemented,
 	},
 	ErrPreconditionFailed: {
 		Code:           "PreconditionFailed",
-		Description:    "At least one of the pre-conditions you specified did not hold",
+		Description:    "At least one of the pre-conditions you specified did not hold.",
 		HTTPStatusCode: http.StatusPreconditionFailed,
 	},
 	ErrInvalidObjectState: {
 		Code:           "InvalidObjectState",
-		Description:    "The operation is not valid for the current state of the object",
+		Description:    "The operation is not valid for the current state of the object.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrInvalidRange: {
@@ -419,52 +502,67 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrObjectLockConfigurationNotFound: {
 		Code:           "ObjectLockConfigurationNotFoundError",
-		Description:    "Object Lock configuration does not exist for this bucket",
+		Description:    "Object Lock configuration does not exist for this bucket.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
-		Description:    "The specified object does not have an ObjectLock configuration",
+		Description:    "The specified object does not have a ObjectLock configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidBucketObjectLockConfiguration: {
 		Code:           "InvalidRequest",
-		Description:    "Bucket is missing ObjectLockConfiguration",
+		Description:    "Bucket is missing Object Lock Configuration.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectLockConfigurationNotAllowed: {
 		Code:           "InvalidBucketState",
-		Description:    "Object Lock configuration cannot be enabled on existing buckets",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrObjectLocked: {
 		Code:           "InvalidRequest",
-		Description:    "Object is WORM protected and cannot be overwritten",
+		Description:    "Object is WORM protected and cannot be overwritten.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrPastObjectLockRetainDate: {
 		Code:           "InvalidRequest",
-		Description:    "the retain until date must be in the future",
+		Description:    "the retain until date must be in the future.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectLockInvalidRetentionPeriod: {
 		Code:           "InvalidRetentionPeriod",
-		Description:    "the retention days/years must be positive integer",
+		Description:    "the retention days/years must be positive integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidLegalHoldStatus: {
+		Code:           "InvalidArgument",
+		Description:    "Legal Hold must be either of 'ON' or 'OFF'",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidObjectLockMode: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown wormMode directive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNoSuchBucketPolicy: {
 		Code:           "NoSuchBucketPolicy",
-		Description:    "The bucket policy does not exist",
+		Description:    "The bucket policy does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrBucketTaggingNotFound: {
 		Code:           "NoSuchTagSet",
-		Description:    "The TagSet does not exist",
+		Description:    "The TagSet does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrObjectLockInvalidHeaders: {
 		Code:           "InvalidRequest",
-		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
+		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectAttributesInvalidHeader: {
+		Code:           "InvalidRequest",
+		Description:    "The x-amz-object-attributes header specifying the attributes to be retrieved is either missing or empty",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrRequestTimeTooSkewed: {
@@ -472,6 +570,96 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The difference between the request time and the server's time is too large.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidBucketAclWithObjectOwnership: {
+		Code:           "ErrInvalidBucketAclWithObjectOwnership",
+		Description:    "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBothCannedAndHeaderGrants: {
+		Code:           "InvalidRequest",
+		Description:    "Specifying both Canned ACLs and Header Grants is not allowed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOwnershipControlsNotFound: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAclNotSupported: {
+		Code:           "AccessControlListNotSupported",
+		Description:    "The bucket does not allow ACLs.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedACL: {
+		Code:           "MalformedACLError",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrUnexpectedContent: {
+		Code:           "UnexpectedContent",
+		Description:    "This request does not support content.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSecurityHeader: {
+		Code:           "MissingSecurityHeader",
+		Description:    "Your request was missing a required header.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrInvalidMetadataDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown metadata directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidTaggingDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown tagging directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidVersionId: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid version id specified",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrKeyTooLong: {
+		Code:           "KeyTooLongError",
+		Description:    "Your key is too long.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNoSuchVersion: {
+		Code:           "NoSuchVersion",
+		Description:    "The specified version does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrSuspendedVersioningNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMultipleChecksumHeaders: {
+		Code:           "InvalidRequest",
+		Description:    "Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumAlgorithm: {
+		Code:           "InvalidRequest",
+		Description:    "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumPart: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid Base64 or multiple checksums present in request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrChecksumTypeWithAlgo: {
+		Code:           "InvalidRequest",
+		Description:    "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidChecksumHeader: {
+		Code:           "InvalidRequest",
+		Description:    "The algorithm type you specified in x-amz-checksum- header is invalid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// non aws errors
 	ErrExistingObjectIsDirectory: {
@@ -489,11 +677,53 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Directory object contains data payload.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrDirectoryNotEmpty: {
+		Code:           "ErrDirectoryNotEmpty",
+		Description:    "Directory object not empty.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrQuotaExceeded: {
 		Code:           "QuotaExceeded",
 		Description:    "Your request was denied due to quota exceeded.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrVersioningNotConfigured: {
+		Code:           "VersioningNotConfigured",
+		Description:    "Versioning has not been configured for the gateway.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+
+	// Admin api errors
+	ErrAdminAccessDenied: {
+		Code:           "XAdminAccessDenied",
+		Description:    "Only admin users have access to this resource.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAdminUserNotFound: {
+		Code:           "XAdminUserNotFound",
+		Description:    "No user exists with the provided access key ID.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminUserExists: {
+		Code:           "XAdminUserExists",
+		Description:    "A user with the provided access key ID already exists.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAdminInvalidUserRole: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User role has to be one of the following: 'user', 'admin', 'userplus'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminMissingUserAcess: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User access key ID is missing.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminMethodNotSupported: {
+		Code:           "XAdminMethodNotSupported",
+		Description:    "The method is not supported in single root user mode.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.
@@ -524,3 +754,72 @@ func encodeResponse(response interface{}) []byte {
 	e.Encode(response)
 	return bytesBuffer.Bytes()
 }
+
+// Returns invalid checksum error with the provided header in the error description
+func GetInvalidChecksumHeaderErr(header string) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Value for %v header is invalid.", header),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch APIError
+func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Checksum Type mismatch occurred, expected checksum Type: %v, actual checksum Type: %v", expected, actual),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns incorrect checksum APIError
+func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError {
+	return APIError{
+		Code:           "BadDigest",
+		Description:    fmt.Sprintf("The %v you specified did not match the calculated checksum.", strings.ToLower(string(algo))),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch error with checksum algorithm
+func GetChecksumSchemaMismatchErr(algo types.ChecksumAlgorithm, t types.ChecksumType) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The %v checksum type cannot be used with the %v checksum algorithm.", algo, strings.ToLower(string(t))),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// Returns checksum type mismatch error for multipart uploads
+func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The upload was created using the %v checksum mode. The complete request must use the same checksum mode.", t),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func GetIncorrectMpObjectSizeErr(expected, actual int64) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("The provided 'x-amz-mp-object-size' header value %v does not match what was computed: %v", expected, actual),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func GetInvalidMpObjectSizeErr(val int64) APIError {
+	return APIError{
+		Code:           "InvalidRequest",
+		Description:    fmt.Sprintf("Value for x-amz-mp-object-size header is less than zero: '%v'", val),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func CreateExceedingRangeErr(objSize int64) APIError {
+	return APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}

s3log/audit-logger.go

@@ -35,11 +35,13 @@ type LogMeta struct {
 	BucketOwner string
 	ObjectSize  int64
 	Action      string
+	HttpStatus  int
 }
 
 type LogConfig struct {
-	LogFile    string
-	WebhookURL string
+	LogFile      string
+	WebhookURL   string
+	AdminLogFile string
 }
 
 type LogFields struct {
@@ -71,18 +73,66 @@ type LogFields struct {
 	AclRequired        string
 }
 
-func InitLogger(cfg *LogConfig) (AuditLogger, error) {
+type AdminLogFields struct {
+	Time               time.Time
+	RemoteIP           string
+	Requester          string
+	RequestID          string
+	Operation          string
+	RequestURI         string
+	HttpStatus         int
+	ErrorCode          string
+	BytesSent          int
+	TotalTime          int64
+	TurnAroundTime     int64
+	Referer            string
+	UserAgent          string
+	SignatureVersion   string
+	CipherSuite        string
+	AuthenticationType string
+	TLSVersion         string
+}
+
+type Loggers struct {
+	S3Logger    AuditLogger
+	AdminLogger AuditLogger
+}
+
+func InitLogger(cfg *LogConfig) (*Loggers, error) {
 	if cfg.WebhookURL != "" && cfg.LogFile != "" {
 		return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
 	}
-	if cfg.WebhookURL != "" {
-		return InitWebhookLogger(cfg.WebhookURL)
-	}
-	if cfg.LogFile != "" {
-		return InitFileLogger(cfg.LogFile)
+	loggers := new(Loggers)
+
+	switch {
+	case cfg.WebhookURL != "":
+		fmt.Printf("initializing S3 access logs with '%v' webhook url\n", cfg.WebhookURL)
+		l, err := InitWebhookLogger(cfg.WebhookURL)
+		if err != nil {
+			return nil, err
+		}
+		loggers.S3Logger = l
+	case cfg.LogFile != "":
+		fmt.Printf("initializing S3 access logs with '%v' file\n", cfg.LogFile)
+		l, err := InitFileLogger(cfg.LogFile)
+		if err != nil {
+			return nil, err
+		}
+
+		loggers.S3Logger = l
 	}
 
-	return nil, nil
+	if cfg.AdminLogFile != "" {
+		fmt.Printf("initializing admin access logs with '%v' file\n", cfg.AdminLogFile)
+		l, err := InitAdminFileLogger(cfg.AdminLogFile)
+		if err != nil {
+			return nil, err
+		}
+
+		loggers.AdminLogger = l
+	}
+
+	return loggers, nil
 }
 
 func genID() string {

s3log/file_admin.go

@@ -0,0 +1,151 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"crypto/tls"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+)
+
+// FileLogger is a local file audit log
+type AdminFileLogger struct {
+	FileLogger
+}
+
+var _ AuditLogger = &AdminFileLogger{}
+
+// InitFileLogger initializes audit logs to local file
+func InitAdminFileLogger(logname string) (AuditLogger, error) {
+	f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("open log: %w", err)
+	}
+
+	f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return &AdminFileLogger{FileLogger: FileLogger{logfile: logname, f: f}}, nil
+}
+
+// Log sends log message to file logger
+func (f *AdminFileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if f.gotErr {
+		return
+	}
+
+	lf := AdminLogFields{}
+
+	access := "-"
+	reqURI := ctx.OriginalURL()
+	errorCode := ""
+	startTime := ctx.Locals("startTime").(time.Time)
+	tlsConnState := ctx.Context().TLSConnectionState()
+	if tlsConnState != nil {
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
+	}
+
+	if err != nil {
+		errorCode = err.Error()
+	}
+
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
+	}
+
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.RequestURI = reqURI
+	lf.HttpStatus = meta.HttpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+
+	f.writeLog(lf)
+}
+
+func (f *AdminFileLogger) writeLog(lf AdminLogFields) {
+	if lf.RemoteIP == "" {
+		lf.RemoteIP = "-"
+	}
+	if lf.Requester == "" {
+		lf.Requester = "-"
+	}
+	if lf.Operation == "" {
+		lf.Operation = "-"
+	}
+	if lf.RequestURI == "" {
+		lf.RequestURI = "-"
+	}
+	if lf.ErrorCode == "" {
+		lf.ErrorCode = "-"
+	}
+	if lf.Referer == "" {
+		lf.Referer = "-"
+	}
+	if lf.UserAgent == "" {
+		lf.UserAgent = "-"
+	}
+	if lf.CipherSuite == "" {
+		lf.CipherSuite = "-"
+	}
+	if lf.TLSVersion == "" {
+		lf.TLSVersion = "-"
+	}
+
+	log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
+		fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
+		lf.RemoteIP,
+		lf.Requester,
+		lf.RequestID,
+		lf.Operation,
+		lf.RequestURI,
+		lf.HttpStatus,
+		lf.ErrorCode,
+		lf.BytesSent,
+		lf.TotalTime,
+		lf.TurnAroundTime,
+		lf.Referer,
+		lf.UserAgent,
+		lf.SignatureVersion,
+		lf.CipherSuite,
+		lf.AuthenticationType,
+		lf.TLSVersion,
+	)
+
+	_, err := f.f.WriteString(log)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
+		// TODO: do we need to terminate on log error?
+		// set err for now so that we don't spew errors
+		f.gotErr = true
+	}
+}

s3response/s3response.go

@@ -16,32 +16,72 @@ package s3response
 
 import (
 	"encoding/xml"
+	"io"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
 )
 
+const (
+	iso8601TimeFormat         = "2006-01-02T15:04:05.000Z"
+	iso8601TimeFormatExtended = "2006-01-02T15:04:05.000000Z"
+	iso8601TimeFormatWithTZ   = "2006-01-02T15:04:05-0700"
+)
+
+type PutObjectOutput struct {
+	ETag              string
+	VersionID         string
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+	ChecksumType      types.ChecksumType
+}
+
 // Part describes part metadata.
 type Part struct {
-	PartNumber   int
-	LastModified string
-	ETag         string
-	Size         int64
+	PartNumber        int
+	LastModified      time.Time
+	ETag              string
+	Size              int64
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+}
+
+func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias Part
+	aux := &struct {
+		LastModified string `xml:"LastModified"`
+		*Alias
+	}{
+		Alias: (*Alias)(&p),
+	}
+
+	aux.LastModified = p.LastModified.UTC().Format(iso8601TimeFormat)
+
+	return e.EncodeElement(aux, start)
 }
 
 // ListPartsResponse - s3 api list parts response.
 type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
-	Bucket   string
-	Key      string
-	UploadID string `xml:"UploadId"`
+	Bucket            string
+	Key               string
+	UploadID          string `xml:"UploadId"`
+	ChecksumAlgorithm types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
 
 	Initiator Initiator
 	Owner     Owner
 
 	// The class of storage used to store the object.
-	StorageClass string
+	StorageClass types.StorageClass
 
 	PartNumberMarker     int
 	NextPartNumberMarker int
@@ -52,13 +92,35 @@ type ListPartsResult struct {
 	Parts []Part `xml:"Part"`
 }
 
-type GetObjectAttributesResult struct {
+type ObjectAttributes string
+
+const (
+	ObjectAttributesEtag         ObjectAttributes = "ETag"
+	ObjectAttributesChecksum     ObjectAttributes = "Checksum"
+	ObjectAttributesObjectParts  ObjectAttributes = "ObjectParts"
+	ObjectAttributesStorageClass ObjectAttributes = "StorageClass"
+	ObjectAttributesObjectSize   ObjectAttributes = "ObjectSize"
+)
+
+func (o ObjectAttributes) IsValid() bool {
+	return o == ObjectAttributesChecksum ||
+		o == ObjectAttributesEtag ||
+		o == ObjectAttributesObjectParts ||
+		o == ObjectAttributesObjectSize ||
+		o == ObjectAttributesStorageClass
+}
+
+type GetObjectAttributesResponse struct {
 	ETag         *string
-	LastModified *time.Time
 	ObjectSize   *int64
-	StorageClass *types.StorageClass
-	VersionId    *string
+	StorageClass types.StorageClass `xml:",omitempty"`
 	ObjectParts  *ObjectParts
+	Checksum     *types.Checksum
+
+	// Not included in the response body
+	VersionId    *string
+	LastModified *time.Time
+	DeleteMarker *bool
 }
 
 type ObjectParts struct {
@@ -91,14 +153,89 @@ type ListMultipartUploadsResult struct {
 	CommonPrefixes []CommonPrefix
 }
 
+type ListObjectsResult struct {
+	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	Name           *string
+	Prefix         *string
+	Marker         *string
+	NextMarker     *string
+	MaxKeys        *int32
+	Delimiter      *string
+	IsTruncated    *bool
+	Contents       []Object
+	CommonPrefixes []types.CommonPrefix
+	EncodingType   types.EncodingType
+}
+
+type ListObjectsV2Result struct {
+	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	Name                  *string
+	Prefix                *string
+	StartAfter            *string
+	ContinuationToken     *string
+	NextContinuationToken *string
+	KeyCount              *int32
+	MaxKeys               *int32
+	Delimiter             *string
+	IsTruncated           *bool
+	Contents              []Object
+	CommonPrefixes        []types.CommonPrefix
+	EncodingType          types.EncodingType
+}
+
+type Object struct {
+	ChecksumAlgorithm []types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
+	ETag              *string
+	Key               *string
+	LastModified      *time.Time
+	Owner             *types.Owner
+	RestoreStatus     *types.RestoreStatus
+	Size              *int64
+	StorageClass      types.ObjectStorageClass
+}
+
+func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias Object
+	aux := &struct {
+		LastModified *string `xml:"LastModified,omitempty"`
+		*Alias
+	}{
+		Alias: (*Alias)(&o),
+	}
+
+	if o.LastModified != nil {
+		formattedTime := o.LastModified.UTC().Format(iso8601TimeFormat)
+		aux.LastModified = &formattedTime
+	}
+
+	return e.EncodeElement(aux, start)
+}
+
 // Upload describes in progress multipart upload
 type Upload struct {
-	Key          string
-	UploadID     string `xml:"UploadId"`
-	Initiator    Initiator
-	Owner        Owner
-	StorageClass string
-	Initiated    string
+	Key               string
+	UploadID          string `xml:"UploadId"`
+	Initiator         Initiator
+	Owner             Owner
+	StorageClass      types.StorageClass
+	Initiated         time.Time
+	ChecksumAlgorithm types.ChecksumAlgorithm
+	ChecksumType      types.ChecksumType
+}
+
+func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias Upload
+	aux := &struct {
+		Initiated string `xml:"Initiated"`
+		*Alias
+	}{
+		Alias: (*Alias)(&u),
+	}
+
+	aux.Initiated = u.Initiated.UTC().Format(iso8601TimeFormat)
+
+	return e.EncodeElement(aux, start)
 }
 
 // CommonPrefix ListObjectsResponse common prefixes (directory abstraction)
@@ -163,10 +300,20 @@ type Bucket struct {
 	Owner string `json:"owner"`
 }
 
+type ListBucketsInput struct {
+	Owner             string
+	IsAdmin           bool
+	ContinuationToken string
+	Prefix            string
+	MaxBuckets        int32
+}
+
 type ListAllMyBucketsResult struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
-	Owner   CanonicalUser
-	Buckets ListAllMyBucketsList
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	Owner             CanonicalUser
+	Buckets           ListAllMyBucketsList
+	ContinuationToken string `xml:"ContinuationToken,omitempty"`
+	Prefix            string `xml:"Prefix,omitempty"`
 }
 
 type ListAllMyBucketsEntry struct {
@@ -174,6 +321,20 @@ type ListAllMyBucketsEntry struct {
 	CreationDate time.Time
 }
 
+func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias ListAllMyBucketsEntry
+	aux := &struct {
+		CreationDate string `xml:"CreationDate"`
+		*Alias
+	}{
+		Alias: (*Alias)(&r),
+	}
+
+	aux.CreationDate = r.CreationDate.UTC().Format(iso8601TimeFormat)
+
+	return e.EncodeElement(aux, start)
+}
+
 type ListAllMyBucketsList struct {
 	Bucket []ListAllMyBucketsEntry
 }
@@ -184,9 +345,38 @@ type CanonicalUser struct {
 }
 
 type CopyObjectResult struct {
-	XMLName      xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
-	LastModified time.Time
-	ETag         string
+	XMLName             xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	LastModified        time.Time
+	ETag                string
+	CopySourceVersionId string `xml:"-"`
+}
+
+type CopyPartResult struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
+	LastModified      time.Time
+	ETag              *string
+	ChecksumCRC32     *string
+	ChecksumCRC32C    *string
+	ChecksumSHA1      *string
+	ChecksumSHA256    *string
+	ChecksumCRC64NVME *string
+
+	// not included in the body
+	CopySourceVersionId string `xml:"-"`
+}
+
+func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	type Alias CopyObjectResult
+	aux := &struct {
+		LastModified string `xml:"LastModified"`
+		*Alias
+	}{
+		Alias: (*Alias)(&r),
+	}
+
+	aux.LastModified = r.LastModified.UTC().Format(iso8601TimeFormat)
+
+	return e.EncodeElement(aux, start)
 }
 
 type AccessControlPolicy struct {
@@ -217,3 +407,230 @@ type Grantee struct {
 	ID          string
 	DisplayName string
 }
+
+type OwnershipControls struct {
+	Rules []types.OwnershipControlsRule `xml:"Rule"`
+}
+
+type InitiateMultipartUploadResult struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+	Bucket   string
+	Key      string
+	UploadId string
+}
+
+type ListVersionsResult struct {
+	XMLName             xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"`
+	CommonPrefixes      []types.CommonPrefix
+	DeleteMarkers       []types.DeleteMarkerEntry `xml:"DeleteMarker"`
+	Delimiter           *string
+	EncodingType        types.EncodingType
+	IsTruncated         *bool
+	KeyMarker           *string
+	MaxKeys             *int32
+	Name                *string
+	NextKeyMarker       *string
+	NextVersionIdMarker *string
+	Prefix              *string
+	VersionIdMarker     *string
+	Versions            []types.ObjectVersion `xml:"Version"`
+}
+
+type GetBucketVersioningOutput struct {
+	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
+	MFADelete *types.MFADeleteStatus
+	Status    *types.BucketVersioningStatus
+}
+
+type PutObjectRetentionInput struct {
+	XMLName         xml.Name `xml:"Retention"`
+	Mode            types.ObjectLockRetentionMode
+	RetainUntilDate AmzDate
+}
+
+type PutObjectInput struct {
+	ContentLength             *int64
+	ObjectLockRetainUntilDate *time.Time
+
+	Bucket                  *string
+	Key                     *string
+	ContentType             *string
+	ContentEncoding         *string
+	ContentDisposition      *string
+	ContentLanguage         *string
+	CacheControl            *string
+	Expires                 *string
+	Tagging                 *string
+	ChecksumCRC32           *string
+	ChecksumCRC32C          *string
+	ChecksumSHA1            *string
+	ChecksumSHA256          *string
+	ChecksumCRC64NVME       *string
+	ContentMD5              *string
+	ExpectedBucketOwner     *string
+	GrantFullControl        *string
+	GrantRead               *string
+	GrantReadACP            *string
+	GrantWriteACP           *string
+	IfMatch                 *string
+	IfNoneMatch             *string
+	SSECustomerAlgorithm    *string
+	SSECustomerKey          *string
+	SSECustomerKeyMD5       *string
+	SSEKMSEncryptionContext *string
+	SSEKMSKeyId             *string
+	WebsiteRedirectLocation *string
+
+	ObjectLockMode            types.ObjectLockMode
+	ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	ChecksumAlgorithm         types.ChecksumAlgorithm
+
+	Metadata map[string]string
+	Body     io.Reader
+}
+
+type CreateMultipartUploadInput struct {
+	Bucket                    *string
+	Key                       *string
+	ExpectedBucketOwner       *string
+	CacheControl              *string
+	ContentDisposition        *string
+	ContentEncoding           *string
+	ContentLanguage           *string
+	ContentType               *string
+	Expires                   *string
+	SSECustomerAlgorithm      *string
+	SSECustomerKey            *string
+	SSECustomerKeyMD5         *string
+	SSEKMSEncryptionContext   *string
+	SSEKMSKeyId               *string
+	GrantFullControl          *string
+	GrantRead                 *string
+	GrantReadACP              *string
+	GrantWriteACP             *string
+	Tagging                   *string
+	WebsiteRedirectLocation   *string
+	BucketKeyEnabled          *bool
+	ObjectLockRetainUntilDate *time.Time
+	Metadata                  map[string]string
+
+	ACL                       types.ObjectCannedACL
+	ChecksumAlgorithm         types.ChecksumAlgorithm
+	ChecksumType              types.ChecksumType
+	ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	ObjectLockMode            types.ObjectLockMode
+	RequestPayer              types.RequestPayer
+	ServerSideEncryption      types.ServerSideEncryption
+	StorageClass              types.StorageClass
+}
+
+type CopyObjectInput struct {
+	Metadata                       map[string]string
+	Bucket                         *string
+	CopySource                     *string
+	Key                            *string
+	CacheControl                   *string
+	ContentDisposition             *string
+	ContentEncoding                *string
+	ContentLanguage                *string
+	ContentType                    *string
+	CopySourceIfMatch              *string
+	CopySourceIfNoneMatch          *string
+	CopySourceSSECustomerAlgorithm *string
+	CopySourceSSECustomerKey       *string
+	CopySourceSSECustomerKeyMD5    *string
+	ExpectedBucketOwner            *string
+	ExpectedSourceBucketOwner      *string
+	Expires                        *string
+	GrantFullControl               *string
+	GrantRead                      *string
+	GrantReadACP                   *string
+	GrantWriteACP                  *string
+	SSECustomerAlgorithm           *string
+	SSECustomerKey                 *string
+	SSECustomerKeyMD5              *string
+	SSEKMSEncryptionContext        *string
+	SSEKMSKeyId                    *string
+	Tagging                        *string
+	WebsiteRedirectLocation        *string
+
+	CopySourceIfModifiedSince   *time.Time
+	CopySourceIfUnmodifiedSince *time.Time
+	ObjectLockRetainUntilDate   *time.Time
+
+	BucketKeyEnabled *bool
+
+	ACL                       types.ObjectCannedACL
+	ChecksumAlgorithm         types.ChecksumAlgorithm
+	MetadataDirective         types.MetadataDirective
+	ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	ObjectLockMode            types.ObjectLockMode
+	RequestPayer              types.RequestPayer
+	ServerSideEncryption      types.ServerSideEncryption
+	StorageClass              types.StorageClass
+	TaggingDirective          types.TaggingDirective
+}
+
+type AmzDate struct {
+	time.Time
+}
+
+// Parses the date from xml string and validates for predefined date formats
+func (d *AmzDate) UnmarshalXML(e *xml.Decoder, startElement xml.StartElement) error {
+	var dateStr string
+	err := e.DecodeElement(&dateStr, &startElement)
+	if err != nil {
+		return err
+	}
+
+	retDate, err := d.ISO8601Parse(dateStr)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	*d = AmzDate{retDate}
+	return nil
+}
+
+// Encodes expiration date if it is non-zero
+// Encodes empty string if it's zero
+func (d AmzDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if d.IsZero() {
+		return nil
+	}
+	return e.EncodeElement(d.UTC().Format(iso8601TimeFormat), startElement)
+}
+
+// Parses ISO8601 date string to time.Time by
+// validating different time layouts
+func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
+	for _, layout := range []string{
+		iso8601TimeFormat,
+		iso8601TimeFormatExtended,
+		iso8601TimeFormatWithTZ,
+		time.RFC3339,
+	} {
+		t, err = time.Parse(layout, date)
+		if err == nil {
+			return t, nil
+		}
+	}
+
+	return t, err
+}
+
+// Admin api response types
+type ListBucketsResult struct {
+	Buckets []Bucket
+}
+
+type Checksum struct {
+	Algorithm types.ChecksumAlgorithm
+	Type      types.ChecksumType
+
+	CRC32     *string
+	CRC32C    *string
+	SHA1      *string
+	SHA256    *string
+	CRC64NVME *string
+}

tests/.env.default

@@ -17,4 +17,16 @@ GOCOVERDIR=$PWD/cover
 USERS_FOLDER=$PWD/iam
 #TEST_LOG_FILE=test.log
 #VERSITY_LOG_FILE=versity.log
-IAM_TYPE=folder
+IAM_TYPE=folder
+DIRECT=false
+#DIRECT_DISPLAY_NAME=
+#COVERAGE_DB=coverage.sql
+USERNAME_ONE=ABCDEFG
+PASSWORD_ONE=HIJKLMN
+USERNAME_TWO=HIJKLMN
+PASSWORD_TWO=OPQRSTU
+TEST_FILE_FOLDER=$PWD/versity-gwtest-files
+REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning
+COMMAND_LOG=command.log
+TIME_LOG=time.log

tests/.env.docker.default

@@ -0,0 +1,29 @@
+AWS_PROFILE=versity
+AWS_ENDPOINT_URL=https://127.0.0.1:7070
+VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
+BACKEND=posix
+LOCAL_FOLDER=/tmp/gw
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+CERT=$PWD/cert-docker.pem
+KEY=$PWD/versitygw-docker.pem
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets
+MC_ALIAS=versity
+LOG_LEVEL=2
+USERS_FOLDER=$PWD/iam
+#TEST_LOG_FILE=test.log
+#VERSITY_LOG_FILE=versity.log
+IAM_TYPE=folder
+DIRECT=false
+#DIRECT_DISPLAY_NAME=
+#COVERAGE_DB=coverage.sql
+USERNAME_ONE=ABCDEFG
+PASSWORD_ONE=HIJKLMN
+USERNAME_TWO=HIJKLMN
+PASSWORD_TWO=OPQRSTU
+TEST_FILE_FOLDER=$PWD/versity-gwtest-files
+RECREATE_BUCKETS=true
+REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning

tests/.secrets.default

@@ -0,0 +1,5 @@
+# change to your account attributes
+AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+AWS_REGION=us-east-1
+AWS_PROFILE=versity

tests/Dockerfile_direct

@@ -0,0 +1,73 @@
+FROM ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG SECRETS_FILE=tests/.secrets.direct
+ARG CONFIG_FILE=tests/.env.direct
+ARG AWS_CLI=awscli-exe-linux-aarch64.zip
+ARG MC_FOLDER=linux-arm64
+
+ENV TZ=Etc/UTC
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    git \
+    wget \
+    curl \
+    unzip \
+    tzdata \
+    s3cmd \
+    jq \
+    bc \
+    libxml2-utils \
+    ca-certificates && \
+    update-ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /tmp
+
+# Install AWS cli
+RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+
+# Install mc
+RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
+  --create-dirs \
+  -o /usr/local/minio-binaries/mc && \
+chmod -R 755 /usr/local/minio-binaries
+ENV PATH=/usr/local/minio-binaries:${PATH}
+
+# Create tester user
+RUN groupadd -r tester && useradd -r -g tester tester
+RUN mkdir /home/tester && chown tester:tester /home/tester
+ENV HOME=/home/tester
+
+# install bats
+RUN git clone https://github.com/bats-core/bats-core.git && \
+    cd bats-core && \
+    ./install.sh /home/tester
+
+USER tester
+RUN mkdir -p /home/tester/tests
+COPY --chown=tester:tester . /home/tester
+
+# add bats support libraries
+RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
+RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
+
+WORKDIR /home/tester
+
+RUN . $SECRETS_FILE && \
+    export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
+    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
+    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
+    aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
+
+RUN mkdir /tmp/gw
+
+RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen_bits:2048 && \
+    openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
+        -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+ENV WORKSPACE=.
+ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
+
+CMD ["tests/run_all.sh"]

tests/Dockerfile_test_bats

@@ -1,8 +1,12 @@
-FROM --platform=linux/arm64 ubuntu:latest
+FROM ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
 ARG SECRETS_FILE=tests/.secrets
 ARG CONFIG_FILE=tests/.env.docker
+ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz
+# see https://github.com/versity/versitygw/issues/1034
+ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip
+ARG MC_FOLDER=linux-arm64
 
 ENV TZ=Etc/UTC
 RUN apt-get update && \
@@ -16,6 +20,7 @@ RUN apt-get update && \
     s3cmd \
     jq \
     bc \
+    libxml2-utils \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*
@@ -24,20 +29,20 @@ RUN apt-get update && \
 WORKDIR /tmp
 
 # Install AWS cli
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
 
 # Install mc
-RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
+RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
   --create-dirs \
   -o /usr/local/minio-binaries/mc && \
 chmod -R 755 /usr/local/minio-binaries
-ENV PATH="/usr/local/minio-binaries":${PATH}
+ENV PATH=/usr/local/minio-binaries:${PATH}
 
 # Download Go 1.21 (adjust the version and platform as needed)
-RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
+RUN wget https://golang.org/dl/${GO_LIBRARY}
 
 # Extract the downloaded archive
-RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
+RUN tar -xvf $GO_LIBRARY -C /usr/local
 
 # Set Go environment variables
 ENV PATH="/usr/local/go/bin:${PATH}"
@@ -60,6 +65,10 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
 USER tester
 COPY --chown=tester:tester . /home/tester
 
+# add bats support libraries
+RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
+RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
+
 WORKDIR /home/tester
 RUN make
 
@@ -77,5 +86,7 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
 
 ENV WORKSPACE=.
 ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
+#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
 
-CMD ["tests/run_all.sh"]
+ENTRYPOINT ["tests/run.sh"]
+CMD ["s3api,s3,s3cmd,mc,rest"]

tests/README.md

@@ -1,5 +1,19 @@
 # Command-Line Tests
 
+## Table of Contents
+
+[Instructions - Running Locally](#instructions---running-locally)<br>
+[* Posix Backend](#posix-backend)<br>
+[* Static Bucket Mode](#static-bucket-mode)<br>
+[* S3 Backend](#s3-backend)<br>
+[* Direct Mode](#direct-mode)<br>
+[Instructions - Running With Docker](#instructions---running-with-docker)<br>
+[Instructions - Running With Docker-Compose](#instructions---running-with-docker-compose)<br>
+[Environment Parameters](#environment-parameters)<br>
+[* Secret](#secret)<br>
+[* Non-Secret](#non-secret)<br>
+[REST Scripts](#rest-scripts)<br>
+
 ## Instructions - Running Locally
 
 ### Posix Backend
@@ -9,10 +23,11 @@
    * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
    * **s3cmd**:  Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
    * **mc**:  Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
-3. Install BATS.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
-4. If running on Mac OS, install **jq** with the command `brew install jq`.
-4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
-5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
+3. Install **BATS**.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
+4. Install **bats-support** and **bats-assert**.  This can be done by saving the root folder of each repo (https://github.com/bats-core/bats-support and https://github.com/ztombol/bats-assert) in the `tests` folder.
+5. If running on Mac OS, install **jq** with the command `brew install jq`.
+6. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, and `AWS_PROFILE` values to the file.
+7. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
 ```
     export AWS_PROFILE=versity-test
     export AWS_ACCESS_KEY_ID=<your account ID>
@@ -22,18 +37,18 @@
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE
     aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
 ```
-6. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
-7. If using SSL, create a local private key and certificate, such as with the commands below.  Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively.
+8. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
+9. If using SSL, create a local private key and certificate, such as with the commands below.  Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively.
 ```
     openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
     openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
 ```
-8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
-9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
+11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
 ### Static Bucket Mode
 
-To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`.  Two utility functions are included, if needed, to create, and delete buckets for this:  `tests/setup_static.sh` and `tests/remove_static.sh`.
+To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`.  Two utility functions are included, if needed, to create, and delete buckets for this:  `tests/setup_static.sh` and `tests/remove_static.sh`.  Note that this creates a bucket with object lock enabled, and some tests may fail if the bucket being tested doesn't have object lock enabled.
 
 ### S3 Backend
 
@@ -57,12 +72,14 @@ To communicate directly with s3, in order to compare the gateway results to dire
 
 ## Instructions - Running With Docker
 
-1.  Create a `.secrets` file in the `tests` folder, and add the `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and the `AWS_PROFILE` fields, as well as the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
-2.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
+1.  Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
+2.  By default, the dockerfile uses the **arm** architecture (usually modern Mac).  If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="<param>=<amd library or folder>"`  Also, you can determine which is used by your OS with `uname -a`.
+3.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
+4.  To run the entire suite, run `docker run -it <image name>`.  To run an individual suite, pass in the name of the suite as defined in `tests/run.sh` (e.g. REST tests -> `docker run -it <image name> rest`).  Also, multiple specific suites can be run, if separated by comma.
 
 ## Instructions - Running with docker-compose
 
-A file named `docker-compose-bats.yml` is provided in the root folder.  Four configurations are provided:
+A file named `docker-compose-bats.yml` is provided in the root folder.  A few configurations are provided, and you can also create your own provided you have a secrets and config file:
 * insecure (without certificates), with creation/removal of buckets
 * secure, posix backend, with static buckets
 * secure, posix backend, with creation/removal of buckets
@@ -76,3 +93,87 @@ To run in insecure mode, comment out the `CERT` and `KEY` parameters in the `.en
 To use static buckets set the `RECREATE_BUCKETS` value to `false`.
 
 For the s3 backend, see the **S3 Backend** instructions above.
+
+If using AMD rather than ARM architecture, add the corresponding **args** values matching those in the Dockerfile for **amd** libraries.
+
+A single instance can be run with `docker-compose -f docker-compose-bats.yml up <service name>`
+
+## Environment Parameters
+
+### Secret
+
+**AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**:  identical to the same parameters in **s3**.
+
+**AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the main user (owner)
+
+**ACL_AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the user to test ACL changes and access by non-owners
+
+**ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**:  for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account.
+
+### Non-Secret
+
+**VERSITY_EXE**:  location of the versity executable relative to test folder.
+
+**RUN_VERSITYGW**:  whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
+
+**BACKEND**:  the storage backend type for the gateway, e.g. **posix** or **s3**.
+
+**LOCAL_FOLDER**:  if running with a **posix** backend, the backend storage folder.
+
+**BUCKET_ONE_NAME**, **BUCKET_TWO_NAME**:  test bucket names.
+
+**RECREATE_BUCKETS**:  whether to delete buckets between tests.  If set to false, the bucket will be restored to an original state for the purpose of ensuring consistent tests, but not deleted.
+
+**CERT**, **KEY**:  certificate and key locations if using SSL.
+
+**S3CMD_CONFIG**:  location of **s3cmd** config file if running **s3cmd** tests.
+
+**SECRETS_FILE**:  file where sensitive values, such as **AWS_SECRET_ACCESS_KEY**, should be stored.
+
+**MC_ALIAS**:  Minio MC alias if running MC tests.
+
+**LOG_LEVEL**:  level for test logger (1 - only critical, 2 - errors, 3 - warnings, 4 - info, 5 - debug info, 6 - tracing)
+
+**GOCOVERDIR**:  folder to put golang coverage info in, if checking coverage info.
+
+**USERS_FOLDER**:  folder to use if storing IAM data in a folder.
+
+**IAM_TYPE**:  how to store IAM data (**s3** or **folder**).
+
+**TEST_LOG_FILE**:  log file location for these bats tests.
+
+**VERSITY_LOG_FILE**:  log file for versity application as it is tested by bats tests.
+
+**DIRECT**:  if **true**, bypass versitygw and run directly against s3 (for comparison and validity-checking purposes).
+
+**DIRECT_DISPLAY_NAME**:  username if **DIRECT** is set to **true**.
+
+**COVERAGE_DB**:  database to store client command coverage info and usage counts, if using.
+
+**USERNAME_ONE**, **PASSWORD_ONE**, **USERNAME_TWO**, **PASSWORD_TWO**:  credentials for users created and tested for non-root user **versitygw** operations.
+
+**TEST_FILE_FOLDER**:  where to put temporary test files.
+
+**REMOVE_TEST_FILE_FOLDER**:  whether to delete the test file folder between tests, should be set to **true** unless checking the files after a single test, or not yet sure that the test folder is in a safe location to avoid deleting other files.
+
+**VERSIONING_DIR**:  where to put gateway file versioning info.
+
+**COMMAND_LOG**:  where to store list of client commands, which if using will be reported during test failures.
+
+**TIME_LOG**:  optional log to show duration of individual tests
+
+**DIRECT_S3_ROOT_ACCOUNT_NAME**:  for direct mode, S3 username
+
+**DELETE_BUCKETS_AFTER_TEST**:  whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked
+
+## REST Scripts
+
+REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder.  To call a script, the following parameters are needed:
+* **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**, etc.
+* **AWS_ENDPOINT_URL** (default:  `https://localhost:7070`)
+* **OUTPUT_FILE**:  file where the command's response data is written
+* Any other parameters specified at the top of the script file, such as payloads and variables.  Sometimes, defaults are included.
+
+Upon success, the script will return a response code, and write the data to the **OUTPUT_FILE** location.
+
+Example:  `AWS_ACCESS_KEY_ID={id} AWS_SECRET_ACCESS_KEY={key} AWS_ENDPOINT_URL=https://s3.amazonaws.com OUTPUT_FILE=./output_file.xml ./tests/rest_scripts/list_buckets.sh`

tests/commands/abort_multipart_upload.sh

@@ -1,15 +1,42 @@
 #!/usr/bin/env bash
 
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 abort_multipart_upload() {
+  record_command "abort-multipart-upload" "client:s3api"
   if [ $# -ne 3 ]; then
-    echo "command to run abort requires bucket, key, upload ID"
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID"
     return 1
   fi
-
-  error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3") || local aborted=$?
-  if [[ $aborted -ne 0 ]]; then
-    echo "Error aborting upload: $error"
+  if ! error=$(send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $error"
     return 1
   fi
   return 0
-}
+}
+
+abort_multipart_upload_with_user() {
+  if [ $# -ne 5 ]; then
+    log 2 "'abort multipart upload' command requires bucket, key, upload ID, username, password"
+    return 1
+  fi
+  record_command "abort-multipart-upload" "client:s3api"
+  if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then
+    log 2 "Error aborting upload: $abort_multipart_upload_error"
+    export abort_multipart_upload_error
+    return 1
+  fi
+  return 0
+}