feat: Fixes all struct fields alignment issues, Adds betteralign check in the pipeline, to check the structs fields alignment

Test/GitHub actions speedup

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,27 +1,23 @@
 ---
-name: Bug report
+name: Bug Report
 about: Create a report to help us improve
-title: ''
+title: '[Bug] - <Short Description>'
 labels: bug
 assignees: ''
 
 ---
 
 **Describe the bug**
-A clear and concise description of what the bug is.
+<!-- A clear and concise description of what the bug is. -->
 
 **To Reproduce**
-Steps to reproduce the behavior.
+<!-- Steps to reproduce the behavior. -->
 
 **Expected behavior**
-A clear and concise description of what you expected to happen.
+<!-- A clear and concise description of what you expected to happen. -->
 
 **Server Version**
- output of
-```
-./versitygw -version
-uname -a
-```
+<!-- output of: './versitygw -version && uname -a'  -->
 
 **Additional context**
-Describe s3 client and version if applicable.
+<!-- Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,14 +1,14 @@
 ---
-name: Feature request
+name: Feature Request
 about: Suggest an idea for this project
-title: ''
+title: '[Feature] - <Short Description>'
 labels: enhancement
 assignees: ''
 
 ---
 
 **Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+<!-- A clear and concise description of what you want to happen. -->
 
 **Additional context**
-Add any other context or screenshots about the feature request here.
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/test_case.md

@@ -0,0 +1,33 @@
+---
+name: Test Case Request
+about: Request new test cases or additional test coverage
+title: '[Test Case] - <Short Description>'
+labels: 'testcase'
+assignees: ''
+
+---
+
+## Description
+<!-- Please provide a detailed description of the test case or test coverage request. -->
+
+## Purpose
+<!-- Explain why this test case is important and what it aims to achieve. -->
+
+## Scope
+<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
+
+## Acceptance Criteria
+<!-- List the criteria that must be met for the test case to be considered complete. -->
+
+1. 
+2. 
+3. 
+
+## Additional Context
+<!-- Add any other context or screenshots about the feature request here. -->
+
+## Resources
+<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
+
+
+**Thank you for contributing to our project!**

.github/workflows/azurite.yml

@@ -0,0 +1,37 @@
+name: azurite functional tests
+
+on: pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+              
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Set up Docker Compose
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
+
+      - name: Wait for Azurite to be ready
+        run: sleep 40
+
+      - name: Get Dependencies
+        run: |
+          go mod download
+  
+      - name: Build and Run
+        run: |
+          make
+          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
+  
+      - name: Shut down services
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/betteralign.yml

@@ -0,0 +1,23 @@
+name: betteralign
+on: pull_request
+jobs:
+  build:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "stable"
+        id: go
+
+      - name: Install betteralign
+        run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest
+
+      - name: Run betteralign
+        run: betteralign -test_files ./...

.github/workflows/docker-bats.yml

@@ -0,0 +1,28 @@
+name: docker bats tests
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        run: |
+          cp tests/.env.docker.default tests/.env.docker
+          cp tests/.secrets.default tests/.secrets
+          docker build \
+            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
+            --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain \
+            -f tests/Dockerfile_test_bats \
+            -t bats_test .
+
+      - name: Run Docker Container
+        run: |
+          docker compose -f tests/docker-compose-bats.yml --project-directory . \
+          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/docker.yml

@@ -15,6 +15,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -43,6 +50,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
           build-args: |
             VERSION=${{ github.event.release.tag_name }}
             TIME=${{ github.event.release.published_at }}

.github/workflows/functional.yml

@@ -1,7 +1,8 @@
 name: functional tests
-on: pull_request
-jobs:
 
+on: pull_request
+
+jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
@@ -18,7 +19,7 @@ jobs:
 
     - name: Get Dependencies
       run: |
-        go get -v -t -d ./...
+        go mod download
 
     - name: Build and Run
       run: |

.github/workflows/shellcheck.yml

@@ -0,0 +1,16 @@
+name: shellcheck
+on: pull_request
+jobs:
+
+  build:
+    name: Run shellcheck
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Run checks
+      run: |
+        shellcheck --version
+        shellcheck -e SC1091 tests/*.sh tests/*/*.sh

.github/workflows/system.yml

@@ -4,16 +4,104 @@ jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - set: "mc, posix, non-file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "mc-non-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "mc, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "mc-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "REST, posix, non-static, all, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3, posix, non-file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-non-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, non-static, s3 IAM"
+            IAM_TYPE: s3
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, multipart, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-multipart"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, object, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-object"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, s3, multipart|object, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "s3"
+          - set: "s3api, s3, policy|user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy,s3api-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "s3"
+          - set: "s3cmd, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, non-user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-non-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
 
-      - name: Install ShellCheck
-        run: sudo apt-get install shellcheck
-
-      - name: Run ShellCheck
-        run: shellcheck -S warning ./tests/*.sh
-
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -28,6 +116,8 @@ jobs:
         run: |
           git clone https://github.com/bats-core/bats-core.git
           cd bats-core && ./install.sh $HOME
+          git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
+          git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
 
       - name: Install s3cmd
         run: |
@@ -38,35 +128,63 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Build and run, posix backend
+      - name: Install xmllint (for rest)
+        run: |
+          sudo apt-get install libxml2-utils
+
+      - name: Build and run
+        env:
+          IAM_TYPE: ${{ matrix.IAM_TYPE }}
+          RUN_SET: ${{ matrix.RUN_SET }}
+          AWS_PROFILE: versity
+          VERSITY_EXE: ${{ github.workspace }}/versitygw
+          RUN_VERSITYGW: true
+          BACKEND: ${{ matrix.BACKEND }}
+          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
+          CERT: ${{ github.workspace }}/cert.pem
+          KEY: ${{ github.workspace }}/versitygw.pem
+          LOCAL_FOLDER: /tmp/gw
+          BUCKET_ONE_NAME: versity-gwtest-bucket-one
+          BUCKET_TWO_NAME: versity-gwtest-bucket-two
+          USERS_FOLDER: /tmp/iam
+          USERS_BUCKET: versity-gwtest-iam
+          AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          PORT: 7070
+          S3CMD_CONFIG: tests/s3cfg.local.default
+          MC_ALIAS: versity
+          LOG_LEVEL: 4
+          GOCOVERDIR: ${{ github.workspace }}/cover
+          USERNAME_ONE: ABCDEFG
+          PASSWORD_ONE: 1234567
+          USERNAME_TWO: HIJKLMN
+          PASSWORD_TWO: 8901234
+          TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
+          REMOVE_TEST_FILE_FOLDER: true
+          VERSIONING_DIR: ${{ github.workspace }}/versioning
+          COMMAND_LOG: command.log
+          TIME_LOG: time.log
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
+          export AWS_ACCESS_KEY_ID_TWO=user
+          export AWS_SECRET_ACCESS_KEY_TWO=pass
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity
-          mkdir /tmp/gw
+          mkdir $LOCAL_FOLDER
           export WORKSPACE=$GITHUB_WORKSPACE
-          openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
-          openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir cover
-          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
+          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          mkdir $GOCOVERDIR $USERS_FOLDER
+          if [[ $RECREATE_BUCKETS == "false" ]]; then
+            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
+          fi
+          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
-      #- name: Build and run, s3 backend
-      #  run: |
-      #    make testbin
-      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export AWS_REGION=us-east-1
-      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
-      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
-      #    aws configure set aws_region $AWS_REGION --profile versity_s3
-      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export WORKSPACE=$GITHUB_WORKSPACE
-      #    VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+      - name: Time report
+        run: cat ${{ github.workspace }}/time.log
 
       - name: Coverage report
         run: |

.gitignore

@@ -45,7 +45,25 @@ tests/.secrets*
 
 # IAM users files often created in testing
 users.json
+users.json.backup
 
 # env files for testing
-.env*
-!.env.default
+**/.env*
+**/!.env.default
+
+# s3cmd config files (testing)
+tests/s3cfg.local*
+tests/!s3cfg.local.default
+
+# keys
+*.pem
+
+# patches
+*.patch
+
+# grafana's local database (kept on filesystem for survival between instantiations)
+metrics-exploration/grafana_data/**
+
+# bats tools
+/tests/bats-assert
+/tests/bats-support

Dockerfile

@@ -32,4 +32,4 @@ RUN mkdir -p $SETUP_DIR
 
 COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
 
-ENTRYPOINT [ "/app/versitygw" ]
+ENTRYPOINT [ "/app/versitygw" ]

Makefile

@@ -18,6 +18,10 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 
+# docker-compose
+DCCMD=docker-compose
+DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
+
 BIN=versitygw
 
 VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -71,19 +75,19 @@ dist:
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
-	docker compose --env-file .env.dev up posix
+	$(DOCKERCOMPOSE) up posix
 
 # Creates and runs S3 gateway proxy instance in a docker container
 .PHONY: up-proxy
 up-proxy:
-	docker compose --env-file .env.dev up proxy
+	$(DOCKERCOMPOSE) up proxy
 
 # Creates and runs S3 gateway to azurite instance in a docker container
 .PHONY: up-azurite
 up-azurite:
-	docker compose --env-file .env.dev up azurite azuritegw
+	$(DOCKERCOMPOSE) up azurite azuritegw
 
 # Creates and runs both S3 gateway and proxy server instances in docker containers
 .PHONY: up-app
 up-app:
-	docker compose --env-file .env.dev up
+	$(DOCKERCOMPOSE) up

README.md

@@ -6,7 +6,7 @@
   <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
 </picture>
 
- [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
 
 ### Binary release builds
 Download [latest release](https://github.com/versity/versitygw/releases)
@@ -14,8 +14,15 @@ Download [latest release](https://github.com/versity/versitygw/releases)
  |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
  |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
  
+### Use Cases
+* Turn your local filesystem into an S3 server with a single command!
+* Proxy S3 requests to S3 storage
+* Simple to deploy S3 server with a single command
+* Protocol compatibility in `posix` allows common access to files via posix or S3
+* Simplified interface for adding new storage system support
+
 ### News
-* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
+Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
 
 ### Mailing List
 Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
@@ -28,12 +35,6 @@ Ask questions in the [community discussions](https://github.com/versity/versityg
 <br>
 Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
 
-### Use Cases
-* Share filesystem directory via S3 protocol
-* Proxy S3 requests to S3 storage
-* Simple to deploy S3 server with a single command
-* Protocol compatibility in `posix` allows common access to files via posix or S3 
-
 ### Overview
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
@@ -67,7 +68,7 @@ The command format is
 ```
 versitygw [global options] command [command options] [arguments...]
 ```
-The global options are specified before the backend type and the backend options are specified after.
+The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
 
 ***
 

auth/acl.go

@@ -17,6 +17,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 
@@ -27,7 +28,6 @@ import (
 )
 
 type ACL struct {
-	ACL      types.BucketCannedACL
 	Owner    string
 	Grantees []Grantee
 }
@@ -35,6 +35,7 @@ type ACL struct {
 type Grantee struct {
 	Permission types.Permission
 	Access     string
+	Type       types.Type
 }
 
 type GetBucketAclOutput struct {
@@ -42,12 +43,36 @@ type GetBucketAclOutput struct {
 	AccessControlList AccessControlList
 }
 
-type AccessControlList struct {
-	Grants []types.Grant `xml:"Grant"`
+type PutBucketAclInput struct {
+	Bucket              *string
+	AccessControlPolicy *AccessControlPolicy
+	GrantFullControl    *string
+	GrantRead           *string
+	GrantReadACP        *string
+	GrantWrite          *string
+	GrantWriteACP       *string
+	ACL                 types.BucketCannedACL
 }
+
 type AccessControlPolicy struct {
+	Owner             *types.Owner
 	AccessControlList AccessControlList `xml:"AccessControlList"`
-	Owner             types.Owner
+}
+
+type AccessControlList struct {
+	Grants []Grant `xml:"Grant"`
+}
+
+type Grant struct {
+	Grantee    *Grt
+	Permission types.Permission
+}
+
+type Grt struct {
+	XMLNS  string     `xml:"xmlns:xsi,attr"`
+	XMLXSI types.Type `xml:"xsi:type,attr"`
+	Type   types.Type `xml:"Type"`
+	ID     string     `xml:"ID"`
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -68,11 +93,19 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
-	grants := []types.Grant{}
+	grants := []Grant{}
 
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
-		grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
+		grants = append(grants, Grant{
+			Grantee: &Grt{
+				XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+				XMLXSI: elem.Type,
+				ID:     acs,
+				Type:   elem.Type,
+			},
+			Permission: elem.Permission,
+		})
 	}
 
 	return GetBucketAclOutput{
@@ -85,20 +118,43 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 	}, nil
 }
 
-func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
+func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
 	if input == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
-	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
-		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+
+	defaultGrantees := []Grantee{
+		{
+			Permission: types.PermissionFullControl,
+			Access:     acl.Owner,
+			Type:       types.TypeCanonicalUser,
+		},
 	}
 
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
-		acl.ACL = input.ACL
-		acl.Grantees = []Grantee{}
+		switch input.ACL {
+		case types.BucketCannedACLPublicRead:
+			defaultGrantees = append(defaultGrantees, Grantee{
+				Permission: types.PermissionRead,
+				Access:     "all-users",
+				Type:       types.TypeGroup,
+			})
+		case types.BucketCannedACLPublicReadWrite:
+			defaultGrantees = append(defaultGrantees, []Grantee{
+				{
+					Permission: types.PermissionRead,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+				{
+					Permission: types.PermissionWrite,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+			}...)
+		}
 	} else {
-		grantees := []Grantee{}
 		accs := []string{}
 
 		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -107,45 +163,71 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionFullControl,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantRead != nil && *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionRead,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionReadAcp,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantWrite != nil && *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionWrite,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
-					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: types.PermissionWriteAcp,
+						Type:       types.TypeCanonicalUser,
+					})
 				}
 			}
 
 			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
 		} else {
 			cache := make(map[string]bool)
-			for _, grt := range input.AccessControlPolicy.Grants {
-				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
+			for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
+				if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
 					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 				}
-				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
-				if _, ok := cache[*grt.Grantee.ID]; !ok {
-					cache[*grt.Grantee.ID] = true
-					accs = append(accs, *grt.Grantee.ID)
+
+				access := grt.Grantee.ID
+				defaultGrantees = append(defaultGrantees, Grantee{
+					Access:     access,
+					Permission: grt.Permission,
+					Type:       types.TypeCanonicalUser,
+				})
+				if _, ok := cache[access]; !ok {
+					cache[access] = true
+					accs = append(accs, access)
 				}
 			}
 		}
@@ -158,11 +240,10 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		if len(accList) > 0 {
 			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
 		}
-
-		acl.Grantees = grantees
-		acl.ACL = ""
 	}
 
+	acl.Grantees = defaultGrantees
+
 	result, err := json.Marshal(acl)
 	if err != nil {
 		return nil, err
@@ -206,34 +287,33 @@ func splitUnique(s, divider string) []string {
 }
 
 func verifyACL(acl ACL, access string, permission types.Permission) error {
-	if acl.ACL != "" {
-		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
+	grantee := Grantee{
+		Access:     access,
+		Permission: permission,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeFullCtrl := Grantee{
+		Access:     access,
+		Permission: types.PermissionFullControl,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeAllUsers := Grantee{
+		Access:     "all-users",
+		Permission: permission,
+		Type:       types.TypeGroup,
+	}
 
+	isFound := false
+
+	for _, grt := range acl.Grantees {
+		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
+			isFound = true
+			break
+		}
+	}
+
+	if isFound {
 		return nil
-	} else {
-		if len(acl.Grantees) == 0 {
-			return nil
-		}
-		grantee := Grantee{Access: access, Permission: permission}
-		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
-
-		isFound := false
-
-		for _, grt := range acl.Grantees {
-			if grt == grantee || grt == granteeFullCtrl {
-				isFound = true
-				break
-			}
-		}
-
-		if isFound {
-			return nil
-		}
 	}
 
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -272,39 +352,38 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 }
 
 type AccessOptions struct {
-	Acl           ACL
 	AclPermission types.Permission
-	IsRoot        bool
-	Acc           Account
 	Bucket        string
 	Object        string
 	Action        Action
+	Acl           ACL
+	Acc           Account
+	IsRoot        bool
+	Readonly      bool
 }
 
 func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	if opts.Readonly {
+		if opts.AclPermission == types.PermissionWrite || opts.AclPermission == types.PermissionWriteAcp {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+	}
 	if opts.IsRoot {
 		return nil
 	}
 	if opts.Acc.Role == RoleAdmin {
 		return nil
 	}
-	if opts.Acc.Access == opts.Acl.Owner {
-		return nil
+
+	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
 	}
 
-	policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
-	if err != nil {
-		return err
-	}
-
-	// If bucket policy is not set and the ACL is default, only the owner has access
-	if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
-		return err
-	}
 	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
 		return err
 	}

auth/bucket_policy.go

@@ -16,12 +16,22 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
+	"errors"
 	"net/http"
 
 	"github.com/versity/versitygw/s3err"
 )
 
+var (
+	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidResource = errors.New("Policy has invalid resource")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidPrincipal = errors.New("Invalid principal in policy")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidAction = errors.New("Policy has invalid action")
+)
+
 type BucketPolicy struct {
 	Statement []BucketPolicyItem `json:"Statement"`
 }
@@ -53,10 +63,10 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri
 }
 
 type BucketPolicyItem struct {
-	Effect     BucketPolicyAccessType `json:"Effect"`
 	Principals Principals             `json:"Principal"`
 	Actions    Actions                `json:"Action"`
 	Resources  Resources              `json:"Resource"`
+	Effect     BucketPolicyAccessType `json:"Effect"`
 }
 
 func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
@@ -75,11 +85,14 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 
 	for action := range bpi.Actions {
 		isObjectAction := action.IsObjectAction()
-		if isObjectAction && !containsObjectAction {
-			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		if isObjectAction == nil {
+			break
 		}
-		if !isObjectAction && !containsBucketAction {
-			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		if *isObjectAction && !containsObjectAction {
+			return errResourceMismatch
+		}
+		if !*isObjectAction && !containsBucketAction {
+			return errResourceMismatch
 		}
 	}
 
@@ -108,6 +121,11 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 		return getMalformedPolicyError(err)
 	}
 
+	if len(policy.Statement) == 0 {
+		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
+	}
+
 	if err := policy.Validate(bucket, iam); err != nil {
 		return getMalformedPolicyError(err)
 	}
@@ -115,12 +133,7 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 	return nil
 }
 
-func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
-	// If bucket policy is not set
-	if len(policy) == 0 {
-		return nil
-	}
-
+func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
 	var bucketPolicy BucketPolicy
 	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
 		return err
@@ -131,7 +144,6 @@ func verifyBucketPolicy(policy []byte, access, bucket, object string, action Act
 		resource += "/" + object
 	}
 
-	fmt.Println(access, action, resource)
 	if !bucketPolicy.isAllowed(access, action, resource) {
 		return s3err.GetAPIError(s3err.ErrAccessDenied)
 	}

auth/bucket_policy_actions.go

@@ -16,92 +16,116 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
 type Action string
 
 const (
-	GetBucketAclAction               Action = "s3:GetBucketAcl"
-	CreateBucketAction               Action = "s3:CreateBucket"
-	PutBucketAclAction               Action = "s3:PutBucketAcl"
-	DeleteBucketAction               Action = "s3:DeleteBucket"
-	PutBucketVersioningAction        Action = "s3:PutBucketVersioning"
-	GetBucketVersioningAction        Action = "s3:GetBucketVersioning"
-	PutBucketPolicyAction            Action = "s3:PutBucketPolicy"
-	GetBucketPolicyAction            Action = "s3:GetBucketPolicy"
-	DeleteBucketPolicyAction         Action = "s3:DeleteBucketPolicy"
-	AbortMultipartUploadAction       Action = "s3:AbortMultipartUpload"
-	ListMultipartUploadPartsAction   Action = "s3:ListMultipartUploadParts"
-	ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
-	PutObjectAction                  Action = "s3:PutObject"
-	GetObjectAction                  Action = "s3:GetObject"
-	DeleteObjectAction               Action = "s3:DeleteObject"
-	GetObjectAclAction               Action = "s3:GetObjectAcl"
-	GetObjectAttributesAction        Action = "s3:GetObjectAttributes"
-	PutObjectAclAction               Action = "s3:PutObjectAcl"
-	RestoreObjectAction              Action = "s3:RestoreObject"
-	GetBucketTaggingAction           Action = "s3:GetBucketTagging"
-	PutBucketTaggingAction           Action = "s3:PutBucketTagging"
-	GetObjectTaggingAction           Action = "s3:GetObjectTagging"
-	PutObjectTaggingAction           Action = "s3:PutObjectTagging"
-	DeleteObjectTaggingAction        Action = "s3:DeleteObjectTagging"
-	ListBucketVersionsAction         Action = "s3:ListBucketVersions"
-	ListBucketAction                 Action = "s3:ListBucket"
-	AllActions                       Action = "s3:*"
+	GetBucketAclAction                     Action = "s3:GetBucketAcl"
+	CreateBucketAction                     Action = "s3:CreateBucket"
+	PutBucketAclAction                     Action = "s3:PutBucketAcl"
+	DeleteBucketAction                     Action = "s3:DeleteBucket"
+	PutBucketVersioningAction              Action = "s3:PutBucketVersioning"
+	GetBucketVersioningAction              Action = "s3:GetBucketVersioning"
+	PutBucketPolicyAction                  Action = "s3:PutBucketPolicy"
+	GetBucketPolicyAction                  Action = "s3:GetBucketPolicy"
+	DeleteBucketPolicyAction               Action = "s3:DeleteBucketPolicy"
+	AbortMultipartUploadAction             Action = "s3:AbortMultipartUpload"
+	ListMultipartUploadPartsAction         Action = "s3:ListMultipartUploadParts"
+	ListBucketMultipartUploadsAction       Action = "s3:ListBucketMultipartUploads"
+	PutObjectAction                        Action = "s3:PutObject"
+	GetObjectAction                        Action = "s3:GetObject"
+	GetObjectVersionAction                 Action = "s3:GetObjectVersion"
+	DeleteObjectAction                     Action = "s3:DeleteObject"
+	GetObjectAclAction                     Action = "s3:GetObjectAcl"
+	GetObjectAttributesAction              Action = "s3:GetObjectAttributes"
+	PutObjectAclAction                     Action = "s3:PutObjectAcl"
+	RestoreObjectAction                    Action = "s3:RestoreObject"
+	GetBucketTaggingAction                 Action = "s3:GetBucketTagging"
+	PutBucketTaggingAction                 Action = "s3:PutBucketTagging"
+	GetObjectTaggingAction                 Action = "s3:GetObjectTagging"
+	PutObjectTaggingAction                 Action = "s3:PutObjectTagging"
+	DeleteObjectTaggingAction              Action = "s3:DeleteObjectTagging"
+	ListBucketVersionsAction               Action = "s3:ListBucketVersions"
+	ListBucketAction                       Action = "s3:ListBucket"
+	GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
+	PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
+	GetObjectLegalHoldAction               Action = "s3:GetObjectLegalHold"
+	PutObjectLegalHoldAction               Action = "s3:PutObjectLegalHold"
+	GetObjectRetentionAction               Action = "s3:GetObjectRetention"
+	PutObjectRetentionAction               Action = "s3:PutObjectRetention"
+	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
+	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
+	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
+	AllActions                             Action = "s3:*"
 )
 
 var supportedActionList = map[Action]struct{}{
-	GetBucketAclAction:               {},
-	CreateBucketAction:               {},
-	PutBucketAclAction:               {},
-	DeleteBucketAction:               {},
-	PutBucketVersioningAction:        {},
-	GetBucketVersioningAction:        {},
-	PutBucketPolicyAction:            {},
-	GetBucketPolicyAction:            {},
-	DeleteBucketPolicyAction:         {},
-	AbortMultipartUploadAction:       {},
-	ListMultipartUploadPartsAction:   {},
-	ListBucketMultipartUploadsAction: {},
-	PutObjectAction:                  {},
-	GetObjectAction:                  {},
-	DeleteObjectAction:               {},
-	GetObjectAclAction:               {},
-	GetObjectAttributesAction:        {},
-	PutObjectAclAction:               {},
-	RestoreObjectAction:              {},
-	GetBucketTaggingAction:           {},
-	PutBucketTaggingAction:           {},
-	GetObjectTaggingAction:           {},
-	PutObjectTaggingAction:           {},
-	DeleteObjectTaggingAction:        {},
-	ListBucketVersionsAction:         {},
-	ListBucketAction:                 {},
-	AllActions:                       {},
+	GetBucketAclAction:                     {},
+	CreateBucketAction:                     {},
+	PutBucketAclAction:                     {},
+	DeleteBucketAction:                     {},
+	PutBucketVersioningAction:              {},
+	GetBucketVersioningAction:              {},
+	PutBucketPolicyAction:                  {},
+	GetBucketPolicyAction:                  {},
+	DeleteBucketPolicyAction:               {},
+	AbortMultipartUploadAction:             {},
+	ListMultipartUploadPartsAction:         {},
+	ListBucketMultipartUploadsAction:       {},
+	PutObjectAction:                        {},
+	GetObjectAction:                        {},
+	GetObjectVersionAction:                 {},
+	DeleteObjectAction:                     {},
+	GetObjectAclAction:                     {},
+	GetObjectAttributesAction:              {},
+	PutObjectAclAction:                     {},
+	RestoreObjectAction:                    {},
+	GetBucketTaggingAction:                 {},
+	PutBucketTaggingAction:                 {},
+	GetObjectTaggingAction:                 {},
+	PutObjectTaggingAction:                 {},
+	DeleteObjectTaggingAction:              {},
+	ListBucketVersionsAction:               {},
+	ListBucketAction:                       {},
+	PutBucketObjectLockConfigurationAction: {},
+	GetObjectLegalHoldAction:               {},
+	PutObjectLegalHoldAction:               {},
+	GetObjectRetentionAction:               {},
+	PutObjectRetentionAction:               {},
+	BypassGovernanceRetentionAction:        {},
+	PutBucketOwnershipControlsAction:       {},
+	GetBucketOwnershipControlsAction:       {},
+	AllActions:                             {},
 }
 
 var supportedObjectActionList = map[Action]struct{}{
-	AbortMultipartUploadAction:     {},
-	ListMultipartUploadPartsAction: {},
-	PutObjectAction:                {},
-	GetObjectAction:                {},
-	DeleteObjectAction:             {},
-	GetObjectAclAction:             {},
-	GetObjectAttributesAction:      {},
-	PutObjectAclAction:             {},
-	RestoreObjectAction:            {},
-	GetObjectTaggingAction:         {},
-	PutObjectTaggingAction:         {},
-	DeleteObjectTaggingAction:      {},
-	AllActions:                     {},
+	AbortMultipartUploadAction:      {},
+	ListMultipartUploadPartsAction:  {},
+	PutObjectAction:                 {},
+	GetObjectAction:                 {},
+	GetObjectVersionAction:          {},
+	DeleteObjectAction:              {},
+	GetObjectAclAction:              {},
+	GetObjectAttributesAction:       {},
+	PutObjectAclAction:              {},
+	RestoreObjectAction:             {},
+	GetObjectTaggingAction:          {},
+	PutObjectTaggingAction:          {},
+	DeleteObjectTaggingAction:       {},
+	GetObjectLegalHoldAction:        {},
+	PutObjectLegalHoldAction:        {},
+	GetObjectRetentionAction:        {},
+	PutObjectRetentionAction:        {},
+	BypassGovernanceRetentionAction: {},
+	AllActions:                      {},
 }
 
 // Validates Action: it should either wildcard match with supported actions list or be in it
 func (a Action) IsValid() error {
 	if !strings.HasPrefix(string(a), "s3:") {
-		return fmt.Errorf("invalid action: %v", a)
+		return errInvalidAction
 	}
 
 	if a == AllActions {
@@ -116,31 +140,39 @@ func (a Action) IsValid() error {
 			}
 		}
 
-		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+		return errInvalidAction
 	}
 
 	_, found := supportedActionList[a]
 	if !found {
-		return fmt.Errorf("unsupported action: %v", a)
+		return errInvalidAction
 	}
 	return nil
 }
 
+func getBoolPtr(bl bool) *bool {
+	return &bl
+}
+
 // Checks if the action is object action
-func (a Action) IsObjectAction() bool {
+// nil points to 's3:*'
+func (a Action) IsObjectAction() *bool {
+	if a == AllActions {
+		return nil
+	}
 	if a[len(a)-1] == '*' {
 		pattern := strings.TrimSuffix(string(a), "*")
 		for act := range supportedObjectActionList {
 			if strings.HasPrefix(string(act), pattern) {
-				return true
+				return getBoolPtr(true)
 			}
 		}
 
-		return false
+		return getBoolPtr(false)
 	}
 
 	_, found := supportedObjectActionList[a]
-	return found
+	return &found
 }
 
 func (a Action) WildCardMatch(act Action) bool {
@@ -159,7 +191,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("actions can't be empty")
+			return errInvalidAction
 		}
 		*a = make(Actions)
 		for _, s := range ss {
@@ -172,7 +204,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("actions can't be empty")
+				return errInvalidAction
 			}
 			*a = make(Actions)
 			err = a.Add(s)

auth/bucket_policy_effect.go

@@ -30,5 +30,6 @@ func (bpat BucketPolicyAccessType) Validate() error {
 		return nil
 	}
 
-	return fmt.Errorf("invalid effect: %v", bpat)
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	return fmt.Errorf("Invalid effect: %v", bpat)
 }

auth/bucket_policy_principals.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 )
 
 type Principals map[string]struct{}
@@ -28,23 +27,50 @@ func (p Principals) Add(key string) {
 // Override UnmarshalJSON method to decode both []string and string properties
 func (p *Principals) UnmarshalJSON(data []byte) error {
 	ss := []string{}
+	var s string
+	var k struct {
+		AWS string
+	}
+
 	var err error
+
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		for _, s := range ss {
 			p.Add(s)
 		}
+		return nil
+	} else if err = json.Unmarshal(data, &s); err == nil {
+		if s == "" {
+			return errInvalidPrincipal
+		}
+		*p = make(Principals)
+		p.Add(s)
+
+		return nil
+	} else if err = json.Unmarshal(data, &k); err == nil {
+		if k.AWS == "" {
+			return errInvalidPrincipal
+		}
+		*p = make(Principals)
+		p.Add(k.AWS)
+
+		return nil
 	} else {
-		var s string
-		if err = json.Unmarshal(data, &s); err == nil {
-			if s == "" {
-				return fmt.Errorf("principals can't be empty")
+		var sk struct {
+			AWS []string
+		}
+		if err = json.Unmarshal(data, &sk); err == nil {
+			if len(sk.AWS) == 0 {
+				return errInvalidPrincipal
 			}
 			*p = make(Principals)
-			p.Add(s)
+			for _, s := range sk.AWS {
+				p.Add(s)
+			}
 		}
 	}
 
@@ -71,7 +97,7 @@ func (p Principals) Validate(iam IAMService) error {
 		if len(p) == 1 {
 			return nil
 		}
-		return fmt.Errorf("principals should either contain * or user access keys")
+		return errInvalidPrincipal
 	}
 
 	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -79,7 +105,7 @@ func (p Principals) Validate(iam IAMService) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user accounts don't exist: %v", accs)
+		return errInvalidPrincipal
 	}
 
 	return nil

auth/bucket_policy_resources.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -30,7 +29,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("resources can't be empty")
+			return errInvalidResource
 		}
 		*r = make(Resources)
 		for _, s := range ss {
@@ -43,7 +42,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("resources can't be empty")
+				return errInvalidResource
 			}
 			*r = make(Resources)
 			err = r.Add(s)
@@ -60,12 +59,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 func (r Resources) Add(rc string) error {
 	ok, pattern := isValidResource(rc)
 	if !ok {
-		return fmt.Errorf("invalid resource: %v", rc)
-	}
-
-	_, found := r[pattern]
-	if found {
-		return fmt.Errorf("duplicate resource: %v", rc)
+		return errInvalidResource
 	}
 
 	r[pattern] = struct{}{}
@@ -99,7 +93,7 @@ func (r Resources) ContainsBucketPattern() bool {
 func (r Resources) Validate(bucket string) error {
 	for resource := range r {
 		if !strings.HasPrefix(resource, bucket) {
-			return fmt.Errorf("incorrect bucket name in %v", resource)
+			return errInvalidResource
 		}
 	}
 

auth/iam.go

@@ -28,14 +28,49 @@ const (
 	RoleUserPlus Role = "userplus"
 )
 
+func (r Role) IsValid() bool {
+	switch r {
+	case RoleAdmin:
+		return true
+	case RoleUser:
+		return true
+	case RoleUserPlus:
+		return true
+	default:
+		return false
+	}
+}
+
 // Account is a gateway IAM account
 type Account struct {
-	Access    string `json:"access"`
-	Secret    string `json:"secret"`
-	Role      Role   `json:"role"`
-	UserID    int    `json:"userID"`
-	GroupID   int    `json:"groupID"`
-	ProjectID int    `json:"projectID"`
+	Access  string `json:"access"`
+	Secret  string `json:"secret"`
+	Role    Role   `json:"role"`
+	UserID  int    `json:"userID"`
+	GroupID int    `json:"groupID"`
+}
+
+type ListUserAccountsResult struct {
+	Accounts []Account
+}
+
+// Mutable props, which could be changed when updating an IAM account
+type MutableProps struct {
+	Secret  *string `json:"secret"`
+	UserID  *int    `json:"userID"`
+	GroupID *int    `json:"groupID"`
+}
+
+func updateAcc(acc *Account, props MutableProps) {
+	if props.Secret != nil {
+		acc.Secret = *props.Secret
+	}
+	if props.GroupID != nil {
+		acc.GroupID = *props.GroupID
+	}
+	if props.UserID != nil {
+		acc.UserID = *props.UserID
+	}
 }
 
 // IAMService is the interface for all IAM service implementations
@@ -44,33 +79,51 @@ type Account struct {
 type IAMService interface {
 	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
+	UpdateUserAccount(access string, props MutableProps) error
 	DeleteUserAccount(access string) error
 	ListUserAccounts() ([]Account, error)
 	Shutdown() error
 }
 
-var ErrNoSuchUser = errors.New("user not found")
+var (
+	// ErrUserExists is returned when the user already exists
+	ErrUserExists = errors.New("user already exists")
+	// ErrNoSuchUser is returned when the user does not exist
+	ErrNoSuchUser = errors.New("user not found")
+)
 
 type Opts struct {
-	Dir                string
-	LDAPServerURL      string
-	LDAPBindDN         string
-	LDAPPassword       string
-	LDAPQueryBase      string
-	LDAPObjClasses     string
-	LDAPAccessAtr      string
-	LDAPSecretAtr      string
-	LDAPRoleAtr        string
-	S3Access           string
-	S3Secret           string
-	S3Region           string
-	S3Bucket           string
-	S3Endpoint         string
-	S3DisableSSlVerfiy bool
-	S3Debug            bool
-	CacheDisable       bool
-	CacheTTL           int
-	CachePrune         int
+	Dir                    string
+	LDAPServerURL          string
+	LDAPBindDN             string
+	LDAPPassword           string
+	LDAPQueryBase          string
+	LDAPObjClasses         string
+	LDAPAccessAtr          string
+	LDAPSecretAtr          string
+	LDAPRoleAtr            string
+	LDAPUserIdAtr          string
+	LDAPGroupIdAtr         string
+	VaultEndpointURL       string
+	VaultSecretStoragePath string
+	VaultMountPath         string
+	VaultRootToken         string
+	VaultRoleId            string
+	VaultRoleSecret        string
+	VaultServerCert        string
+	VaultClientCert        string
+	VaultClientCertKey     string
+	S3Access               string
+	S3Secret               string
+	S3Region               string
+	S3Bucket               string
+	S3Endpoint             string
+	RootAccount            Account
+	CacheTTL               int
+	CachePrune             int
+	S3DisableSSlVerfiy     bool
+	S3Debug                bool
+	CacheDisable           bool
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -79,18 +132,23 @@ func New(o *Opts) (IAMService, error) {
 
 	switch {
 	case o.Dir != "":
-		svc, err = NewInternal(o.Dir)
+		svc, err = NewInternal(o.RootAccount, o.Dir)
 		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
-		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
-			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
-			o.LDAPObjClasses)
+		svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
+			o.LDAPGroupIdAtr, o.LDAPObjClasses)
 		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
 	case o.S3Endpoint != "":
-		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+		svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
 			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
 		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
 			o.S3Endpoint, o.S3Bucket)
+	case o.VaultEndpointURL != "":
+		svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
+			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
+			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
+		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")

auth/iam_cache.go

@@ -36,14 +36,14 @@ type IAMCache struct {
 var _ IAMService = &IAMCache{}
 
 type item struct {
-	value Account
 	exp   time.Time
+	value Account
 }
 
 type icache struct {
-	sync.RWMutex
-	expire time.Duration
 	items  map[string]item
+	expire time.Duration
+	sync.RWMutex
 }
 
 func (i *icache) set(k string, v Account) {
@@ -66,6 +66,21 @@ func (i *icache) get(k string) (Account, bool) {
 	return v.value, true
 }
 
+func (i *icache) update(k string, props MutableProps) {
+	i.Lock()
+	defer i.Unlock()
+
+	item, found := i.items[k]
+	if found {
+		updateAcc(&item.value, props)
+
+		// refresh the expiration date
+		item.exp = time.Now().Add(i.expire)
+
+		i.items[k] = item
+	}
+}
+
 func (i *icache) Delete(k string) {
 	i.Lock()
 	delete(i.items, k)
@@ -166,6 +181,16 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
 	return nil
 }
 
+func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
+	err := c.service.UpdateUserAccount(access, props)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.update(access, props)
+	return nil
+}
+
 // ListUserAccounts is a passthrough to the underlying service and
 // does not make use of the cache
 func (c *IAMCache) ListUserAccounts() ([]Account, error) {

auth/iam_internal.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"sync"
 	"time"
 )
 
@@ -32,7 +33,15 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	dir string
+	dir     string
+	rootAcc Account
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -46,9 +55,10 @@ type iAMConfig struct {
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(dir string) (*IAMServiceInternal, error) {
+func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		dir: dir,
+		dir:     dir,
+		rootAcc: rootAcc,
 	}
 
 	err := i.initIAM()
@@ -62,6 +72,13 @@ func NewInternal(dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	if account.Access == s.rootAcc.Access {
+		return ErrUserExists
+	}
+
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -70,7 +87,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 
 		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
-			return nil, fmt.Errorf("account already exists")
+			return nil, ErrUserExists
 		}
 		conf.AccessAccounts[account.Access] = account
 
@@ -86,6 +103,13 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -99,9 +123,41 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+// UpdateUserAccount updates the specified user account fields. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		acc, found := conf.AccessAccounts[access]
+		if !found {
+			return nil, ErrNoSuchUser
+		}
+
+		updateAcc(&acc, props)
+		conf.AccessAccounts[access] = acc
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -121,6 +177,9 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 
 // ListUserAccounts lists all the user accounts stored.
 func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return []Account{}, fmt.Errorf("get iam data: %w", err)
@@ -135,12 +194,11 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
 	var accs []Account
 	for _, k := range keys {
 		accs = append(accs, Account{
-			Access:    k,
-			Secret:    conf.AccessAccounts[k].Secret,
-			Role:      conf.AccessAccounts[k].Role,
-			UserID:    conf.AccessAccounts[k].UserID,
-			GroupID:   conf.AccessAccounts[k].GroupID,
-			ProjectID: conf.AccessAccounts[k].ProjectID,
+			Access:  k,
+			Secret:  conf.AccessAccounts[k].Secret,
+			Role:    conf.AccessAccounts[k].Role,
+			UserID:  conf.AccessAccounts[k].UserID,
+			GroupID: conf.AccessAccounts[k].GroupID,
 		})
 	}
 
@@ -189,6 +247,10 @@ func parseIAM(b []byte) (iAMConfig, error) {
 		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
 	}
 
+	if conf.AccessAccounts == nil {
+		conf.AccessAccounts = make(map[string]Account)
+	}
+
 	return conf, nil
 }
 

auth/iam_ldap.go

@@ -1,7 +1,22 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package auth
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/go-ldap/ldap/v3"
@@ -14,12 +29,16 @@ type LdapIAMService struct {
 	accessAtr  string
 	secretAtr  string
 	roleAtr    string
+	groupIdAtr string
+	userIdAtr  string
+	rootAcc    Account
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
-	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
+		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
 	conn, err := ldap.DialURL(url)
@@ -38,15 +57,23 @@ func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objCl
 		accessAtr:  accAtr,
 		secretAtr:  secAtr,
 		roleAtr:    roleAtr,
+		userIdAtr:  userIdAtr,
+		groupIdAtr: groupIdAtr,
+		rootAcc:    rootAcc,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
-	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	if ld.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
 	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
+	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
+	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -57,6 +84,9 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
 }
 
 func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ld.rootAcc.Access {
+		return ld.rootAcc, nil
+	}
 	searchRequest := ldap.NewSearchRequest(
 		ld.queryBase,
 		ldap.ScopeWholeSubtree,
@@ -65,7 +95,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
 		nil,
 	)
 
@@ -74,14 +104,48 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		return Account{}, err
 	}
 
+	if len(result.Entries) == 0 {
+		return Account{}, ErrNoSuchUser
+	}
+
 	entry := result.Entries[0]
+	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
+	}
+	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
+	}
 	return Account{
-		Access: entry.GetAttributeValue(ld.accessAtr),
-		Secret: entry.GetAttributeValue(ld.secretAtr),
-		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
+		Access:  entry.GetAttributeValue(ld.accessAtr),
+		Secret:  entry.GetAttributeValue(ld.secretAtr),
+		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
+		GroupID: groupId,
+		UserID:  userId,
 	}, nil
 }
 
+func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+	if props.Secret != nil {
+		req.Replace(ld.secretAtr, []string{*props.Secret})
+	}
+	if props.GroupID != nil {
+		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
+	}
+	if props.UserID != nil {
+		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
+	}
+
+	err := ld.conn.Modify(req)
+	//TODO: Handle non existing user case
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (ld *LdapIAMService) DeleteUserAccount(access string) error {
 	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
 
@@ -106,7 +170,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(&%v)", searchFilter),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
 		nil,
 	)
 
@@ -117,10 +181,20 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 
 	result := []Account{}
 	for _, el := range resp.Entries {
+		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
+		}
+		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
+		}
 		result = append(result, Account{
-			Access: el.GetAttributeValue(ld.accessAtr),
-			Secret: el.GetAttributeValue(ld.secretAtr),
-			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
+			Access:  el.GetAttributeValue(ld.accessAtr),
+			Secret:  el.GetAttributeValue(ld.secretAtr),
+			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
+			GroupID: groupId,
+			UserID:  userId,
 		})
 	}
 

auth/iam_s3_object.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"sort"
+	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -41,19 +42,29 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
-	access        string
-	secret        string
-	region        string
-	bucket        string
-	endpoint      string
+	client *s3.Client
+
+	access   string
+	secret   string
+	region   string
+	bucket   string
+	endpoint string
+	rootAcc  Account
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+
 	sslSkipVerify bool
 	debug         bool
-	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}
 
-func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
 	if access == "" {
 		return nil, fmt.Errorf("must provide s3 IAM service access key")
 	}
@@ -78,6 +89,7 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 		endpoint:      endpoint,
 		sslSkipVerify: sslSkipVerify,
 		debug:         debug,
+		rootAcc:       rootAcc,
 	}
 
 	cfg, err := i.getConfig()
@@ -85,11 +97,25 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 		return nil, fmt.Errorf("init s3 IAM: %v", err)
 	}
 
+	if endpoint != "" {
+		i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &endpoint
+		})
+		return i, nil
+	}
+
 	i.client = s3.NewFromConfig(cfg)
 	return i, nil
 }
 
 func (s *IAMServiceS3) CreateAccount(account Account) error {
+	if s.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -97,7 +123,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 
 	_, ok := conf.AccessAccounts[account.Access]
 	if ok {
-		return fmt.Errorf("account already exists")
+		return ErrUserExists
 	}
 	conf.AccessAccounts[account.Access] = account
 
@@ -105,6 +131,13 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 }
 
 func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return Account{}, err
@@ -118,7 +151,30 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	acc, ok := conf.AccessAccounts[access]
+	if !ok {
+		return ErrNoSuchUser
+	}
+
+	updateAcc(&acc, props)
+	conf.AccessAccounts[access] = acc
+
+	return s.storeAccts(conf)
+}
+
 func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -134,6 +190,9 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
 }
 
 func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return nil, err
@@ -148,28 +207,17 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
 	var accs []Account
 	for _, k := range keys {
 		accs = append(accs, Account{
-			Access:    k,
-			Secret:    conf.AccessAccounts[k].Secret,
-			Role:      conf.AccessAccounts[k].Role,
-			UserID:    conf.AccessAccounts[k].UserID,
-			GroupID:   conf.AccessAccounts[k].GroupID,
-			ProjectID: conf.AccessAccounts[k].ProjectID,
+			Access:  k,
+			Secret:  conf.AccessAccounts[k].Secret,
+			Role:    conf.AccessAccounts[k].Role,
+			UserID:  conf.AccessAccounts[k].UserID,
+			GroupID: conf.AccessAccounts[k].GroupID,
 		})
 	}
 
 	return accs, nil
 }
 
-// ResolveEndpoint is used for on prem or non-aws endpoints
-func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
-	return aws.Endpoint{
-		PartitionID:       "aws",
-		URL:               s.endpoint,
-		SigningRegion:     s.region,
-		HostnameImmutable: true,
-	}, nil
-}
-
 func (s *IAMServiceS3) Shutdown() error {
 	return nil
 }
@@ -188,11 +236,6 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
 		config.WithHTTPClient(client),
 	}
 
-	if s.endpoint != "" {
-		opts = append(opts,
-			config.WithEndpointResolverWithOptions(s))
-	}
-
 	if s.debug {
 		opts = append(opts,
 			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
@@ -210,15 +253,15 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
 	})
 	if err != nil {
 		// if the error is object not exists,
-		// init empty accounts stuct and return that
+		// init empty accounts struct and return that
 		var nsk *types.NoSuchKey
 		if errors.As(err, &nsk) {
-			return iAMConfig{}, nil
+			return iAMConfig{AccessAccounts: map[string]Account{}}, nil
 		}
 		var apiErr smithy.APIError
 		if errors.As(err, &apiErr) {
 			if apiErr.ErrorCode() == "NotFound" {
-				return iAMConfig{}, nil
+				return iAMConfig{AccessAccounts: map[string]Account{}}, nil
 			}
 		}
 

auth/iam_single.go

@@ -32,7 +32,12 @@ func (IAMServiceSingle) CreateAccount(account Account) error {
 
 // GetUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	return Account{}, ErrNotSupported
+	return Account{}, ErrNoSuchUser
+}
+
+// UpdateUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
+	return ErrNotSupported
 }
 
 // DeleteUserAccount no accounts in single tenant mode

auth/iam_vault.go

@@ -0,0 +1,256 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	vault "github.com/hashicorp/vault-client-go"
+	"github.com/hashicorp/vault-client-go/schema"
+)
+
+type VaultIAMService struct {
+	client            *vault.Client
+	reqOpts           []vault.RequestOption
+	secretStoragePath string
+	rootAcc           Account
+}
+
+var _ IAMService = &VaultIAMService{}
+
+func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
+	opts := []vault.ClientOption{
+		vault.WithAddress(endpoint),
+		// set request timeout to 10 secs
+		vault.WithRequestTimeout(10 * time.Second),
+	}
+	if serverCert != "" {
+		tls := vault.TLSConfiguration{}
+
+		tls.ServerCertificate.FromBytes = []byte(serverCert)
+		if clientCert != "" {
+			if clientCertKey == "" {
+				return nil, fmt.Errorf("client certificate and client certificate should both be specified")
+			}
+
+			tls.ClientCertificate.FromBytes = []byte(clientCert)
+			tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
+		}
+
+		opts = append(opts, vault.WithTLS(tls))
+	}
+
+	client, err := vault.New(opts...)
+	if err != nil {
+		return nil, fmt.Errorf("init vault client: %w", err)
+	}
+
+	reqOpts := []vault.RequestOption{}
+	// if mount path is not specified, it defaults to "approle"
+	if mountPath != "" {
+		reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
+	}
+
+	// Authentication
+	switch {
+	case rootToken != "":
+		err := client.SetToken(rootToken)
+		if err != nil {
+			return nil, fmt.Errorf("root token authentication failure: %w", err)
+		}
+	case roleID != "":
+		if roleSecret == "" {
+			return nil, fmt.Errorf("role id and role secret must both be specified")
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
+			RoleId:   roleID,
+			SecretId: roleSecret,
+		}, reqOpts...)
+		cancel()
+		if err != nil {
+			return nil, fmt.Errorf("approle authentication failure: %w", err)
+		}
+
+		if err := client.SetToken(resp.Auth.ClientToken); err != nil {
+			return nil, fmt.Errorf("approle authentication set token failure: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
+	}
+
+	return &VaultIAMService{
+		client:            client,
+		reqOpts:           reqOpts,
+		secretStoragePath: secretStoragePath,
+		rootAcc:           rootAcc,
+	}, nil
+}
+
+func (vt *VaultIAMService) CreateAccount(account Account) error {
+	if vt.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
+		Data: map[string]any{
+			account.Access: account,
+		},
+		Options: map[string]interface{}{
+			"cas": 0,
+		},
+	}, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		if strings.Contains(err.Error(), "check-and-set") {
+			return ErrUserExists
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
+	if vt.rootAcc.Access == access {
+		return vt.rootAcc, nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acc, err := parseVaultUserAccount(resp.Data.Data, access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	return acc, nil
+}
+
+func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	//TODO: We need something like a transaction here ?
+	acc, err := vt.GetUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	updateAcc(&acc, props)
+
+	err = vt.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	err = vt.CreateAccount(acc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (vt *VaultIAMService) DeleteUserAccount(access string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
+	cancel()
+	if err != nil {
+		if vault.IsErrorStatus(err, 404) {
+			return []Account{}, nil
+		}
+		return nil, err
+	}
+
+	accs := []Account{}
+
+	for _, acss := range resp.Data.Keys {
+		acc, err := vt.GetUserAccount(acss)
+		if err != nil {
+			return nil, err
+		}
+		accs = append(accs, acc)
+	}
+
+	return accs, nil
+}
+
+// the client doesn't have explicit shutdown, as it uses http.Client
+func (vt *VaultIAMService) Shutdown() error {
+	return nil
+}
+
+var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
+
+func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
+	usrAcc, ok := data[access].(map[string]interface{})
+	if !ok {
+		return acc, errInvalidUser
+	}
+
+	acss, ok := usrAcc["access"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	secret, ok := usrAcc["secret"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	role, ok := usrAcc["role"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userIdJson, ok := usrAcc["userID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userId, err := userIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+	groupIdJson, ok := usrAcc["groupID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	groupId, err := groupIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+
+	return Account{
+		Access:  acss,
+		Secret:  secret,
+		Role:    Role(role),
+		UserID:  int(userId),
+		GroupID: int(groupId),
+	}, nil
+}

auth/object_lock.go

@@ -0,0 +1,269 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+type BucketLockConfig struct {
+	DefaultRetention *types.DefaultRetention
+	CreatedAt        *time.Time
+	Enabled          bool
+}
+
+func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
+	var lockConfig types.ObjectLockConfiguration
+	if err := xml.Unmarshal(input, &lockConfig); err != nil {
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	config := BucketLockConfig{
+		Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
+	}
+
+	if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
+		retention := lockConfig.Rule.DefaultRetention
+
+		if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
+			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+		}
+		if retention.Years != nil && retention.Days != nil {
+			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+		}
+
+		if retention.Days != nil && *retention.Days <= 0 {
+			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
+		}
+		if retention.Years != nil && *retention.Years <= 0 {
+			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
+		}
+
+		config.DefaultRetention = retention
+		now := time.Now()
+		config.CreatedAt = &now
+	}
+
+	return json.Marshal(config)
+}
+
+func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
+	var config BucketLockConfig
+	if err := json.Unmarshal(input, &config); err != nil {
+		return nil, fmt.Errorf("parse object lock config: %w", err)
+	}
+
+	result := &types.ObjectLockConfiguration{
+		Rule: &types.ObjectLockRule{
+			DefaultRetention: config.DefaultRetention,
+		},
+	}
+
+	if config.Enabled {
+		result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
+	}
+
+	return result, nil
+}
+
+func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
+	var retention s3response.PutObjectRetentionInput
+	if err := xml.Unmarshal(input, &retention); err != nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	if retention.RetainUntilDate.Before(time.Now()) {
+		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
+	}
+	switch retention.Mode {
+	case types.ObjectLockRetentionModeCompliance:
+	case types.ObjectLockRetentionModeGovernance:
+	default:
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	return json.Marshal(retention)
+}
+
+func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
+	var retention types.ObjectLockRetention
+	if err := json.Unmarshal(input, &retention); err != nil {
+		return nil, fmt.Errorf("parse object lock retention: %w", err)
+	}
+
+	return &retention, nil
+}
+
+func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
+	if status == nil {
+		return nil
+	}
+
+	if *status {
+		return &types.ObjectLockLegalHold{
+			Status: types.ObjectLockLegalHoldStatusOn,
+		}
+	}
+
+	return &types.ObjectLockLegalHold{
+		Status: types.ObjectLockLegalHoldStatusOff,
+	}
+}
+
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
+	data, err := be.GetObjectLockConfiguration(ctx, bucket)
+	if err != nil {
+		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
+			return nil
+		}
+
+		return err
+	}
+
+	var bucketLockConfig BucketLockConfig
+	if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse object lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return nil
+	}
+
+	checkDefaultRetention := false
+
+	if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
+		expirationDate := *bucketLockConfig.CreatedAt
+		if bucketLockConfig.DefaultRetention.Days != nil {
+			expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
+		}
+		if bucketLockConfig.DefaultRetention.Years != nil {
+			expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
+		}
+
+		if expirationDate.After(time.Now()) {
+			checkDefaultRetention = true
+		}
+	}
+
+	for _, obj := range objects {
+		var key, versionId string
+		if obj.Key != nil {
+			key = *obj.Key
+		}
+		if obj.VersionId != nil {
+			versionId = *obj.VersionId
+		}
+		checkRetention := true
+		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
+		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+			continue
+		}
+		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
+			checkRetention = false
+		}
+		if err != nil && checkRetention {
+			return err
+		}
+
+		if checkRetention {
+			retention, err := ParseObjectLockRetentionOutput(retentionData)
+			if err != nil {
+				return err
+			}
+
+			if retention.Mode != "" && retention.RetainUntilDate != nil {
+				if retention.RetainUntilDate.After(time.Now()) {
+					switch retention.Mode {
+					case types.ObjectLockRetentionModeGovernance:
+						if !bypass {
+							return s3err.GetAPIError(s3err.ErrObjectLocked)
+						} else {
+							policy, err := be.GetBucketPolicy(ctx, bucket)
+							if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+								return s3err.GetAPIError(s3err.ErrObjectLocked)
+							}
+							if err != nil {
+								return err
+							}
+							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+							if err != nil {
+								return s3err.GetAPIError(s3err.ErrObjectLocked)
+							}
+						}
+					case types.ObjectLockRetentionModeCompliance:
+						return s3err.GetAPIError(s3err.ErrObjectLocked)
+					}
+				}
+			}
+		}
+
+		checkLegalHold := true
+
+		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
+		if err != nil {
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+				continue
+			}
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
+				checkLegalHold = false
+			} else {
+				return err
+			}
+		}
+
+		if checkLegalHold && *status {
+			return s3err.GetAPIError(s3err.ErrObjectLocked)
+		}
+
+		if checkDefaultRetention {
+			switch bucketLockConfig.DefaultRetention.Mode {
+			case types.ObjectLockRetentionModeGovernance:
+				if !bypass {
+					return s3err.GetAPIError(s3err.ErrObjectLocked)
+				} else {
+					policy, err := be.GetBucketPolicy(ctx, bucket)
+					if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+						return s3err.GetAPIError(s3err.ErrObjectLocked)
+					}
+					if err != nil {
+						return err
+					}
+					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+					if err != nil {
+						return s3err.GetAPIError(s3err.ErrObjectLocked)
+					}
+				}
+			case types.ObjectLockRetentionModeCompliance:
+				return s3err.GetAPIError(s3err.ErrObjectLocked)
+			}
+		}
+	}
+
+	return nil
+}

aws/signer/v4/functional_test.go

@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 	}
 }
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but recieved %v", e, a)
+			t.Errorf("expected %v, but received %v", e, a)
 		}
 	}
 }

aws/signer/v4/v4.go

@@ -85,6 +85,10 @@ type keyDerivator interface {
 
 // SignerOptions is the SigV4 Signer options.
 type SignerOptions struct {
+
+	// The logger to send log messages to.
+	Logger logging.Logger
+
 	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
 	// request header to the request's query string. This is most commonly used
 	// with pre-signed requests preventing headers from being added to the
@@ -100,9 +104,6 @@ type SignerOptions struct {
 	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
 	DisableURIPathEscaping bool
 
-	// The logger to send log messages to.
-	Logger logging.Logger
-
 	// Enable logging of signed requests.
 	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
 	// presigned URL.
@@ -117,8 +118,8 @@ type SignerOptions struct {
 // Signer applies AWS v4 signing to given request. Use this to sign requests
 // that need to be signed with AWS V4 Signatures.
 type Signer struct {
-	options      SignerOptions
 	keyDerivator keyDerivator
+	options      SignerOptions
 }
 
 // NewSigner returns a new SigV4 Signer
@@ -133,17 +134,19 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
 }
 
 type httpSigner struct {
+	KeyDerivator keyDerivator
 	Request      *http.Request
+	Credentials  aws.Credentials
+	Time         v4Internal.SigningTime
 	ServiceName  string
 	Region       string
-	Time         v4Internal.SigningTime
-	Credentials  aws.Credentials
-	KeyDerivator keyDerivator
-	IsPreSign    bool
-	SignedHdrs   []string
 
 	PayloadHash string
 
+	SignedHdrs []string
+
+	IsPreSign bool
+
 	DisableHeaderHoisting  bool
 	DisableURIPathEscaping bool
 	DisableSessionToken    bool

backend/backend.go

@@ -18,9 +18,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 	"github.com/versity/versitygw/s3select"
@@ -32,20 +32,23 @@ type Backend interface {
 	Shutdown()
 
 	// bucket operations
-	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
 	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
 	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
-	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
-	PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
-	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	DeleteBucket(_ context.Context, bucket string) error
+	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
+	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	DeleteBucketPolicy(_ context.Context, bucket string) error
+	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
+	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
+	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
 
 	// multipart operations
-	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
@@ -54,18 +57,18 @@ type Backend interface {
 	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
 
 	// standard object operations
-	PutObject(context.Context, *s3.PutObjectInput) (string, error)
+	PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
 	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
-	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
-	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
-	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
 	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
-	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
+	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
 
 	// special case object operations
 	RestoreObject(context.Context, *s3.RestoreObjectInput) error
@@ -81,8 +84,16 @@ type Backend interface {
 	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
 	DeleteObjectTagging(_ context.Context, bucket, object string) error
 
+	// object lock operations
+	PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
+	GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
+	PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
+	GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
+	PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
+	GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
+
 	// non AWS actions
-	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
 	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
@@ -97,7 +108,7 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -112,14 +123,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
 func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
+func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -130,9 +141,18 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
 func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -153,32 +173,32 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu
 	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+	return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -205,8 +225,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
 	}
 }
 
-func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
@@ -229,7 +249,26 @@ func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {

backend/common.go

@@ -18,7 +18,9 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"io/fs"
+	"io"
+	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -28,9 +30,10 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
-var (
-	// RFC3339TimeFormat RFC3339 time format
-	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+const (
+	// this is the media type for directories in AWS and Nextcloud
+	DirContentType     = "application/x-directory"
+	DefaultContentType = "binary/octet-stream"
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -47,8 +50,18 @@ func (d ByObjectName) Len() int           { return len(d) }
 func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
 
-func GetStringPtr(s string) *string {
-	return &s
+func GetPtrFromString(str string) *string {
+	if str == "" {
+		return nil
+	}
+	return &str
+}
+
+func GetStringFromPtr(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
 }
 
 func GetTimePtr(t time.Time) *time.Time {
@@ -61,9 +74,9 @@ var (
 
 // ParseRange parses input range header and returns startoffset, length, and
 // error. If no endoffset specified, then length is set to -1.
-func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
+func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, fi.Size(), nil
+		return 0, size, nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
@@ -99,6 +112,38 @@ func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
+// ParseCopySource parses x-amz-copy-source header and returns source bucket,
+// source object, versionId, error respectively
+func ParseCopySource(copySourceHeader string) (string, string, string, error) {
+	if copySourceHeader[0] == '/' {
+		copySourceHeader = copySourceHeader[1:]
+	}
+
+	var copySource, versionId string
+	i := strings.LastIndex(copySourceHeader, "?versionId=")
+	if i == -1 {
+		copySource = copySourceHeader
+	} else {
+		copySource = copySourceHeader[:i]
+		versionId = copySourceHeader[i+11:]
+	}
+
+	srcBucket, srcObject, ok := strings.Cut(copySource, "/")
+	if !ok {
+		return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	return srcBucket, srcObject, versionId, nil
+}
+
+func CreateExceedingRangeErr(objSize int64) s3err.APIError {
+	return s3err.APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
@@ -120,3 +165,16 @@ func md5String(data []byte) string {
 	sum := md5.Sum(data)
 	return hex.EncodeToString(sum[:])
 }
+
+type FileSectionReadCloser struct {
+	R io.Reader
+	F *os.File
+}
+
+func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
+	return f.R.Read(p)
+}
+
+func (f *FileSectionReadCloser) Close() error {
+	return f.F.Close()
+}

backend/meta/meta.go

@@ -0,0 +1,42 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import "os"
+
+// MetadataStorer defines the interface for managing metadata.
+// When object == "", the operation is on the bucket.
+type MetadataStorer interface {
+	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+	// Returns the value of the attribute, or an error if the attribute does not exist.
+	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
+
+	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+	// If attribute already exists, new attribute should replace existing.
+	// Returns an error if the operation fails.
+	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
+
+	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+	// Returns an error if the operation fails.
+	DeleteAttribute(bucket, object, attribute string) error
+
+	// ListAttributes lists all attributes for an object or a bucket.
+	// Returns list of attribute names, or an error if the operation fails.
+	ListAttributes(bucket, object string) ([]string, error)
+
+	// DeleteAttributes removes all attributes for an object or a bucket.
+	// Returns an error if the operation fails.
+	DeleteAttributes(bucket, object string) error
+}

backend/meta/xattr.go

@@ -0,0 +1,114 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/xattr"
+)
+
+const (
+	xattrPrefix = "user."
+)
+
+var (
+	// ErrNoSuchKey is returned when the key does not exist.
+	ErrNoSuchKey = errors.New("no such key")
+)
+
+type XattrMeta struct{}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
+	if f != nil {
+		b, err := xattr.FGet(f, xattrPrefix+attribute)
+		if errors.Is(err, xattr.ENOATTR) {
+			return nil, ErrNoSuchKey
+		}
+		return b, err
+	}
+
+	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
+	if errors.Is(err, xattr.ENOATTR) {
+		return nil, ErrNoSuchKey
+	}
+	return b, err
+}
+
+// StoreAttribute stores the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
+	if f != nil {
+		return xattr.FSet(f, xattrPrefix+attribute, value)
+	}
+
+	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
+	err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
+	if errors.Is(err, xattr.ENOATTR) {
+		return ErrNoSuchKey
+	}
+	return err
+}
+
+// DeleteAttributes is not implemented for xattr since xattrs
+// are automatically removed when the file is deleted.
+func (x XattrMeta) DeleteAttributes(bucket, object string) error {
+	return nil
+}
+
+// ListAttributes lists all attributes for an object in a bucket.
+func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
+	attrs, err := xattr.List(filepath.Join(bucket, object))
+	if err != nil {
+		return nil, err
+	}
+	attributes := make([]string, 0, len(attrs))
+	for _, attr := range attrs {
+		if !isUserAttr(attr) {
+			continue
+		}
+		attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
+	}
+	return attributes, nil
+}
+
+func isUserAttr(attr string) bool {
+	return strings.HasPrefix(attr, xattrPrefix)
+}
+
+// Test is a helper function to test if xattrs are supported.
+func (x XattrMeta) Test(path string) error {
+	// check for platform support
+	if !xattr.XATTR_SUPPORTED {
+		return fmt.Errorf("xattrs are not supported on this platform")
+	}
+
+	// check if the filesystem supports xattrs
+	_, err := xattr.Get(path, "user.test")
+	if errors.Is(err, syscall.ENOTSUP) {
+		return fmt.Errorf("xattrs are not supported on this filesystem")
+	}
+
+	return nil
+}

backend/mkdir.go

@@ -1,6 +1,7 @@
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+// Copyright 2024 Versity Software
 
 // MkdirAll borrowed from stdlib to add ability to set ownership
 // as directories are created
@@ -14,11 +15,6 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
-var (
-	// TODO: make this configurable
-	defaultDirPerm fs.FileMode = 0755
-)
-
 // MkdirAll is similar to os.MkdirAll but it will return
 // ErrObjectParentIsFile when appropriate
 // MkdirAll creates a directory named path,
@@ -29,9 +25,9 @@ var (
 // Any newly created directory is set to provided uid/gid ownership.
 // If path is already a directory, MkdirAll does nothing
 // and returns nil.
-// Any directoy created will be set to provided uid/gid ownership
+// Any directory created will be set to provided uid/gid ownership
 // if doChown is true.
-func MkdirAll(path string, uid, gid int, doChown bool) error {
+func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
 	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
 	dir, err := os.Stat(path)
 	if err == nil {
@@ -54,14 +50,14 @@ func MkdirAll(path string, uid, gid int, doChown bool) error {
 
 	if j > 1 {
 		// Create parent.
-		err = MkdirAll(path[:j-1], uid, gid, doChown)
+		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
 		if err != nil {
 			return err
 		}
 	}
 
 	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, defaultDirPerm)
+	err = os.Mkdir(path, dirPerm)
 	if err != nil {
 		// Handle arguments like "foo/." by
 		// double-checking that directory doesn't exist.

backend/posix/with_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build !freebsd && !openbsd && !netbsd
-// +build !freebsd,!openbsd,!netbsd
-
-package posix
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENODATA
-)

backend/posix/with_otmpfile.go

@@ -38,11 +38,12 @@ type tmpfile struct {
 	f          *os.File
 	bucket     string
 	objname    string
-	isOTmp     bool
 	size       int64
-	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
+	isOTmp     bool
+	needsChown bool
 }
 
 var (
@@ -50,7 +51,7 @@ var (
 	defaultFilePerm uint32 = 0644
 )
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
@@ -62,7 +63,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown)
+		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -81,7 +82,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 			gid:        gid,
 		}
 		// falloc is best effort, its fine if this fails
-		if size > 0 {
+		if size > 0 && dofalloc {
 			tmp.falloc()
 		}
 
@@ -108,10 +109,11 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
+		newDirPerm: p.newDirPerm,
 	}
 
 	// falloc is best effort, its fine if this fails
-	if size > 0 {
+	if size > 0 && dofalloc {
 		tmp.falloc()
 	}
 
@@ -134,6 +136,9 @@ func (tmp *tmpfile) falloc() error {
 }
 
 func (tmp *tmpfile) link() error {
+	// make sure this is cleaned up in all error cases
+	defer tmp.f.Close()
+
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
 	// with any other simultaneous uploads. The final operation is to move the
@@ -148,7 +153,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -170,11 +175,21 @@ func (tmp *tmpfile) link() error {
 	}
 	defer dirf.Close()
 
-	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-	if err != nil {
-		return fmt.Errorf("link tmpfile (%q in %q): %w",
-			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
+	for {
+		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		if errors.Is(err, syscall.EEXIST) {
+			err := os.Remove(objPath)
+			if err != nil && !errors.Is(err, fs.ErrNotExist) {
+				return fmt.Errorf("remove stale path: %w", err)
+			}
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("link tmpfile (fd %q as %q): %w",
+				filepath.Base(tmp.f.Name()), objPath, err)
+		}
+		break
 	}
 
 	err = tmp.f.Close()
@@ -221,3 +236,7 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
+
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}

backend/posix/without_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build freebsd || openbsd || netbsd
-// +build freebsd openbsd netbsd
-
-package posix
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENOATTR
-)

backend/posix/without_otmpfile.go

@@ -36,12 +36,12 @@ type tmpfile struct {
 	size    int64
 }
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
 	uid, gid, doChown := p.getChownIDs(acct)
 
 	// Create a temp file for upload while in progress (see link comments below).
 	var err error
-	err = backend.MkdirAll(dir, uid, gid, doChown)
+	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
@@ -112,3 +112,7 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
+
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}

backend/s3proxy/client.go

@@ -33,6 +33,12 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
 		return nil, err
 	}
 
+	if s.endpoint != "" {
+		return s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &s.endpoint
+		}), nil
+	}
+
 	return s3.NewFromConfig(cfg), nil
 }
 
@@ -50,11 +56,6 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 		config.WithHTTPClient(client),
 	}
 
-	if s.endpoint != "" {
-		opts = append(opts,
-			config.WithEndpointResolverWithOptions(s))
-	}
-
 	if s.disableChecksum {
 		opts = append(opts,
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -67,13 +68,3 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 
 	return config.LoadDefaultConfig(ctx, opts...)
 }
-
-// ResolveEndpoint is used for on prem or non-aws endpoints
-func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
-	return aws.Endpoint{
-		PartitionID:       "aws",
-		URL:               s.endpoint,
-		SigningRegion:     s.awsRegion,
-		HostnameImmutable: true,
-	}, nil
-}

backend/s3proxy/s3.go

@@ -33,6 +33,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
@@ -54,6 +55,8 @@ type S3Proxy struct {
 	debug           bool
 }
 
+var _ backend.Backend = &S3Proxy{}
+
 func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
 	s := &S3Proxy{
 		access:          access,
@@ -72,8 +75,12 @@ func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify
 	return s, nil
 }
 
-func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
-	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
+func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{
+		ContinuationToken: &input.ContinuationToken,
+		MaxBuckets:        &input.MaxBuckets,
+		Prefix:            &input.Prefix,
+	})
 	if err != nil {
 		return s3response.ListAllMyBucketsResult{}, handleError(err)
 	}
@@ -93,6 +100,8 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
 		Buckets: s3response.ListAllMyBucketsList{
 			Bucket: buckets,
 		},
+		ContinuationToken: backend.GetStringFromPtr(output.ContinuationToken),
+		Prefix:            backend.GetStringFromPtr(output.Prefix),
 	}, nil
 }
 
@@ -109,8 +118,8 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 
 	var tagSet []types.Tag
 	tagSet = append(tagSet, types.Tag{
-		Key:   backend.GetStringPtr(aclKey),
-		Value: backend.GetStringPtr(base64Encode(acl)),
+		Key:   backend.GetPtrFromString(aclKey),
+		Value: backend.GetPtrFromString(base64Encode(acl)),
 	})
 
 	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
@@ -122,14 +131,100 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 	return handleError(err)
 }
 
-func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
-	_, err := s.client.DeleteBucket(ctx, input)
+func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: &bucket,
+	})
 	return handleError(err)
 }
 
-func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
+	_, err := s.client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
+		Bucket: &bucket,
+		OwnershipControls: &types.OwnershipControls{
+			Rules: []types.OwnershipControlsRule{
+				{
+					ObjectOwnership: ownership,
+				},
+			},
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
+	var ownship types.ObjectOwnership
+	resp, err := s.client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return ownship, handleError(err)
+	}
+	return resp.OwnershipControls.Rules[0].ObjectOwnership, nil
+}
+func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
+		Bucket: &bucket,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
+	_, err := s.client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
+		Bucket: &bucket,
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: status,
+		},
+	})
+
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
+		Bucket: &bucket,
+	})
+
+	return s3response.GetBucketVersioningOutput{
+		Status:    &out.Status,
+		MFADelete: &out.MFADelete,
+	}, handleError(err)
+}
+
+func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	out, err := s.client.ListObjectVersions(ctx, input)
+	if err != nil {
+		return s3response.ListVersionsResult{}, handleError(err)
+	}
+
+	return s3response.ListVersionsResult{
+		CommonPrefixes:      out.CommonPrefixes,
+		DeleteMarkers:       out.DeleteMarkers,
+		Delimiter:           out.Delimiter,
+		EncodingType:        out.EncodingType,
+		IsTruncated:         out.IsTruncated,
+		KeyMarker:           out.KeyMarker,
+		MaxKeys:             out.MaxKeys,
+		Name:                out.Name,
+		NextKeyMarker:       out.NextKeyMarker,
+		NextVersionIdMarker: out.NextVersionIdMarker,
+		Prefix:              out.Prefix,
+		VersionIdMarker:     input.VersionIdMarker,
+		Versions:            out.Versions,
+	}, nil
+}
+
+func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
 	out, err := s.client.CreateMultipartUpload(ctx, input)
-	return out, handleError(err)
+	if err != nil {
+		return s3response.InitiateMultipartUploadResult{}, handleError(err)
+	}
+
+	return s3response.InitiateMultipartUploadResult{
+		Bucket:   *out.Bucket,
+		Key:      *out.Key,
+		UploadId: *out.UploadId,
+	}, nil
 }
 
 func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
@@ -161,8 +256,8 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
 				ID:          *u.Owner.ID,
 				DisplayName: *u.Owner.DisplayName,
 			},
-			StorageClass: string(u.StorageClass),
-			Initiated:    u.Initiated.Format(backend.RFC3339TimeFormat),
+			StorageClass: u.StorageClass,
+			Initiated:    *u.Initiated,
 		})
 	}
 
@@ -199,7 +294,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
 	for _, p := range output.Parts {
 		parts = append(parts, s3response.Part{
 			PartNumber:   int(*p.PartNumber),
-			LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
+			LastModified: *p.LastModified,
 			ETag:         *p.ETag,
 			Size:         *p.Size,
 		})
@@ -228,7 +323,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
 			ID:          *output.Owner.ID,
 			DisplayName: *output.Owner.DisplayName,
 		},
-		StorageClass:         string(output.StorageClass),
+		StorageClass:         output.StorageClass,
 		PartNumberMarker:     pnm,
 		NextPartNumberMarker: npmn,
 		MaxParts:             int(*output.MaxParts),
@@ -262,17 +357,25 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
 	}, nil
 }
 
-func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
 	// streaming backend is not seekable,
 	// use unsigned payload for streaming ops
 	output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
 		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
 	))
 	if err != nil {
-		return "", handleError(err)
+		return s3response.PutObjectOutput{}, handleError(err)
 	}
 
-	return *output.ETag, nil
+	var versionID string
+	if output.VersionId != nil {
+		versionID = *output.VersionId
+	}
+
+	return s3response.PutObjectOutput{
+		ETag:      *output.ETag,
+		VersionID: versionID,
+	}, nil
 }
 
 func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
@@ -280,24 +383,49 @@ func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	return out, handleError(err)
 }
 
-func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	output, err := s.client.GetObject(ctx, input)
 	if err != nil {
 		return nil, handleError(err)
 	}
-	defer output.Body.Close()
-
-	_, err = io.Copy(w, output.Body)
-	if err != nil {
-		return nil, err
-	}
 
 	return output, nil
 }
 
-func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	out, err := s.client.GetObjectAttributes(ctx, input)
-	return out, handleError(err)
+
+	parts := s3response.ObjectParts{}
+	objParts := out.ObjectParts
+	if objParts != nil {
+		if objParts.PartNumberMarker != nil {
+			partNumberMarker, err := strconv.Atoi(*objParts.PartNumberMarker)
+			if err != nil {
+				parts.PartNumberMarker = partNumberMarker
+			}
+			if objParts.NextPartNumberMarker != nil {
+				nextPartNumberMarker, err := strconv.Atoi(*objParts.NextPartNumberMarker)
+				if err != nil {
+					parts.NextPartNumberMarker = nextPartNumberMarker
+				}
+			}
+			if objParts.IsTruncated != nil {
+				parts.IsTruncated = *objParts.IsTruncated
+			}
+			if objParts.MaxParts != nil {
+				parts.MaxParts = int(*objParts.MaxParts)
+			}
+			parts.Parts = objParts.Parts
+		}
+	}
+
+	return s3response.GetObjectAttributesResponse{
+		ETag:         out.ETag,
+		LastModified: out.LastModified,
+		ObjectSize:   out.ObjectSize,
+		StorageClass: out.StorageClass,
+		ObjectParts:  &parts,
+	}, handleError(err)
 }
 
 func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
@@ -305,19 +433,52 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s
 	return out, handleError(err)
 }
 
-func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	out, err := s.client.ListObjects(ctx, input)
-	return out, handleError(err)
+	if err != nil {
+		return s3response.ListObjectsResult{}, handleError(err)
+	}
+
+	contents := convertObjects(out.Contents)
+
+	return s3response.ListObjectsResult{
+		CommonPrefixes: out.CommonPrefixes,
+		Contents:       contents,
+		Delimiter:      out.Delimiter,
+		IsTruncated:    out.IsTruncated,
+		Marker:         out.Marker,
+		MaxKeys:        out.MaxKeys,
+		Name:           out.Name,
+		NextMarker:     out.NextMarker,
+		Prefix:         out.Prefix,
+	}, nil
 }
 
-func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 	out, err := s.client.ListObjectsV2(ctx, input)
-	return out, handleError(err)
+	if err != nil {
+		return s3response.ListObjectsV2Result{}, handleError(err)
+	}
+
+	contents := convertObjects(out.Contents)
+
+	return s3response.ListObjectsV2Result{
+		CommonPrefixes:        out.CommonPrefixes,
+		Contents:              contents,
+		Delimiter:             out.Delimiter,
+		IsTruncated:           out.IsTruncated,
+		ContinuationToken:     out.ContinuationToken,
+		MaxKeys:               out.MaxKeys,
+		Name:                  out.Name,
+		NextContinuationToken: out.NextContinuationToken,
+		Prefix:                out.Prefix,
+		KeyCount:              out.KeyCount,
+	}, nil
 }
 
-func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
-	_, err := s.client.DeleteObject(ctx, input)
-	return handleError(err)
+func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	res, err := s.client.DeleteObject(ctx, input)
+	return res, handleError(err)
 }
 
 func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
@@ -369,8 +530,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
 	for i, tag := range tagout.TagSet {
 		if *tag.Key == aclKey {
 			tagout.TagSet[i] = types.Tag{
-				Key:   backend.GetStringPtr(aclKey),
-				Value: backend.GetStringPtr(base64Encode(data)),
+				Key:   backend.GetPtrFromString(aclKey),
+				Value: backend.GetPtrFromString(base64Encode(data)),
 			}
 			found = true
 			break
@@ -378,8 +539,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
 	}
 	if !found {
 		tagout.TagSet = append(tagout.TagSet, types.Tag{
-			Key:   backend.GetStringPtr(aclKey),
-			Value: backend.GetStringPtr(base64Encode(data)),
+			Key:   backend.GetPtrFromString(aclKey),
+			Value: backend.GetPtrFromString(base64Encode(data)),
 		})
 	}
 
@@ -436,8 +597,135 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
 	return handleError(err)
 }
 
-func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
+func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
+	_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+		Bucket: &bucket,
+		Policy: backend.GetPtrFromString(string(policy)),
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
+	policy, err := s.client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	result := []byte{}
+	if policy.Policy != nil {
+		result = []byte(*policy.Policy)
+	}
+
+	return result, nil
+}
+
+func (s *S3Proxy) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+		Bucket: &bucket,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
+	cfg, err := auth.ParseBucketLockConfigurationOutput(config)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{
+		Bucket:                  &bucket,
+		ObjectLockConfiguration: cfg,
+	})
+
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
+	resp, err := s.client.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	config := auth.BucketLockConfig{
+		Enabled:          resp.ObjectLockConfiguration.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
+		DefaultRetention: resp.ObjectLockConfiguration.Rule.DefaultRetention,
+	}
+
+	return json.Marshal(config)
+}
+
+func (s *S3Proxy) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
+	ret, err := auth.ParseObjectLockRetentionOutput(retention)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
+		Bucket:                    &bucket,
+		Key:                       &object,
+		VersionId:                 &versionId,
+		Retention:                 ret,
+		BypassGovernanceRetention: &bypass,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
+	resp, err := s.client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{
+		Bucket:    &bucket,
+		Key:       &object,
+		VersionId: &versionId,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	return json.Marshal(resp.Retention)
+}
+
+func (s *S3Proxy) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
+	var st types.ObjectLockLegalHoldStatus
+	if status {
+		st = types.ObjectLockLegalHoldStatusOn
+	} else {
+		st = types.ObjectLockLegalHoldStatusOff
+	}
+
+	_, err := s.client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{
+		Bucket:    &bucket,
+		Key:       &object,
+		VersionId: &versionId,
+		LegalHold: &types.ObjectLockLegalHold{
+			Status: st,
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
+	resp, err := s.client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{
+		Bucket:    &bucket,
+		Key:       &object,
+		VersionId: &versionId,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	status := resp.LegalHold.Status == types.ObjectLockLegalHoldStatusOn
+	return &status, nil
+}
+
+func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
+	var acll auth.ACL
+	if err := json.Unmarshal(acl, &acll); err != nil {
+		return fmt.Errorf("unmarshal acl: %w", err)
+	}
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, acll.Owner), nil)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
@@ -467,7 +755,7 @@ func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string
 			return err
 		}
 		defer resp.Body.Close()
-		return fmt.Errorf(string(body))
+		return fmt.Errorf("%v", string(body))
 	}
 
 	return nil
@@ -543,3 +831,21 @@ func base64Decode(encoded string) ([]byte, error) {
 	}
 	return decoded, nil
 }
+
+func convertObjects(objs []types.Object) []s3response.Object {
+	result := make([]s3response.Object, len(objs))
+
+	for _, obj := range objs {
+		result = append(result, s3response.Object{
+			ETag:          obj.ETag,
+			Key:           obj.Key,
+			LastModified:  obj.LastModified,
+			Owner:         obj.Owner,
+			Size:          obj.Size,
+			RestoreStatus: obj.RestoreStatus,
+			StorageClass:  obj.StorageClass,
+		})
+	}
+
+	return result
+}

backend/scoutfs/scoutfs.go

@@ -26,27 +26,44 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
 type ScoutfsOpts struct {
 	ChownUID    bool
 	ChownGID    bool
 	GlacierMode bool
+	BucketLinks bool
+	NewDirPerm  fs.FileMode
 }
 
 type ScoutFS struct {
+
+	// bucket/object metadata storage facility
+	meta meta.MetadataStorer
+
 	*posix.Posix
 	rootfd  *os.File
 	rootdir string
 
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
+
+	// newDirPerm is the permissions to use when creating new directories
+	newDirPerm fs.FileMode
+
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
 	// HEAD object: if file offline, set obj storage class to GLACIER
@@ -62,11 +79,6 @@ type ScoutFS struct {
 	// when objects are uploaded
 	chownuid bool
 	chowngid bool
-
-	// euid/egid are the effective uid/gid of the running versitygw process
-	// used to determine if chowning is needed
-	euid int
-	egid int
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -75,8 +87,13 @@ const (
 	metaTmpDir          = ".sgwtmp"
 	metaTmpMultipartDir = metaTmpDir + "/multipart"
 	tagHdr              = "X-Amz-Tagging"
+	metaHdr             = "X-Amz-Meta"
+	contentTypeHdr      = "content-type"
+	contentEncHdr       = "content-encoding"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
-	etagkey             = "user.etag"
+	etagkey             = "etag"
+	objectRetentionKey  = "object-retention"
+	objectLegalHoldKey  = "object-legal-hold"
 )
 
 var (
@@ -87,11 +104,12 @@ var (
 
 const (
 	// ScoutFS special xattr types
-
 	systemPrefix = "scoutfs.hide."
 	onameAttr    = systemPrefix + "objname"
 	flagskey     = systemPrefix + "sam_flags"
 	stagecopykey = systemPrefix + "sam_stagereq"
+
+	fsBlocksize = 4096
 )
 
 const (
@@ -179,18 +197,20 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		return nil, err
 	}
 
-	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+	objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
 
 	// check all parts ok
 	last := len(parts) - 1
 	partsize := int64(0)
 	var totalsize int64
-	for i, p := range parts {
-		if p.PartNumber == nil {
+	for i, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
-		fi, err := os.Lstat(partPath)
+
+		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
+		fullPartPath := filepath.Join(bucket, partObjPath)
+		fi, err := os.Lstat(fullPartPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
@@ -198,23 +218,25 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		if i == 0 {
 			partsize = fi.Size()
 		}
+
+		// partsize must be a multiple of the filesystem blocksize
+		// except for last part
+		if i < last && partsize%fsBlocksize != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
 		totalsize += fi.Size()
 		// all parts except the last need to be the same size
 		if i < last && partsize != fi.Size() {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
-		// non-last part sizes need to be multiples of 4k for move blocks
-		// TODO: fallback to no move blocks if not 4k aligned?
-		if i == 0 && i < last && fi.Size()%4096 != 0 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
 
-		b, err := xattr.Get(partPath, "user.etag")
+		b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
 		etag := string(b)
 		if err != nil {
 			etag = ""
 		}
-		if etag != *parts[i].ETag {
+		if parts[i].ETag == nil || etag != *parts[i].ETag {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 	}
@@ -230,59 +252,106 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 	}
 	defer f.cleanup()
 
-	for _, p := range parts {
-		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
+	for _, part := range parts {
+		if part.PartNumber == nil || *part.PartNumber < 1 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
+		fullPartPath := filepath.Join(bucket, partObjPath)
+		pf, err := os.Open(fullPartPath)
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
+			return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = moveData(pf, f.f)
+		err = moveData(pf, f.File())
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
+			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
 		}
 	}
 
 	userMetaData := make(map[string]string)
 	upiddir := filepath.Join(objdir, uploadID)
-	loadUserMetaData(upiddir, userMetaData)
+	cType, _ := s.loadUserMetaData(bucket, upiddir, userMetaData)
 
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
 		uid, gid, doChown := s.getChownIDs(acct)
-		err = backend.MkdirAll(dir, uid, gid, doChown)
+		err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
 		if err != nil {
 			return nil, err
 		}
 	}
-	err = f.link()
-	if err != nil {
-		return nil, fmt.Errorf("link object in namespace: %w", err)
-	}
 
 	for k, v := range userMetaData {
-		err = xattr.Set(objname, "user."+k, []byte(v))
+		err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
 		if err != nil {
-			// cleanup object if returning error
-			os.Remove(objname)
 			return nil, fmt.Errorf("set user attr %q: %w", k, err)
 		}
 	}
 
+	// load and set tagging
+	tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object tagging: %w", err)
+	}
+	if err == nil {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
+		if err != nil {
+			return nil, fmt.Errorf("set object tagging: %w", err)
+		}
+	}
+
+	// set content-type
+	if cType != "" {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
+		if err != nil {
+			return nil, fmt.Errorf("set object content type: %w", err)
+		}
+	}
+
+	// load and set legal hold
+	lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object legal hold: %w", err)
+	}
+	if err == nil {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
+		if err != nil {
+			return nil, fmt.Errorf("set object legal hold: %w", err)
+		}
+	}
+
+	// load and set retention
+	ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
+	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+		return nil, fmt.Errorf("get object retention: %w", err)
+	}
+	if err == nil {
+		err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
+		if err != nil {
+			return nil, fmt.Errorf("set object retention: %w", err)
+		}
+	}
+
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
-	err = xattr.Set(objname, "user.etag", []byte(s3MD5))
+	err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
 	if err != nil {
-		// cleanup object if returning error
-		os.Remove(objname)
 		return nil, fmt.Errorf("set etag attr: %w", err)
 	}
 
+	err = f.link()
+	if err != nil {
+		return nil, fmt.Errorf("link object in namespace: %w", err)
+	}
+
 	// cleanup tmp dirs
 	os.RemoveAll(upiddir)
 	// use Remove for objdir in case there are still other uploads
@@ -310,61 +379,107 @@ func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte
 	return sum, nil
 }
 
-func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
-	ents, err := xattr.List(path)
+// fll out the user metadata map with the metadata for the object
+// and return the content type and encoding
+func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
+	ents, err := s.meta.ListAttributes(bucket, object)
 	if err != nil || len(ents) == 0 {
-		return
+		return "", ""
 	}
 	for _, e := range ents {
 		if !isValidMeta(e) {
 			continue
 		}
-		b, err := xattr.Get(path, e)
-		if err == errNoData {
-			m[strings.TrimPrefix(e, "user.")] = ""
-			continue
-		}
+		b, err := s.meta.RetrieveAttribute(nil, bucket, object, e)
 		if err != nil {
 			continue
 		}
-		m[strings.TrimPrefix(e, "user.")] = string(b)
+		if b == nil {
+			m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = ""
+			continue
+		}
+		m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
 	}
 
-	b, err := xattr.Get(path, "user.content-type")
+	var contentType, contentEncoding string
+	b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
 	contentType = string(b)
-	if err != nil {
-		contentType = ""
-	}
 	if contentType != "" {
-		m["content-type"] = contentType
+		m[contentTypeHdr] = contentType
 	}
 
-	b, err = xattr.Get(path, "user.content-encoding")
+	b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
 	contentEncoding = string(b)
-	if err != nil {
-		contentEncoding = ""
-	}
 	if contentEncoding != "" {
-		m["content-encoding"] = contentEncoding
+		m[contentEncHdr] = contentEncoding
 	}
 
-	return
+	return contentType, contentEncoding
 }
 
 func isValidMeta(val string) bool {
-	if strings.HasPrefix(val, "user.X-Amz-Meta") {
+	if strings.HasPrefix(val, metaHdr) {
 		return true
 	}
-	if strings.EqualFold(val, "user.Expires") {
+	if strings.EqualFold(val, "Expires") {
 		return true
 	}
 	return false
 }
 
-func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	if input.Key == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
 	bucket := *input.Bucket
 	object := *input.Key
 
+	if input.PartNumber != nil {
+		uploadId, sum, err := s.retrieveUploadId(bucket, object)
+		if err != nil {
+			return nil, err
+		}
+
+		ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read parts: %w", err)
+		}
+
+		partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
+
+		part, err := os.Stat(filepath.Join(bucket, partPath))
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		if errors.Is(err, syscall.ENAMETOOLONG) {
+			return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat part: %w", err)
+		}
+
+		b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
+		etag := string(b)
+		if err != nil {
+			etag = ""
+		}
+		partsCount := int32(len(ents))
+		size := part.Size()
+
+		return &s3.HeadObjectOutput{
+			LastModified:  backend.GetTimePtr(part.ModTime()),
+			ETag:          &etag,
+			PartsCount:    &partsCount,
+			ContentLength: &size,
+		}, nil
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -374,18 +489,30 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 	}
 
 	objPath := filepath.Join(bucket, object)
+
 	fi, err := os.Stat(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
+	if errors.Is(err, syscall.ENAMETOOLONG) {
+		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
+	if strings.HasSuffix(object, "/") && !fi.IsDir() {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
 
 	userMetaData := make(map[string]string)
-	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := xattr.Get(objPath, etagkey)
+	if fi.IsDir() {
+		// this is the media type for directories in AWS and Nextcloud
+		contentType = "application/x-directory"
+	}
+
+	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -424,19 +551,55 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 
 	contentLength := fi.Size()
 
+	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
+	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
+	if err == nil {
+		if *status {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
+		} else {
+			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
+		}
+	}
+
+	var objectLockMode types.ObjectLockMode
+	var objectLockRetainUntilDate *time.Time
+	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
+	if err == nil {
+		var config types.ObjectLockRetention
+		if err := json.Unmarshal(retention, &config); err == nil {
+			objectLockMode = types.ObjectLockMode(config.Mode)
+			objectLockRetainUntilDate = config.RetainUntilDate
+		}
+	}
+
 	return &s3.HeadObjectOutput{
-		ContentLength:   &contentLength,
-		ContentType:     &contentType,
-		ContentEncoding: &contentEncoding,
-		ETag:            &etag,
-		LastModified:    backend.GetTimePtr(fi.ModTime()),
-		Metadata:        userMetaData,
-		StorageClass:    stclass,
-		Restore:         &requestOngoing,
+		ContentLength:             &contentLength,
+		ContentType:               &contentType,
+		ContentEncoding:           &contentEncoding,
+		ETag:                      &etag,
+		LastModified:              backend.GetTimePtr(fi.ModTime()),
+		Metadata:                  userMetaData,
+		StorageClass:              stclass,
+		Restore:                   &requestOngoing,
+		ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
+		ObjectLockMode:            objectLockMode,
+		ObjectLockRetainUntilDate: objectLockRetainUntilDate,
 	}, nil
 }
 
-func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
+	sum := sha256.Sum256([]byte(object))
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	entries, err := os.ReadDir(objdir)
+	if err != nil || len(entries) == 0 {
+		return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
+	return entries[0].Name(), sum, nil
+}
+
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	acceptRange := *input.Range
@@ -450,15 +613,23 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 	}
 
 	objPath := filepath.Join(bucket, object)
+
 	fi, err := os.Stat(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
+	if errors.Is(err, syscall.ENAMETOOLONG) {
+		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
 
-	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	if strings.HasSuffix(object, "/") && !fi.IsDir() {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
+	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
 	if err != nil {
 		return nil, err
 	}
@@ -505,19 +676,14 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 	if err != nil {
 		return nil, fmt.Errorf("open object: %w", err)
 	}
-	defer f.Close()
 
 	rdr := io.NewSectionReader(f, startOffset, length)
-	_, err = io.Copy(writer, rdr)
-	if err != nil {
-		return nil, fmt.Errorf("copy data: %w", err)
-	}
 
 	userMetaData := make(map[string]string)
 
-	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
 
-	b, err := xattr.Get(objPath, etagkey)
+	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -541,6 +707,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
 		ContentRange:    &contentRange,
+		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
 	}, nil
 }
 
@@ -565,9 +732,9 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
 	return tags, nil
 }
 
-func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
 	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
 	bucket := *input.Bucket
 	prefix := ""
@@ -589,20 +756,20 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err)
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
+	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
 		s.fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
-		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+		return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
-	return &s3.ListObjectsOutput{
+	return s3response.ListObjectsResult{
 		CommonPrefixes: results.CommonPrefixes,
 		Contents:       results.Objects,
 		Delimiter:      &delim,
@@ -615,9 +782,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
 	}, nil
 }
 
-func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
 	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
 	bucket := *input.Bucket
 	prefix := ""
@@ -639,20 +806,20 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
 	if err != nil {
-		return nil, fmt.Errorf("stat bucket: %w", err)
+		return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err)
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
+	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys),
 		s.fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
-		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+		return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
-	return &s3.ListObjectsV2Output{
+	return s3response.ListObjectsV2Result{
 		CommonPrefixes:        results.CommonPrefixes,
 		Contents:              results.Objects,
 		Delimiter:             &delim,
@@ -666,53 +833,58 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
 }
 
 func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
-	return func(path string, d fs.DirEntry) (types.Object, error) {
+	return func(path string, d fs.DirEntry) (s3response.Object, error) {
 		objPath := filepath.Join(bucket, path)
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			etagBytes, err := xattr.Get(objPath, etagkey)
-			if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
-				return types.Object{}, backend.ErrSkipObj
+			etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
+			if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
+				return s3response.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return types.Object{}, fmt.Errorf("get etag: %w", err)
+				return s3response.Object{}, fmt.Errorf("get etag: %w", err)
 			}
 			etag := string(etagBytes)
 
 			fi, err := d.Info()
 			if errors.Is(err, fs.ErrNotExist) {
-				return types.Object{}, backend.ErrSkipObj
+				return s3response.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+				return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 			}
 
 			key := path + "/"
+			mtime := fi.ModTime()
 
-			return types.Object{
+			return s3response.Object{
 				ETag:         &etag,
 				Key:          &key,
-				LastModified: backend.GetTimePtr(fi.ModTime()),
+				LastModified: &mtime,
+				StorageClass: types.ObjectStorageClassStandard,
 			}, nil
 		}
 
 		// file object, get object info and fill out object data
-		etagBytes, err := xattr.Get(objPath, etagkey)
+		b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
-			return types.Object{}, backend.ErrSkipObj
+			return s3response.Object{}, backend.ErrSkipObj
 		}
-		if err != nil && !isNoAttr(err) {
-			return types.Object{}, fmt.Errorf("get etag: %w", err)
+		if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+			return s3response.Object{}, fmt.Errorf("get etag: %w", err)
 		}
-		etag := string(etagBytes)
+		// note: meta.ErrNoSuchKey will return etagBytes = []byte{}
+		// so this will just set etag to "" if its not already set
+
+		etag := string(b)
 
 		fi, err := d.Info()
 		if errors.Is(err, fs.ErrNotExist) {
-			return types.Object{}, backend.ErrSkipObj
+			return s3response.Object{}, backend.ErrSkipObj
 		}
 		if err != nil {
-			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
 
 		sc := types.ObjectStorageClassStandard
@@ -721,10 +893,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 			// If so, we will return the InvalidObjectState error.
 			st, err := statMore(objPath)
 			if errors.Is(err, fs.ErrNotExist) {
-				return types.Object{}, backend.ErrSkipObj
+				return s3response.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return types.Object{}, fmt.Errorf("stat more: %w", err)
+				return s3response.Object{}, fmt.Errorf("stat more: %w", err)
 			}
 			if st.Offline_blocks != 0 {
 				sc = types.ObjectStorageClassGlacier
@@ -732,11 +904,12 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		}
 
 		size := fi.Size()
+		mtime := fi.ModTime()
 
-		return types.Object{
+		return s3response.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: backend.GetTimePtr(fi.ModTime()),
+			LastModified: &mtime,
 			Size:         &size,
 			StorageClass: sc,
 		}, nil
@@ -819,15 +992,9 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 }
 
 func isNoAttr(err error) bool {
-	if err == nil {
-		return false
-	}
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == errNoData {
-		return true
-	}
 	return false
 }

backend/scoutfs/scoutfs_compat.go

@@ -23,20 +23,24 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
-	"syscall"
 
 	"golang.org/x/sys/unix"
 
 	"github.com/versity/scoutfs-go"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
 func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
-	p, err := posix.New(rootdir, posix.PosixOpts{
-		ChownUID: opts.ChownUID,
-		ChownGID: opts.ChownGID,
+	metastore := meta.XattrMeta{}
+
+	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
+		ChownUID:    opts.ChownUID,
+		ChownGID:    opts.ChownGID,
+		BucketLinks: opts.BucketLinks,
+		NewDirPerm:  opts.NewDirPerm,
 	})
 	if err != nil {
 		return nil, err
@@ -48,11 +52,14 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:    p,
-		rootfd:   f,
-		rootdir:  rootdir,
-		chownuid: opts.ChownUID,
-		chowngid: opts.ChownGID,
+		Posix:       p,
+		rootfd:      f,
+		rootdir:     rootdir,
+		meta:        metastore,
+		chownuid:    opts.ChownUID,
+		chowngid:    opts.ChownGID,
+		glaciermode: opts.GlacierMode,
+		newDirPerm:  opts.NewDirPerm,
 	}, nil
 }
 
@@ -63,13 +70,13 @@ type tmpfile struct {
 	bucket     string
 	objname    string
 	size       int64
-	needsChown bool
 	uid        int
 	gid        int
+	newDirPerm fs.FileMode
+	needsChown bool
 }
 
 var (
-	// TODO: make this configurable
 	defaultFilePerm uint32 = 0644
 )
 
@@ -97,11 +104,7 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 		needsChown: doChown,
 		uid:        uid,
 		gid:        gid,
-	}
-
-	// falloc is best effort, its fine if this fails
-	if size > 0 {
-		tmp.falloc()
+		newDirPerm: s.newDirPerm,
 	}
 
 	if doChown {
@@ -114,14 +117,6 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
 	return tmp, nil
 }
 
-func (tmp *tmpfile) falloc() error {
-	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
-	if err != nil {
-		return fmt.Errorf("fallocate: %v", err)
-	}
-	return nil
-}
-
 func (tmp *tmpfile) link() error {
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
@@ -137,7 +132,7 @@ func (tmp *tmpfile) link() error {
 
 	dir := filepath.Dir(objPath)
 
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
 	if err != nil {
 		return fmt.Errorf("make parent dir: %w", err)
 	}
@@ -182,6 +177,10 @@ func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}
+
 func moveData(from *os.File, to *os.File) error {
 	return scoutfs.MoveData(from, to)
 }

backend/scoutfs/scoutfs_incompat.go

@@ -28,9 +28,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
-type tmpfile struct {
-	f *os.File
-}
+type tmpfile struct{}
 
 var (
 	errNotSupported = errors.New("not supported")
@@ -56,6 +54,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
+func (tmp *tmpfile) File() *os.File {
+	return nil
+}
+
 func moveData(_, _ *os.File) error {
 	return errNotSupported
 }

backend/scoutfs/with_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build !freebsd && !openbsd && !netbsd
-// +build !freebsd,!openbsd,!netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENODATA
-)

backend/scoutfs/without_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build freebsd || openbsd || netbsd
-// +build freebsd openbsd netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENOATTR
-)

backend/walk.go

@@ -15,32 +15,33 @@
 package backend
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/fs"
-	"os"
 	"sort"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3response"
 )
 
 type WalkResults struct {
-	CommonPrefixes []types.CommonPrefix
-	Objects        []types.Object
-	Truncated      bool
 	NextMarker     string
+	CommonPrefixes []types.CommonPrefix
+	Objects        []s3response.Object
+	Truncated      bool
 }
 
-type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
+type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
 
 var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
-	var objects []types.Object
+	var objects []s3response.Object
 
 	var pastMarker bool
 	if marker == "" {
@@ -51,10 +52,21 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 	var newMarker string
 	var truncated bool
 
-	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+	root := "."
+	if strings.Contains(prefix, "/") {
+		idx := strings.LastIndex(prefix, "/")
+		if idx > 0 {
+			root = prefix[:idx]
+		}
+	}
+
+	err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
 		// Ignore the root directory
 		if path == "." {
 			return nil
@@ -71,6 +83,9 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			return fs.SkipAll
 		}
 
+		// After this point, return skipflag instead of nil
+		// so we can skip a directory without an early return
+		var skipflag error
 		if d.IsDir() {
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
@@ -80,44 +95,57 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			// building to match. So only skip if path isn't a prefix of prefix
 			// and prefix isn't a prefix of path.
 			if prefix != "" &&
-				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
-				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
+				!strings.HasPrefix(path+"/", prefix) &&
+				!strings.HasPrefix(prefix, path+"/") {
 				return fs.SkipDir
 			}
 
-			// TODO: can we do better here rather than a second readdir
-			// per directory?
-			ents, err := fs.ReadDir(fileSystem, path)
-			if err != nil {
-				return fmt.Errorf("readdir %q: %w", path, err)
-			}
-			if len(ents) == 0 {
-				dirobj, err := getObj(path, d)
-				if err == ErrSkipObj {
-					return nil
-				}
+			// Don't recurse into subdirectories which contain the delimiter
+			// after reaching the prefix
+			if delimiter != "" &&
+				strings.HasPrefix(path+"/", prefix) &&
+				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
+				skipflag = fs.SkipDir
+			} else {
+				// TODO: can we do better here rather than a second readdir
+				// per directory?
+				ents, err := fs.ReadDir(fileSystem, path)
 				if err != nil {
-					return fmt.Errorf("directory to object %q: %w", path, err)
+					return fmt.Errorf("readdir %q: %w", path, err)
 				}
-				objects = append(objects, dirobj)
-			}
+				if len(ents) == 0 && delimiter == "" {
+					dirobj, err := getObj(path+"/", d)
+					if err == ErrSkipObj {
+						return skipflag
+					}
+					if err != nil {
+						return fmt.Errorf("directory to object %q: %w", path, err)
+					}
+					objects = append(objects, dirobj)
 
-			return nil
+					return skipflag
+				}
+
+				if len(ents) != 0 {
+					return skipflag
+				}
+			}
+			path += "/"
 		}
 
 		if !pastMarker {
 			if path == marker {
 				pastMarker = true
-				return nil
+				return skipflag
 			}
 			if path < marker {
-				return nil
+				return skipflag
 			}
 		}
 
 		// If object doesn't have prefix, don't include in results.
 		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return nil
+			return skipflag
 		}
 
 		if delimiter == "" {
@@ -125,7 +153,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			// prefix are included in results
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return nil
+				return skipflag
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -136,7 +164,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 				pastMax = true
 			}
 
-			return nil
+			return skipflag
 		}
 
 		// Since delimiter is specified, we only want results that
@@ -165,7 +193,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		if !found {
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return nil
+				return skipflag
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -174,20 +202,38 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 			if (len(objects) + len(cpmap)) == int(max) {
 				pastMax = true
 			}
-			return nil
+			return skipflag
 		}
 
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
-		// delimiter at the end.
-		cpmap[prefix+before+delimiter] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
-			pastMax = true
+		// delimiter at the end when we add to the map.
+		cprefNoDelim := prefix + before
+		cpref := prefix + before + delimiter
+		if cpref == marker {
+			pastMarker = true
+			return skipflag
 		}
 
-		return nil
+		if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
+			// skip common prefixes that are before the marker
+			return skipflag
+		}
+
+		cpmap[cpref] = struct{}{}
+		if (len(objects) + len(cpmap)) == int(max) {
+			newMarker = cpref
+			truncated = true
+			return fs.SkipAll
+		}
+
+		return skipflag
 	})
 	if err != nil {
+		// suppress file not found caused by user's prefix
+		if errors.Is(err, fs.ErrNotExist) {
+			return WalkResults{}, nil
+		}
 		return WalkResults{}, err
 	}
 
@@ -220,3 +266,213 @@ func contains(a string, strs []string) bool {
 	}
 	return false
 }
+
+type WalkVersioningResults struct {
+	NextMarker          string
+	NextVersionIdMarker string
+	CommonPrefixes      []types.CommonPrefix
+	ObjectVersions      []types.ObjectVersion
+	DelMarkers          []types.DeleteMarkerEntry
+	Truncated           bool
+}
+
+type ObjVersionFuncResult struct {
+	NextVersionIdMarker string
+	ObjectVersions      []types.ObjectVersion
+	DelMarkers          []types.DeleteMarkerEntry
+	Truncated           bool
+}
+
+type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
+
+// WalkVersions walks the supplied fs.FS and returns results compatible with
+// ListObjectVersions action response
+func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
+	cpmap := make(map[string]struct{})
+	var objects []types.ObjectVersion
+	var delMarkers []types.DeleteMarkerEntry
+
+	var pastMarker bool
+	if keyMarker == "" {
+		pastMarker = true
+	}
+	var nextMarker string
+	var nextVersionIdMarker string
+	var truncated bool
+
+	pastVersionIdMarker := versionIdMarker == ""
+
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
+
+		if !pastMarker {
+			if path == keyMarker {
+				pastMarker = true
+			}
+			if path < keyMarker {
+				return nil
+			}
+		}
+
+		if d.IsDir() {
+			// If prefix is defined and the directory does not match prefix,
+			// do not descend into the directory because nothing will
+			// match this prefix. Make sure to append the / at the end of
+			// directories since this is implied as a directory path name.
+			// If path is a prefix of prefix, then path could still be
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
+			if prefix != "" &&
+				!strings.HasPrefix(path+"/", prefix) &&
+				!strings.HasPrefix(prefix, path+"/") {
+				return fs.SkipDir
+			}
+
+			// Don't recurse into subdirectories when listing with delimiter.
+			if delimiter == "/" &&
+				prefix != path+"/" &&
+				strings.HasPrefix(path+"/", prefix) {
+				cpmap[path+"/"] = struct{}{}
+				return fs.SkipDir
+			}
+
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("directory to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+
+			return nil
+		}
+
+		// If object doesn't have prefix, don't include in results.
+		if prefix != "" && !strings.HasPrefix(path, prefix) {
+			return nil
+		}
+
+		if delimiter == "" {
+			// If no delimiter specified, then all files with matching
+			// prefix are included in results
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+
+			return nil
+		}
+
+		// Since delimiter is specified, we only want results that
+		// do not contain the delimiter beyond the prefix.  If the
+		// delimiter exists past the prefix, then the substring
+		// between the prefix and delimiter is part of common prefixes.
+		//
+		// For example:
+		// prefix = A/
+		// delimiter = /
+		// and objects:
+		// A/file
+		// A/B/file
+		// B/C
+		// would return:
+		// objects: A/file
+		// common prefix: A/B/
+		//
+		// Note: No objects are included past the common prefix since
+		// these are all rolled up into the common prefix.
+		// Note: The delimiter can be anything, so we have to operate on
+		// the full path without any assumptions on posix directory hierarchy
+		// here.  Usually the delimiter will be "/", but thats not required.
+		suffix := strings.TrimPrefix(path, prefix)
+		before, _, found := strings.Cut(suffix, delimiter)
+		if !found {
+			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, res.ObjectVersions...)
+			delMarkers = append(delMarkers, res.DelMarkers...)
+
+			if res.Truncated {
+				truncated = true
+				nextMarker = path
+				nextVersionIdMarker = res.NextVersionIdMarker
+				return fs.SkipAll
+			}
+			return nil
+		}
+
+		// Common prefixes are a set, so should not have duplicates.
+		// These are abstractly a "directory", so need to include the
+		// delimiter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
+		if (len(objects) + len(cpmap)) == int(max) {
+			nextMarker = path
+			truncated = true
+
+			return fs.SkipAll
+		}
+
+		return nil
+	})
+	if err != nil {
+		return WalkVersioningResults{}, err
+	}
+
+	var commonPrefixStrings []string
+	for k := range cpmap {
+		commonPrefixStrings = append(commonPrefixStrings, k)
+	}
+	sort.Strings(commonPrefixStrings)
+	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
+	for _, cp := range commonPrefixStrings {
+		pfx := cp
+		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
+			Prefix: &pfx,
+		})
+	}
+
+	return WalkVersioningResults{
+		CommonPrefixes:      commonPrefixes,
+		ObjectVersions:      objects,
+		DelMarkers:          delMarkers,
+		Truncated:           truncated,
+		NextMarker:          nextMarker,
+		NextVersionIdMarker: nextVersionIdMarker,
+	}, nil
+}

backend/walk_test.go

@@ -15,36 +15,50 @@
 package backend_test
 
 import (
+	"context"
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
 	"io/fs"
+	"sync"
 	"testing"
 	"testing/fstest"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3response"
 )
 
 type walkTest struct {
-	fsys     fs.FS
-	expected backend.WalkResults
-	getobj   backend.GetObjFunc
+	fsys   fs.FS
+	getobj backend.GetObjFunc
+	cases  []testcase
 }
 
-func getObj(path string, d fs.DirEntry) (types.Object, error) {
+type testcase struct {
+	name      string
+	prefix    string
+	delimiter string
+	marker    string
+	expected  backend.WalkResults
+	maxObjs   int32
+}
+
+func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
 	if d.IsDir() {
 		etag := getMD5(path)
 
 		fi, err := d.Info()
 		if err != nil {
-			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
+		mtime := fi.ModTime()
 
-		return types.Object{
+		return s3response.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: backend.GetTimePtr(fi.ModTime()),
+			LastModified: &mtime,
 		}, nil
 	}
 
@@ -52,15 +66,16 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
 
 	fi, err := d.Info()
 	if err != nil {
-		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
 	}
 
 	size := fi.Size()
+	mtime := fi.ModTime()
 
-	return types.Object{
+	return s3response.Object{
 		ETag:         &etag,
 		Key:          &path,
-		LastModified: backend.GetTimePtr(fi.ModTime()),
+		LastModified: &mtime,
 		Size:         &size,
 	}, nil
 }
@@ -82,50 +97,154 @@ func TestWalk(t *testing.T) {
 				"photos/2006/February/sample3.jpg": {},
 				"photos/2006/February/sample4.jpg": {},
 			},
-			expected: backend.WalkResults{
-				CommonPrefixes: []types.CommonPrefix{{
-					Prefix: backend.GetStringPtr("photos/"),
-				}},
-				Objects: []types.Object{{
-					Key: backend.GetStringPtr("sample.jpg"),
-				}},
-			},
 			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "aws example",
+					delimiter: "/",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photos/"),
+						}},
+						Objects: []s3response.Object{{
+							Key: backend.GetPtrFromString("sample.jpg"),
+						}},
+					},
+				},
+			},
 		},
 		{
 			// test case single dir/single file
 			fsys: fstest.MapFS{
 				"test/file": {},
 			},
-			expected: backend.WalkResults{
-				CommonPrefixes: []types.CommonPrefix{{
-					Prefix: backend.GetStringPtr("test/"),
-				}},
-				Objects: []types.Object{},
+			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "single dir single file",
+					delimiter: "/",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("test/"),
+						}},
+						Objects: []s3response.Object{},
+					},
+				},
+			},
+		},
+		{
+			// non-standard delimiter
+			fsys: fstest.MapFS{
+				"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
+				"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
+				"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
 			},
 			getobj: getObj,
+			cases: []testcase{
+				{
+					name:      "different delimiter 1",
+					delimiter: "|",
+					maxObjs:   1000,
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 2",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 3",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 4",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 5",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|6/Januar|",
+					expected: backend.WalkResults{
+						CommonPrefixes: []types.CommonPrefix{{
+							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
+						}},
+					},
+				},
+				{
+					name:      "different delimiter 6",
+					delimiter: "|",
+					maxObjs:   1000,
+					prefix:    "photo|s/200|6/Januar|y/sampl|",
+					expected: backend.WalkResults{
+						Objects: []s3response.Object{
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
+							},
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
+							},
+							{
+								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
+							},
+						},
+					},
+				},
+			},
 		},
 	}
 
 	for _, tt := range tests {
-		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
-		if err != nil {
-			t.Fatalf("walk: %v", err)
-		}
+		for _, tc := range tt.cases {
+			res, err := backend.Walk(context.Background(),
+				tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
+				tt.getobj, []string{})
+			if err != nil {
+				t.Errorf("tc.name: walk: %v", err)
+			}
 
-		compareResults(res, tt.expected, t)
+			compareResults(tc.name, res, tc.expected, t)
+		}
 	}
 }
 
-func compareResults(got, wanted backend.WalkResults, t *testing.T) {
+func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
 	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
-		t.Errorf("unexpected common prefix, got %v wanted %v",
+		t.Errorf("%v: unexpected common prefix, got %v wanted %v",
+			name,
 			printCommonPrefixes(got.CommonPrefixes),
 			printCommonPrefixes(wanted.CommonPrefixes))
 	}
 
 	if !compareObjects(got.Objects, wanted.Objects) {
-		t.Errorf("unexpected object, got %v wanted %v",
+		t.Errorf("%v: unexpected object, got %v wanted %v",
+			name,
 			printObjects(got.Objects),
 			printObjects(wanted.Objects))
 	}
@@ -168,7 +287,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
 	return res + "]"
 }
 
-func compareObjects(a, b []types.Object) bool {
+func compareObjects(a, b []s3response.Object) bool {
 	if len(a) == 0 && len(b) == 0 {
 		return true
 	}
@@ -184,7 +303,7 @@ func compareObjects(a, b []types.Object) bool {
 	return false
 }
 
-func containsObject(c types.Object, list []types.Object) bool {
+func containsObject(c s3response.Object, list []s3response.Object) bool {
 	for _, cp := range list {
 		if *c.Key == *cp.Key {
 			return true
@@ -193,14 +312,67 @@ func containsObject(c types.Object, list []types.Object) bool {
 	return false
 }
 
-func printObjects(list []types.Object) string {
+func printObjects(list []s3response.Object) string {
 	res := "["
 	for _, cp := range list {
-		if res == "[" {
-			res = res + *cp.Key
+		var key string
+		if cp.Key == nil {
+			key = "<nil>"
 		} else {
-			res = res + ", " + *cp.Key
+			key = *cp.Key
+		}
+		if res == "[" {
+			res = res + key
+		} else {
+			res = res + ", " + key
 		}
 	}
 	return res + "]"
 }
+
+type slowFS struct {
+	fstest.MapFS
+}
+
+const (
+	readDirPause = 100 * time.Millisecond
+
+	// walkTimeOut should be less than the tree traversal time
+	// which is the readdirPause time * the number of directories
+	walkTimeOut = 500 * time.Millisecond
+)
+
+func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
+	time.Sleep(readDirPause)
+	return s.MapFS.ReadDir(name)
+}
+
+func TestWalkStop(t *testing.T) {
+	s := &slowFS{MapFS: fstest.MapFS{
+		"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
+	}}
+
+	ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
+	defer cancel()
+
+	var err error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, err = backend.Walk(ctx, s, "", "/", "", 1000,
+			func(path string, d fs.DirEntry) (s3response.Object, error) {
+				return s3response.Object{}, nil
+			}, []string{})
+	}()
+
+	select {
+	case <-time.After(1 * time.Second):
+		t.Fatalf("walk is not terminated in time")
+	case <-ctx.Done():
+	}
+	wg.Wait()
+	if err != ctx.Err() {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}

cmd/versitygw/admin.go

@@ -19,7 +19,7 @@ import (
 	"crypto/sha256"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,6 +29,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3response"
@@ -37,6 +38,7 @@ import (
 var (
 	adminAccess   string
 	adminSecret   string
+	adminRegion   string
 	adminEndpoint string
 	allowInsecure bool
 )
@@ -80,10 +82,33 @@ func adminCommand() *cli.Command {
 						Usage:   "groupID for the new user",
 						Aliases: []string{"gi"},
 					},
+				},
+			},
+			{
+				Name:   "update-user",
+				Usage:  "Updates a user account",
+				Action: updateUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "user access key id to be updated",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:    "secret",
+						Usage:   "secret access key for the new user",
+						Aliases: []string{"s"},
+					},
 					&cli.IntFlag{
-						Name:    "project-id",
-						Usage:   "projectID for the new user",
-						Aliases: []string{"pi"},
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
 					},
 				},
 			},
@@ -148,6 +173,14 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminSecret,
 			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "admin s3 region string",
+				EnvVars:     []string{"ADMIN_REGION"},
+				Value:       "us-east-1",
+				Destination: &adminRegion,
+				Aliases:     []string{"r"},
+			},
 			&cli.StringFlag{
 				Name:        "endpoint-url",
 				Usage:       "admin apis endpoint url",
@@ -176,41 +209,40 @@ func initHTTPClient() *http.Client {
 
 func createUser(ctx *cli.Context) error {
 	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
-	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
+	userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
 	if access == "" || secret == "" {
-		return fmt.Errorf("invalid input parameters for the new user")
+		return fmt.Errorf("invalid input parameters for the new user access/secret keys")
 	}
 	if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
 		return fmt.Errorf("invalid input parameter for role: %v", role)
 	}
 
 	acc := auth.Account{
-		Access:    access,
-		Secret:    secret,
-		Role:      auth.Role(role),
-		UserID:    userID,
-		GroupID:   groupID,
-		ProjectID: projectID,
+		Access:  access,
+		Secret:  secret,
+		Role:    auth.Role(role),
+		UserID:  userID,
+		GroupID: groupID,
 	}
 
-	accJson, err := json.Marshal(acc)
+	accxml, err := xml.Marshal(acc)
 	if err != nil {
 		return fmt.Errorf("failed to parse user data: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(accJson)
+	hashedPayload := sha256.Sum256(accxml)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -229,18 +261,16 @@ func createUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
-
 	return nil
 }
 
 func deleteUser(ctx *cli.Context) error {
 	access := ctx.String("access")
 	if access == "" {
-		return fmt.Errorf("invalid input parameter for the new user")
+		return fmt.Errorf("invalid input parameter for the user access key")
 	}
 
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
@@ -255,7 +285,7 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -274,10 +304,63 @@ func deleteUser(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	fmt.Printf("%s\n", body)
+	return nil
+}
+
+func updateUser(ctx *cli.Context) error {
+	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
+	props := auth.MutableProps{}
+	if ctx.IsSet("secret") {
+		props.Secret = &secret
+	}
+	if ctx.IsSet("user-id") {
+		props.UserID = &userId
+	}
+	if ctx.IsSet("group-id") {
+		props.GroupID = &groupId
+	}
+
+	propsxml, err := xml.Marshal(props)
+	if err != nil {
+		return fmt.Errorf("failed to parse user attributes: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(propsxml)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
 
 	return nil
 }
@@ -295,7 +378,7 @@ func listUsers(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -314,15 +397,15 @@ func listUsers(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var accs []auth.Account
-	if err := json.Unmarshal(body, &accs); err != nil {
+	var accs auth.ListUserAccountsResult
+	if err := xml.Unmarshal(body, &accs); err != nil {
 		return err
 	}
 
-	printAcctTable(accs)
+	printAcctTable(accs.Accounts)
 
 	return nil
 }
@@ -339,10 +422,10 @@ const (
 func printAcctTable(accs []auth.Account) {
 	w := new(tabwriter.Writer)
 	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
-	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
-	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------")
 	for _, acc := range accs {
-		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
 	}
 	fmt.Fprintln(w)
 	w.Flush()
@@ -362,25 +445,27 @@ func changeBucketOwner(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := http.Client{}
+	client := initHTTPClient()
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
+	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
 
-	fmt.Println(string(body))
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
 
 	return nil
 }
@@ -410,7 +495,7 @@ func listBuckets(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -429,15 +514,26 @@ func listBuckets(ctx *cli.Context) error {
 	}
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s", body)
+		return parseApiError(body)
 	}
 
-	var buckets []s3response.Bucket
-	if err := json.Unmarshal(body, &buckets); err != nil {
+	var result s3response.ListBucketsResult
+	if err := xml.Unmarshal(body, &result); err != nil {
 		return err
 	}
 
-	printBuckets(buckets)
+	printBuckets(result.Buckets)
 
 	return nil
 }
+
+func parseApiError(body []byte) error {
+	var apiErr smithy.GenericAPIError
+	err := xml.Unmarshal(body, &apiErr)
+	if err != nil {
+		apiErr.Code = "InternalServerError"
+		apiErr.Message = err.Error()
+	}
+
+	return &apiErr
+}

cmd/versitygw/gateway_test.go

@@ -7,7 +7,9 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/tests/integration"
 )
@@ -56,7 +58,9 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir, posix.PosixOpts{})
+	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
+		NewDirPerm: 0755,
+	})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}
@@ -74,6 +78,9 @@ func initPosix(ctx context.Context) {
 		}
 		wg.Done()
 	}()
+
+	// wait for server to start
+	time.Sleep(1 * time.Second)
 }
 
 func TestIntegration(t *testing.T) {

cmd/versitygw/main.go

@@ -19,14 +19,17 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
+	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api"
 	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
@@ -34,33 +37,43 @@ import (
 )
 
 var (
-	port, admPort                          string
-	rootUserAccess                         string
-	rootUserSecret                         string
-	region                                 string
-	admCertFile, admKeyFile                string
-	certFile, keyFile                      string
-	kafkaURL, kafkaTopic, kafkaKey         string
-	natsURL, natsTopic                     string
-	eventWebhookURL                        string
-	eventConfigFilePath                    string
-	logWebhookURL                          string
-	accessLog                              string
-	healthPath                             string
-	debug                                  bool
-	pprof                                  string
-	quiet                                  bool
-	iamDir                                 string
-	ldapURL, ldapBindDN, ldapPassword      string
-	ldapQueryBase, ldapObjClasses          string
-	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
-	s3IamAccess, s3IamSecret               string
-	s3IamRegion, s3IamBucket               string
-	s3IamEndpoint                          string
-	s3IamSslNoVerify, s3IamDebug           bool
-	iamCacheDisable                        bool
-	iamCacheTTL                            int
-	iamCachePrune                          int
+	port, admPort                            string
+	rootUserAccess                           string
+	rootUserSecret                           string
+	region                                   string
+	admCertFile, admKeyFile                  string
+	certFile, keyFile                        string
+	kafkaURL, kafkaTopic, kafkaKey           string
+	natsURL, natsTopic                       string
+	eventWebhookURL                          string
+	eventConfigFilePath                      string
+	logWebhookURL, accessLog                 string
+	adminLogFile                             string
+	healthPath                               string
+	debug                                    bool
+	pprof                                    string
+	quiet                                    bool
+	readonly                                 bool
+	iamDir                                   string
+	ldapURL, ldapBindDN, ldapPassword        string
+	ldapQueryBase, ldapObjClasses            string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr   string
+	ldapUserIdAtr, ldapGroupIdAtr            string
+	vaultEndpointURL, vaultSecretStoragePath string
+	vaultMountPath, vaultRootToken           string
+	vaultRoleId, vaultRoleSecret             string
+	vaultServerCert, vaultClientCert         string
+	vaultClientCertKey                       string
+	s3IamAccess, s3IamSecret                 string
+	s3IamRegion, s3IamBucket                 string
+	s3IamEndpoint                            string
+	s3IamSslNoVerify, s3IamDebug             bool
+	iamCacheDisable                          bool
+	iamCacheTTL                              int
+	iamCachePrune                            int
+	metricsService                           string
+	statsdServers                            string
+	dogstatsServers                          string
 )
 
 var (
@@ -101,10 +114,13 @@ func main() {
 
 func initApp() *cli.App {
 	return &cli.App{
-		Name:  "versitygw",
-		Usage: "Start S3 gateway service with specified backend storage.",
-		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
-to access the supported backend storage as if it was a native S3 service.`,
+		Usage: "Versity S3 Gateway",
+		Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.
+VersityGW is an open-source project licensed under the Apache 2.0 License. The
+source code is hosted on GitHub at https://github.com/versity/versitygw, and
+documentation can be found in the GitHub wiki.`,
+		Copyright: "Copyright (c) 2023-2024 Versity Software",
 		Action: func(ctx *cli.Context) error {
 			return ctx.App.Command("help").Run(ctx)
 		},
@@ -212,6 +228,12 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
 			Destination: &accessLog,
 		},
+		&cli.StringFlag{
+			Name:        "admin-access-log",
+			Usage:       "enable admin server access logging to specified file",
+			EnvVars:     []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
+			Destination: &adminLogFile,
+		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
@@ -321,6 +343,72 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-user-id-atr",
+			Usage:       "ldap server user id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_USER_ID_ATR"},
+			Destination: &ldapUserIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-group-id-atr",
+			Usage:       "ldap server user group id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
+			Destination: &ldapGroupIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-endpoint-url",
+			Usage:       "vault server url",
+			EnvVars:     []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
+			Destination: &vaultEndpointURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-secret-storage-path",
+			Usage:       "vault server secret storage path",
+			EnvVars:     []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
+			Destination: &vaultSecretStoragePath,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-mount-path",
+			Usage:       "vault server mount path",
+			EnvVars:     []string{"VGW_IAM_VAULT_MOUNT_PATH"},
+			Destination: &vaultMountPath,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-root-token",
+			Usage:       "vault server root token",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
+			Destination: &vaultRootToken,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-role-id",
+			Usage:       "vault server user role id",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_ID"},
+			Destination: &vaultRoleId,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-role-secret",
+			Usage:       "vault server user role secret",
+			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_SECRET"},
+			Destination: &vaultRoleSecret,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-server_cert",
+			Usage:       "vault server TLS certificate",
+			EnvVars:     []string{"VGW_IAM_VAULT_SERVER_CERT"},
+			Destination: &vaultServerCert,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-client_cert",
+			Usage:       "vault client TLS certificate",
+			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT"},
+			Destination: &vaultClientCert,
+		},
+		&cli.StringFlag{
+			Name:        "iam-vault-client_cert_key",
+			Usage:       "vault client TLS certificate key",
+			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
+			Destination: &vaultClientCertKey,
+		},
 		&cli.StringFlag{
 			Name:        "s3-iam-access",
 			Usage:       "s3 IAM access key",
@@ -391,6 +479,33 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_HEALTH"},
 			Destination: &healthPath,
 		},
+		&cli.BoolFlag{
+			Name:        "readonly",
+			Usage:       "allow only read operations across all the gateway",
+			EnvVars:     []string{"VGW_READ_ONLY"},
+			Destination: &readonly,
+		},
+		&cli.StringFlag{
+			Name:        "metrics-service-name",
+			Usage:       "service name tag for metrics, hostname if blank",
+			EnvVars:     []string{"VGW_METRICS_SERVICE_NAME"},
+			Aliases:     []string{"msn"},
+			Destination: &metricsService,
+		},
+		&cli.StringFlag{
+			Name:        "metrics-statsd-servers",
+			Usage:       "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
+			EnvVars:     []string{"VGW_METRICS_STATSD_SERVERS"},
+			Aliases:     []string{"mss"},
+			Destination: &statsdServers,
+		},
+		&cli.StringFlag{
+			Name:        "metrics-dogstatsd-servers",
+			Usage:       "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
+			EnvVars:     []string{"VGW_METRICS_DOGSTATS_SERVERS"},
+			Aliases:     []string{"mds"},
+			Destination: &dogstatsServers,
+		},
 	}
 }
 
@@ -408,10 +523,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	app := fiber.New(fiber.Config{
-		AppName:           "versitygw",
-		ServerHeader:      "VERSITYGW",
-		StreamRequestBody: true,
-		DisableKeepalive:  true,
+		AppName:               "versitygw",
+		ServerHeader:          "VERSITYGW",
+		StreamRequestBody:     true,
+		DisableKeepalive:      true,
+		Network:               fiber.NetworkTCP,
+		DisableStartupMessage: true,
 	})
 
 	var opts []s3api.Option
@@ -442,10 +559,15 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if healthPath != "" {
 		opts = append(opts, s3api.WithHealth(healthPath))
 	}
+	if readonly {
+		opts = append(opts, s3api.WithReadOnly())
+	}
 
 	admApp := fiber.New(fiber.Config{
-		AppName:      "versitygw",
-		ServerHeader: "VERSITYGW",
+		AppName:               "versitygw",
+		ServerHeader:          "VERSITYGW",
+		Network:               fiber.NetworkTCP,
+		DisableStartupMessage: true,
 	})
 
 	var admOpts []s3api.AdminOpt
@@ -466,38 +588,64 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
-		Dir:                iamDir,
-		LDAPServerURL:      ldapURL,
-		LDAPBindDN:         ldapBindDN,
-		LDAPPassword:       ldapPassword,
-		LDAPQueryBase:      ldapQueryBase,
-		LDAPObjClasses:     ldapObjClasses,
-		LDAPAccessAtr:      ldapAccessAtr,
-		LDAPSecretAtr:      ldapSecAtr,
-		LDAPRoleAtr:        ldapRoleAtr,
-		S3Access:           s3IamAccess,
-		S3Secret:           s3IamSecret,
-		S3Region:           s3IamRegion,
-		S3Bucket:           s3IamBucket,
-		S3Endpoint:         s3IamEndpoint,
-		S3DisableSSlVerfiy: s3IamSslNoVerify,
-		S3Debug:            s3IamDebug,
-		CacheDisable:       iamCacheDisable,
-		CacheTTL:           iamCacheTTL,
-		CachePrune:         iamCachePrune,
+		RootAccount: auth.Account{
+			Access: rootUserAccess,
+			Secret: rootUserSecret,
+			Role:   auth.RoleAdmin,
+		},
+		Dir:                    iamDir,
+		LDAPServerURL:          ldapURL,
+		LDAPBindDN:             ldapBindDN,
+		LDAPPassword:           ldapPassword,
+		LDAPQueryBase:          ldapQueryBase,
+		LDAPObjClasses:         ldapObjClasses,
+		LDAPAccessAtr:          ldapAccessAtr,
+		LDAPSecretAtr:          ldapSecAtr,
+		LDAPRoleAtr:            ldapRoleAtr,
+		LDAPUserIdAtr:          ldapUserIdAtr,
+		LDAPGroupIdAtr:         ldapGroupIdAtr,
+		VaultEndpointURL:       vaultEndpointURL,
+		VaultSecretStoragePath: vaultSecretStoragePath,
+		VaultMountPath:         vaultMountPath,
+		VaultRootToken:         vaultRootToken,
+		VaultRoleId:            vaultRoleId,
+		VaultRoleSecret:        vaultRoleSecret,
+		VaultServerCert:        vaultServerCert,
+		VaultClientCert:        vaultClientCert,
+		VaultClientCertKey:     vaultClientCertKey,
+		S3Access:               s3IamAccess,
+		S3Secret:               s3IamSecret,
+		S3Region:               s3IamRegion,
+		S3Bucket:               s3IamBucket,
+		S3Endpoint:             s3IamEndpoint,
+		S3DisableSSlVerfiy:     s3IamSslNoVerify,
+		S3Debug:                s3IamDebug,
+		CacheDisable:           iamCacheDisable,
+		CacheTTL:               iamCacheTTL,
+		CachePrune:             iamCachePrune,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)
 	}
 
-	logger, err := s3log.InitLogger(&s3log.LogConfig{
-		LogFile:    accessLog,
-		WebhookURL: logWebhookURL,
+	loggers, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:      accessLog,
+		WebhookURL:   logWebhookURL,
+		AdminLogFile: adminLogFile,
 	})
 	if err != nil {
 		return fmt.Errorf("setup logger: %w", err)
 	}
 
+	metricsManager, err := metrics.NewManager(ctx, metrics.Config{
+		ServiceName:      metricsService,
+		StatsdServers:    statsdServers,
+		DogStatsdServers: dogstatsServers,
+	})
+	if err != nil {
+		return fmt.Errorf("init metrics manager: %w", err)
+	}
+
 	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
 		KafkaURL:             kafkaURL,
 		KafkaTopic:           kafkaTopic,
@@ -514,12 +662,16 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
 		Access: rootUserAccess,
 		Secret: rootUserSecret,
-	}, port, region, iam, logger, evSender, opts...)
+	}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
 	if err != nil {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
+
+	if !quiet {
+		printBanner(port, admPort, certFile != "", admCertFile != "")
+	}
 
 	c := make(chan error, 2)
 	go func() { c <- srv.Serve() }()
@@ -536,10 +688,17 @@ Loop:
 		case err = <-c:
 			break Loop
 		case <-sigHup:
-			if logger != nil {
-				err = logger.HangUp()
+			if loggers.S3Logger != nil {
+				err = loggers.S3Logger.HangUp()
 				if err != nil {
-					err = fmt.Errorf("HUP logger: %w", err)
+					err = fmt.Errorf("HUP s3 logger: %w", err)
+					break Loop
+				}
+			}
+			if loggers.AdminLogger != nil {
+				err = loggers.AdminLogger.HangUp()
+				if err != nil {
+					err = fmt.Errorf("HUP admin logger: %w", err)
 					break Loop
 				}
 			}
@@ -557,13 +716,22 @@ Loop:
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
-	if logger != nil {
-		err := logger.Shutdown()
+	if loggers.S3Logger != nil {
+		err := loggers.S3Logger.Shutdown()
 		if err != nil {
 			if saveErr == nil {
 				saveErr = err
 			}
-			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
+			fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
+		}
+	}
+	if loggers.AdminLogger != nil {
+		err := loggers.AdminLogger.Shutdown()
+		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
+			fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
 		}
 	}
 
@@ -577,5 +745,183 @@ Loop:
 		}
 	}
 
+	if metricsManager != nil {
+		metricsManager.Close()
+	}
+
 	return saveErr
 }
+
+func printBanner(port, admPort string, ssl, admSsl bool) {
+	interfaces, err := getMatchingIPs(port)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
+		return
+	}
+
+	var admInterfaces []string
+	if admPort != "" {
+		admInterfaces, err = getMatchingIPs(admPort)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
+			return
+		}
+	}
+
+	title := "VersityGW"
+	version := fmt.Sprintf("Version %v, Build %v", Version, Build)
+	urls := []string{}
+
+	hst, prt, err := net.SplitHostPort(port)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
+		return
+	}
+
+	for _, ip := range interfaces {
+		url := fmt.Sprintf("http://%s:%s", ip, prt)
+		if ssl {
+			url = fmt.Sprintf("https://%s:%s", ip, prt)
+		}
+		urls = append(urls, url)
+	}
+
+	if hst == "" {
+		hst = "0.0.0.0"
+	}
+
+	boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
+
+	lines := []string{
+		centerText(title),
+		centerText(version),
+		centerText(boundHost),
+		centerText(""),
+	}
+
+	if len(admInterfaces) > 0 {
+		lines = append(lines,
+			leftText("S3 service listening on:"),
+		)
+	} else {
+		lines = append(lines,
+			leftText("Admin/S3 service listening on:"),
+		)
+	}
+
+	for _, url := range urls {
+		lines = append(lines, leftText("  "+url))
+	}
+
+	if len(admInterfaces) > 0 {
+		lines = append(lines,
+			centerText(""),
+			leftText("Admin service listening on:"),
+		)
+
+		_, prt, err := net.SplitHostPort(admPort)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
+			return
+		}
+
+		for _, ip := range admInterfaces {
+			url := fmt.Sprintf("http://%s:%s", ip, prt)
+			if admSsl {
+				url = fmt.Sprintf("https://%s:%s", ip, prt)
+			}
+			lines = append(lines, leftText("  "+url))
+		}
+	}
+
+	// Print the top border
+	fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
+
+	// Print each line
+	for _, line := range lines {
+		fmt.Printf("│%-*s│\n", columnWidth-2, line)
+	}
+
+	// Print the bottom border
+	fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
+}
+
+// getMatchingIPs returns all IP addresses for local system interfaces that
+// match the input address specification.
+func getMatchingIPs(spec string) ([]string, error) {
+	// Split the input spec into IP and port
+	host, _, err := net.SplitHostPort(spec)
+	if err != nil {
+		return nil, fmt.Errorf("parse address/port: %v", err)
+	}
+
+	// Handle cases where IP is omitted (e.g., ":1234")
+	if host == "" {
+		host = "0.0.0.0"
+	}
+
+	ipaddr, err := net.ResolveIPAddr("ip", host)
+	if err != nil {
+		return nil, err
+	}
+
+	parsedInputIP := ipaddr.IP
+
+	var result []string
+
+	// Get all network interfaces
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, iface := range interfaces {
+		// Get all addresses associated with the interface
+		addrs, err := iface.Addrs()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, addr := range addrs {
+			// Parse the address to get the IP part
+			ipAddr, _, err := net.ParseCIDR(addr.String())
+			if err != nil {
+				return nil, err
+			}
+
+			if ipAddr.IsLinkLocalUnicast() {
+				continue
+			}
+			if ipAddr.IsInterfaceLocalMulticast() {
+				continue
+			}
+			if ipAddr.IsLinkLocalMulticast() {
+				continue
+			}
+
+			// Check if the IP matches the input specification
+			if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
+				result = append(result, ipAddr.String())
+			}
+		}
+	}
+
+	return result, nil
+}
+
+const columnWidth = 70
+
+func centerText(text string) string {
+	padding := (columnWidth - 2 - len(text)) / 2
+	if padding < 0 {
+		padding = 0
+	}
+	return strings.Repeat(" ", padding) + text
+}
+
+func leftText(text string) string {
+	if len(text) > columnWidth-2 {
+		return text
+	}
+	return text + strings.Repeat(" ", columnWidth-2-len(text))
+}

cmd/versitygw/posix.go

@@ -16,13 +16,19 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
 var (
 	chownuid, chowngid bool
+	bucketlinks        bool
+	versioningDir      string
+	dirPerms           uint
 )
 
 func posixCommand() *cli.Command {
@@ -53,6 +59,26 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_CHOWN_GID"},
 				Destination: &chowngid,
 			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
+			&cli.StringFlag{
+				Name:        "versioning-dir",
+				Usage:       "the directory path to enable bucket versioning",
+				EnvVars:     []string{"VGW_VERSIONING_DIR"},
+				Destination: &versioningDir,
+			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
 		},
 	}
 }
@@ -62,9 +88,22 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	be, err := posix.New(ctx.Args().Get(0), posix.PosixOpts{
-		ChownUID: chownuid,
-		ChownGID: chowngid,
+	gwroot := (ctx.Args().Get(0))
+	err := meta.XattrMeta{}.Test(gwroot)
+	if err != nil {
+		return fmt.Errorf("posix xattr check: %v", err)
+	}
+
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
+	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
+		ChownUID:      chownuid,
+		ChownGID:      chowngid,
+		BucketLinks:   bucketlinks,
+		VersioningDir: versioningDir,
+		NewDirPerm:    fs.FileMode(dirPerms),
 	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)

cmd/versitygw/scoutfs.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
+	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/scoutfs"
@@ -63,6 +65,20 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_CHOWN_GID"},
 				Destination: &chowngid,
 			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
 		},
 	}
 }
@@ -72,10 +88,16 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
 	var opts scoutfs.ScoutfsOpts
 	opts.GlacierMode = glacier
 	opts.ChownUID = chownuid
 	opts.ChownGID = chowngid
+	opts.BucketLinks = bucketlinks
+	opts.NewDirPerm = fs.FileMode(dirPerms)
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

cmd/versitygw/test.go

@@ -22,20 +22,22 @@ import (
 )
 
 var (
-	awsID           string
-	awsSecret       string
-	endpoint        string
-	prefix          string
-	dstBucket       string
-	partSize        int64
-	objSize         int64
-	concurrency     int
-	files           int
-	totalReqs       int
-	upload          bool
-	download        bool
-	pathStyle       bool
-	checksumDisable bool
+	awsID             string
+	awsSecret         string
+	endpoint          string
+	prefix            string
+	dstBucket         string
+	partSize          int64
+	objSize           int64
+	concurrency       int
+	files             int
+	totalReqs         int
+	upload            bool
+	download          bool
+	pathStyle         bool
+	checksumDisable   bool
+	versioningEnabled bool
+	azureTests        bool
 )
 
 func testCommand() *cli.Command {
@@ -87,11 +89,33 @@ func initTestCommands() []*cli.Command {
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+				&cli.BoolFlag{
+					Name:        "azure-test-mode",
+					Usage:       "Skips tests that are not supported by Azure",
+					Destination: &azureTests,
+					Aliases:     []string{"azure"},
+				},
+			},
 		},
 		{
 			Name:   "posix",
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test posix when versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		},
 		{
 			Name:   "iam",
@@ -276,6 +300,12 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if debug {
 			opts = append(opts, integration.WithDebug())
 		}
+		if versioningEnabled {
+			opts = append(opts, integration.WithVersioningEnabled())
+		}
+		if azureTests {
+			opts = append(opts, integration.WithAzureMode())
+		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -307,11 +337,22 @@ func extractIntTests() (commands []*cli.Command) {
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
+				if versioningEnabled {
+					opts = append(opts, integration.WithVersioningEnabled())
+				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)
 				return err
 			},
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
 		})
 	}
 	return

cmd/versitygw/utils.go

@@ -54,22 +54,24 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
 	}
 
 	config := s3event.EventFilter{
-		s3event.EventObjectCreated:           true,
-		s3event.EventObjectCreatedPut:        true,
-		s3event.EventObjectCreatedPost:       true,
-		s3event.EventObjectCreatedCopy:       true,
-		s3event.EventCompleteMultipartUpload: true,
-		s3event.EventObjectDeleted:           true,
-		s3event.EventObjectTagging:           true,
-		s3event.EventObjectTaggingPut:        true,
-		s3event.EventObjectTaggingDelete:     true,
-		s3event.EventObjectAclPut:            true,
-		s3event.EventObjectRestore:           true,
-		s3event.EventObjectRestorePost:       true,
-		s3event.EventObjectRestoreCompleted:  true,
+		s3event.EventObjectCreated:              true,
+		s3event.EventObjectCreatedPut:           true,
+		s3event.EventObjectCreatedPost:          true,
+		s3event.EventObjectCreatedCopy:          true,
+		s3event.EventCompleteMultipartUpload:    true,
+		s3event.EventObjectRemoved:              true,
+		s3event.EventObjectRemovedDelete:        true,
+		s3event.EventObjectRemovedDeleteObjects: true,
+		s3event.EventObjectTagging:              true,
+		s3event.EventObjectTaggingPut:           true,
+		s3event.EventObjectTaggingDelete:        true,
+		s3event.EventObjectAclPut:               true,
+		s3event.EventObjectRestore:              true,
+		s3event.EventObjectRestorePost:          true,
+		s3event.EventObjectRestoreCompleted:     true,
 	}
 
-	configBytes, err := json.Marshal(config)
+	configBytes, err := json.MarshalIndent(config, "", "  ")
 	if err != nil {
 		return fmt.Errorf("parse event config: %w", err)
 	}

extra/dashboard/README.md

@@ -0,0 +1,19 @@
+# Versity Gateway Dashboard
+
+This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana. 
+
+The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
+
+This implementation uses the influxql query language. 
+
+## Usage
+
+From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
+
+To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`. 
+
+The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
+
+The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with  username: `admin` and password: `admin`.
+
+To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

extra/dashboard/aws_env_setup.sh

@@ -0,0 +1,4 @@
+export AWS_SECRET_ACCESS_KEY=password
+export AWS_ACCESS_KEY_ID=user
+export AWS_ENDPOINT_URL=http://127.0.0.1:7070
+export AWS_REGION=us-east-1

extra/dashboard/docker-compose-metrics.yml

@@ -0,0 +1,64 @@
+services:
+  telegraf:
+    image: telegraf
+    container_name: telegraf
+    restart: always
+    volumes:
+    - ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      - influxdb
+    links:
+      - influxdb
+    ports:
+    - '8125:8125/udp'
+
+  influxdb:
+    image: influxdb
+    container_name: influxdb
+    restart: always
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
+      - DOCKER_INFLUXDB_INIT_ORG=myorg
+      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb_data:/var/lib/influxdb
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana-server
+    restart: always
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_INSTALL_PLUGINS=
+    links:
+      - influxdb
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./metrics-exploration/grafana_data:/var/lib/grafana
+      - ./metrics-exploration/provisioning:/etc/grafana/provisioning
+
+  versitygw:
+    image: versity/versitygw:latest
+    container_name: versitygw
+    ports:
+      - "7070:7070"
+    environment:
+      - ROOT_ACCESS_KEY=user
+      - ROOT_SECRET_KEY=password
+      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
+    depends_on:
+      - telegraf
+    command: >
+      posix /tmp/vgw
+
+volumes:
+  influxdb_data: {}

extra/dashboard/docker-compose.yml

@@ -0,0 +1,64 @@
+services:
+  telegraf:
+    image: telegraf
+    container_name: telegraf
+    restart: always
+    volumes:
+    - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      - influxdb
+    links:
+      - influxdb
+    ports:
+    - '8125:8125/udp'
+
+  influxdb:
+    image: influxdb
+    container_name: influxdb
+    restart: always
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
+      - DOCKER_INFLUXDB_INIT_ORG=myorg
+      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb_data:/var/lib/influxdb
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana-server
+    restart: always
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_INSTALL_PLUGINS=
+    links:
+      - influxdb
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./grafana_data:/var/lib/grafana
+      - ./provisioning:/etc/grafana/provisioning
+
+  versitygw:
+    image: versity/versitygw:latest
+    container_name: versitygw
+    ports:
+      - "7070:7070"
+    environment:
+      - ROOT_ACCESS_KEY=user
+      - ROOT_SECRET_KEY=password
+      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
+    depends_on:
+      - telegraf
+    command: >
+      posix /tmp/vgw
+
+volumes:
+  influxdb_data: {}

extra/dashboard/provisioning/dashboards/influxql.yaml

@@ -0,0 +1,25 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'influxql'
+    # <int> Org id. Default to 1
+    orgId: 1
+    # <string> name of the dashboard folder.
+    folder: 'influxql'
+    # <string> folder UID. will be automatically generated if not specified
+    folderUid: ''
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: true
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /etc/grafana/provisioning/dashboards/influxql
+      # <bool> use folder names from filesystem to create folders in Grafana
+      foldersFromFilesStructure: true
+

extra/dashboard/provisioning/datasources/influxdb.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+datasources:
+  - name: influxdb
+    type: influxdb
+    isDefault: true
+    access: proxy
+    url: http://influxdb:8086
+    jsonData:
+      dbName: 'metrics'
+      httpHeaderName1: 'Authorization'
+    secureJsonData:
+      httpHeaderValue1: 'Token my-super-secret-auth-token'

extra/dashboard/telegraf.conf

@@ -0,0 +1,34 @@
+[global_tags]
+
+[agent]
+  debug = true
+  quiet = false
+  interval = "60s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  hostname = "versitygw"
+  omit_hostname = false
+
+[[outputs.file]]
+  files = ["stdout"]
+
+[[outputs.influxdb_v2]]
+  urls = ["http://influxdb:8086"]
+  timeout = "5s"
+  token = "my-super-secret-auth-token"
+  organization = "myorg"
+  bucket = "metrics"
+
+[[inputs.statsd]]
+  protocol = "udp4"
+  service_address = ":8125"
+  percentiles = [90]
+  metric_separator = "_"
+  datadog_extensions = false
+  allowed_pending_messages = 10000
+  percentile_limit = 1000

extra/dashboard/test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+. ./aws_env_setup.sh
+
+aws s3 mb s3://test
+aws s3 cp docker-compose.yml s3://test/test.yaml

extra/example.conf

@@ -153,6 +153,14 @@ ROOT_SECRET_ACCESS_KEY=
 # specified, all configured bucket events will be sent to the webhook.
 #VGW_EVENT_WEBHOOK_URL=
 
+# Bucket events can be filtered for any of the above event types. The
+# VGW_EVENT_FILTER option specifies a config file that contains the event
+# filter rules. The event filter rules are used to determine which events are
+# sent to the configured event services. Run:
+# versitygw utils gen-event-filter-config --path .
+# to generate a default rules file "event_config.json" in the current directory.
+#VGW_EVENT_FILTER=
+
 #######################
 # Debug / Diagnostics #
 #######################
@@ -182,20 +190,25 @@ ROOT_SECRET_ACCESS_KEY=
 # as a dedicated IAM service.
 #VGW_IAM_DIR=
 
-# The ldap options will enable the LDAP IAM service with accounts stored in an
-# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
-# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
-# secret credentials and role respectively. The other options are used to
-# connect to the LDAP service.
-#VGW_IAM_LDAP_URL=
-#VGW_IAM_LDAP_BASE_DN=
-#VGW_IAM_LDAP_BIND_DN=
-#VGW_IAM_LDAP_BIND_PASS=
-#VGW_IAM_LDAP_QUERY_BASE=
-#VGW_IAM_LDAP_OBJECT_CLASSES=
-#VGW_IAM_LDAP_ACCESS_ATR=
-#VGW_IAM_LDAP_SECRET_ATR=
-#VGW_IAM_LDAP_ROLE_ATR=
+# The Vault options will enable the Vault IAM service with accounts stored in
+# the HashiCorp Vault service. The Vault URL is the address and port of the
+# Vault server with the format <IP/host>:<port>. A root taken can be used for
+# testing, but it is recommended to use the role based authentication in
+# production. The Vault server certificate, client certificate, and client
+# certificate key are optional, and will default to not verifying the server
+# certificate and not using client certificates. The Vault server certificate
+# is used to verify the Vault server, and the client certificate and key are
+# used to authenticate the gateway to the Vault server. See wiki documentation
+# for an example of using Vault in dev mode with the gateway.
+#VGW_IAM_VAULT_ENDPOINT_URL=
+#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
+#VGW_IAM_VAULT_MOUNT_PATH=
+#VGW_IAM_VAULT_ROOT_TOKEN=
+#VGW_IAM_VAULT_ROLE_ID=
+#VGW_IAM_VAULT_ROLE_SECRET=
+#VGW_IAM_VAULT_SERVER_CERT=
+#VGW_IAM_VAULT_CLIENT_CERT=
+#VGW_IAM_VAULT_CLIENT_CERT_KEY=
 
 # The VGW_S3 IAM service is similar to the internal IAM service, but instead
 # stores the account information JSON encoded in an S3 object. This should use
@@ -210,6 +223,23 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_S3_IAM_BUCKET=
 #VGW_S3_IAM_NO_VERIFY=
 
+# The LDAP options will enable the LDAP IAM service with accounts stored in an
+# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
+# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
+# secret credentials and role respectively. The other options are used to
+# connect to the LDAP service.
+#VGW_IAM_LDAP_URL=
+#VGW_IAM_LDAP_BASE_DN=
+#VGW_IAM_LDAP_BIND_DN=
+#VGW_IAM_LDAP_BIND_PASS=
+#VGW_IAM_LDAP_QUERY_BASE=
+#VGW_IAM_LDAP_OBJECT_CLASSES=
+#VGW_IAM_LDAP_ACCESS_ATR=
+#VGW_IAM_LDAP_SECRET_ATR=
+#VGW_IAM_LDAP_ROLE_ATR=
+#VGW_IAM_LDAP_USER_ID_ATR=
+#VGW_IAM_LDAP_GROUP_ID_ATR=
+
 ###############
 # IAM caching #
 ###############
@@ -228,6 +258,29 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_IAM_CACHE_TTL=120
 #VGW_IAM_CACHE_PRUNE=3600
 
+###########
+# Metrics #
+###########
+
+# The metrics service name is a tag that is added to all metrics to help
+# identify the source of the metrics. This is especially useful when multiple
+# gateways are running. The default is the hostname of the system.
+#VGW_METRICS_SERVICE_NAME=$HOSTNAME
+
+# The metrics service will send metrics to the configured statsd servers. The
+# servers are specified as a comma separated list of host:port pairs. The
+# default is to not send metrics to any statsd servers. The gateway uses
+# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
+#VGW_METRICS_STATSD_SERVERS=
+
+# The metrics service will send metrics to the configured dogstatsd servers.
+# The servers are specified as a comma separated list of host:port pairs. The
+# default is to not send metrics to any dogstatsd servers. Generally
+# DataDog recommends installing a local agent to collect metrics and forward
+# them to the DataDog service. In this case the option value would be the
+# local agent address: 127.0.0.1:8125.
+#VGW_METRICS_DOGSTATS_SERVERS=
+
 ######################################
 # VersityGW Backend Specific Options #
 ######################################
@@ -254,6 +307,16 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_CHOWN_UID=false
 #VGW_CHOWN_GID=false
 
+# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
+# to directories at the top level gateway directory as buckets.
+#VGW_BUCKET_LINKS=false
+
+# The default permissions mode when creating new directories is 0755. Use
+# VGW_DIR_PERMS option to set a different mode for any new directory that the
+# gateway creates. This applies to buckets created through the gateway as well
+# as any parent directories automatically created with object uploads.
+#VGW_DIR_PERMS=0755
+
 ###########
 # scoutfs #
 ###########
@@ -285,6 +348,16 @@ ROOT_SECRET_ACCESS_KEY=
 #VGW_CHOWN_UID=false
 #VGW_CHOWN_GID=false
 
+# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
+# to directories at the top level gateway directory as buckets.
+#VGW_BUCKET_LINKS=false
+
+# The default permissions mode when creating new directories is 0755. Use
+# VGW_DIR_PERMS option to set a different mode for any new directory that the
+# gateway creates. This applies to buckets created through the gateway as well
+# as any parent directories automatically created with object uploads.
+#VGW_DIR_PERMS=0755
+
 ######
 # s3 #
 ######

go.mod

@@ -1,70 +1,82 @@
 module github.com/versity/versitygw
 
-go 1.21
+go 1.21.0
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
-	github.com/aws/aws-sdk-go-v2 v1.26.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
-	github.com/aws/smithy-go v1.20.2
-	github.com/go-ldap/ldap/v3 v3.4.7
-	github.com/gofiber/fiber/v2 v2.52.4
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
+	github.com/DataDog/datadog-go/v5 v5.5.0
+	github.com/aws/aws-sdk-go-v2 v1.32.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
+	github.com/aws/smithy-go v1.22.0
+	github.com/go-ldap/ldap/v3 v3.4.8
+	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.34.1
-	github.com/pkg/xattr v0.4.9
+	github.com/hashicorp/vault-client-go v0.4.3
+	github.com/nats-io/nats.go v1.37.0
+	github.com/oklog/ulid/v2 v2.1.0
+	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
-	github.com/urfave/cli/v2 v2.27.1
-	github.com/valyala/fasthttp v1.52.0
+	github.com/smira/go-statsd v1.3.4
+	github.com/urfave/cli/v2 v2.27.5
+	github.com/valyala/fasthttp v1.57.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sys v0.19.0
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.26.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.11
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.28.1
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.42
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 )

go.sum

@@ -1,83 +1,110 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
+github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
-github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
-github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
-github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
-github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
-github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
-github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
+github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
+github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw=
+github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35 h1:ihPPdcCVSN0IvBByXwqVp28/l4VosBZ6sDulcvU2J7w=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35/go.mod h1:JkgEhs3SVF51Dj3m1Bj+yL8IznpxzkwlA3jLg3x7Kls=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE=
+github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
+github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
-github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.7 h1:3Hbd7mIB1qjd3Ra59fI3JYea/t5kykFu2CVHBca9koE=
-github.com/go-ldap/ldap/v3 v3.4.7/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
-github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
+github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
+github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
+github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
+github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
+github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
+github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@@ -90,13 +117,11 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
-github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -104,45 +129,60 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
-github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
+github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
+github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
+github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
-github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
-github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
+github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
+github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
+github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
+github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
-github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
-github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
+github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
+github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -153,22 +193,29 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@@ -176,13 +223,21 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -195,8 +250,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -211,18 +266,21 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

metrics/actions.go

@@ -0,0 +1,269 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+type Action struct {
+	Name    string
+	Service string
+}
+
+var (
+	ActionMap map[string]Action
+)
+
+var (
+	ActionUndetected                    = "ActionUnDetected"
+	ActionAbortMultipartUpload          = "s3_AbortMultipartUpload"
+	ActionCompleteMultipartUpload       = "s3_CompleteMultipartUpload"
+	ActionCopyObject                    = "s3_CopyObject"
+	ActionCreateBucket                  = "s3_CreateBucket"
+	ActionCreateMultipartUpload         = "s3_CreateMultipartUpload"
+	ActionDeleteBucket                  = "s3_DeleteBucket"
+	ActionDeleteBucketPolicy            = "s3_DeleteBucketPolicy"
+	ActionDeleteBucketTagging           = "s3_DeleteBucketTagging"
+	ActionDeleteObject                  = "s3_DeleteObject"
+	ActionDeleteObjectTagging           = "s3_DeleteObjectTagging"
+	ActionDeleteObjects                 = "s3_DeleteObjects"
+	ActionGetBucketAcl                  = "s3_GetBucketAcl"
+	ActionGetBucketPolicy               = "s3_GetBucketPolicy"
+	ActionGetBucketTagging              = "s3_GetBucketTagging"
+	ActionGetBucketVersioning           = "s3_GetBucketVersioning"
+	ActionGetObject                     = "s3_GetObject"
+	ActionGetObjectAcl                  = "s3_GetObjectAcl"
+	ActionGetObjectAttributes           = "s3_GetObjectAttributes"
+	ActionGetObjectLegalHold            = "s3_GetObjectLegalHold"
+	ActionGetObjectLockConfiguration    = "s3_GetObjectLockConfiguration"
+	ActionGetObjectRetention            = "s3_GetObjectRetention"
+	ActionGetObjectTagging              = "s3_GetObjectTagging"
+	ActionHeadBucket                    = "s3_HeadBucket"
+	ActionHeadObject                    = "s3_HeadObject"
+	ActionListAllMyBuckets              = "s3_ListAllMyBuckets"
+	ActionListMultipartUploads          = "s3_ListMultipartUploads"
+	ActionListObjectVersions            = "s3_ListObjectVersions"
+	ActionListObjects                   = "s3_ListObjects"
+	ActionListObjectsV2                 = "s3_ListObjectsV2"
+	ActionListParts                     = "s3_ListParts"
+	ActionPutBucketAcl                  = "s3_PutBucketAcl"
+	ActionPutBucketPolicy               = "s3_PutBucketPolicy"
+	ActionPutBucketTagging              = "s3_PutBucketTagging"
+	ActionPutBucketVersioning           = "s3_PutBucketVersioning"
+	ActionPutObject                     = "s3_PutObject"
+	ActionPutObjectAcl                  = "s3_PutObjectAcl"
+	ActionPutObjectLegalHold            = "s3_PutObjectLegalHold"
+	ActionPutObjectLockConfiguration    = "s3_PutObjectLockConfiguration"
+	ActionPutObjectRetention            = "s3_PutObjectRetention"
+	ActionPutObjectTagging              = "s3_PutObjectTagging"
+	ActionRestoreObject                 = "s3_RestoreObject"
+	ActionSelectObjectContent           = "s3_SelectObjectContent"
+	ActionUploadPart                    = "s3_UploadPart"
+	ActionUploadPartCopy                = "s3_UploadPartCopy"
+	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
+	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
+	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
+
+	// Admin actions
+	ActionAdminCreateUser        = "admin_CreateUser"
+	ActionAdminUpdateUser        = "admin_UpdateUser"
+	ActionAdminDeleteUser        = "admin_DeleteUser"
+	ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
+	ActionAdminListUsers         = "admin_ListUsers"
+	ActionAdminListBuckets       = "admin_ListBuckets"
+)
+
+func init() {
+	ActionMap = make(map[string]Action)
+
+	ActionMap[ActionUndetected] = Action{
+		Name:    "ActionUnDetected",
+		Service: "unknown",
+	}
+
+	ActionMap[ActionAbortMultipartUpload] = Action{
+		Name:    "AbortMultipartUpload",
+		Service: "s3",
+	}
+	ActionMap[ActionCompleteMultipartUpload] = Action{
+		Name:    "CompleteMultipartUpload",
+		Service: "s3",
+	}
+	ActionMap[ActionCopyObject] = Action{
+		Name:    "CopyObject",
+		Service: "s3",
+	}
+	ActionMap[ActionCreateBucket] = Action{
+		Name:    "CreateBucket",
+		Service: "s3",
+	}
+	ActionMap[ActionCreateMultipartUpload] = Action{
+		Name:    "CreateMultipartUpload",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucket] = Action{
+		Name:    "DeleteBucket",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucketPolicy] = Action{
+		Name:    "DeleteBucketPolicy",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteBucketTagging] = Action{
+		Name:    "DeleteBucketTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteObject] = Action{
+		Name:    "DeleteObject",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteObjectTagging] = Action{
+		Name:    "DeleteObjectTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionDeleteObjects] = Action{
+		Name:    "DeleteObjects",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketAcl] = Action{
+		Name:    "GetBucketAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketPolicy] = Action{
+		Name:    "GetBucketPolicy",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketTagging] = Action{
+		Name:    "GetBucketTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionGetBucketVersioning] = Action{
+		Name:    "GetBucketVersioning",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObject] = Action{
+		Name:    "GetObject",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectAcl] = Action{
+		Name:    "GetObjectAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectAttributes] = Action{
+		Name:    "GetObjectAttributes",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectLegalHold] = Action{
+		Name:    "GetObjectLegalHold",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectLockConfiguration] = Action{
+		Name:    "GetObjectLockConfiguration",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectRetention] = Action{
+		Name:    "GetObjectRetention",
+		Service: "s3",
+	}
+	ActionMap[ActionGetObjectTagging] = Action{
+		Name:    "GetObjectTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionHeadBucket] = Action{
+		Name:    "HeadBucket",
+		Service: "s3",
+	}
+	ActionMap[ActionHeadObject] = Action{
+		Name:    "HeadObject",
+		Service: "s3",
+	}
+	ActionMap[ActionListAllMyBuckets] = Action{
+		Name:    "ListAllMyBuckets",
+		Service: "s3",
+	}
+	ActionMap[ActionListMultipartUploads] = Action{
+		Name:    "ListMultipartUploads",
+		Service: "s3",
+	}
+	ActionMap[ActionListObjectVersions] = Action{
+		Name:    "ListObjectVersions",
+		Service: "s3",
+	}
+	ActionMap[ActionListObjects] = Action{
+		Name:    "ListObjects",
+		Service: "s3",
+	}
+	ActionMap[ActionListObjectsV2] = Action{
+		Name:    "ListObjectsV2",
+		Service: "s3",
+	}
+	ActionMap[ActionListParts] = Action{
+		Name:    "ListParts",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketAcl] = Action{
+		Name:    "PutBucketAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketPolicy] = Action{
+		Name:    "PutBucketPolicy",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketTagging] = Action{
+		Name:    "PutBucketTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionPutBucketVersioning] = Action{
+		Name:    "PutBucketVersioning",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObject] = Action{
+		Name:    "PutObject",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectAcl] = Action{
+		Name:    "PutObjectAcl",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectLegalHold] = Action{
+		Name:    "PutObjectLegalHold",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectLockConfiguration] = Action{
+		Name:    "PutObjectLockConfiguration",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectRetention] = Action{
+		Name:    "PutObjectRetention",
+		Service: "s3",
+	}
+	ActionMap[ActionPutObjectTagging] = Action{
+		Name:    "PutObjectTagging",
+		Service: "s3",
+	}
+	ActionMap[ActionRestoreObject] = Action{
+		Name:    "RestoreObject",
+		Service: "s3",
+	}
+	ActionMap[ActionSelectObjectContent] = Action{
+		Name:    "SelectObjectContent",
+		Service: "s3",
+	}
+	ActionMap[ActionUploadPart] = Action{
+		Name:    "UploadPart",
+		Service: "s3",
+	}
+	ActionMap[ActionUploadPartCopy] = Action{
+		Name:    "UploadPartCopy",
+		Service: "s3",
+	}
+}

metrics/dogstats.go

@@ -0,0 +1,65 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+import (
+	"fmt"
+
+	dogstats "github.com/DataDog/datadog-go/v5/statsd"
+)
+
+// vgwDogStatsd metrics type
+type vgwDogStatsd struct {
+	c *dogstats.Client
+}
+
+var (
+	rateSampleAlways = 1.0
+)
+
+// newDogStatsd takes a server address and returns a statsd merics
+func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
+	c, err := dogstats.New(server,
+		dogstats.WithMaxMessagesPerPayload(1000),
+		dogstats.WithNamespace("versitygw"),
+		dogstats.WithTags([]string{
+			"service:" + service,
+		}))
+	if err != nil {
+		return nil, err
+	}
+	return &vgwDogStatsd{c: c}, nil
+}
+
+// Close closes statsd connections
+func (s *vgwDogStatsd) Close() {
+	s.c.Close()
+}
+
+func (t Tag) ddString() string {
+	if t.Value == "" {
+		return t.Key
+	}
+	return fmt.Sprintf("%v:%v", t.Key, t.Value)
+}
+
+// Add adds value to key
+func (s *vgwDogStatsd) Add(key string, value int64, tags ...Tag) {
+	stags := make([]string, len(tags))
+	for i, t := range tags {
+		stags[i] = t.ddString()
+	}
+	s.c.Count(key, value, stags, rateSampleAlways)
+}

metrics/metrics.go

@@ -0,0 +1,226 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	// max size of data items to buffer before dropping
+	// new incoming data items
+	dataItemCount = 100000
+)
+
+// Tag is added metadata for metrics
+type Tag struct {
+	// Key is tag name
+	Key string
+	// Value is tag data
+	Value string
+}
+
+// Manager is a manager of metrics plugins
+type Manager struct {
+	ctx context.Context
+
+	addDataChan chan datapoint
+
+	config Config
+
+	publishers []publisher
+	wg         sync.WaitGroup
+}
+
+type Config struct {
+	ServiceName      string
+	StatsdServers    string
+	DogStatsdServers string
+}
+
+// NewManager initializes metrics plugins and returns a new metrics manager
+func NewManager(ctx context.Context, conf Config) (*Manager, error) {
+	if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
+		return nil, nil
+	}
+
+	if conf.ServiceName == "" {
+		hostname, err := os.Hostname()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get hostname: %w", err)
+		}
+		conf.ServiceName = hostname
+	}
+
+	addDataChan := make(chan datapoint, dataItemCount)
+
+	mgr := &Manager{
+		addDataChan: addDataChan,
+		ctx:         ctx,
+		config:      conf,
+	}
+
+	// setup statsd endpoints
+	if len(conf.StatsdServers) > 0 {
+		statsdServers := strings.Split(conf.StatsdServers, ",")
+
+		for _, server := range statsdServers {
+			statsd, err := newStatsd(server, conf.ServiceName)
+			if err != nil {
+				return nil, err
+			}
+			mgr.publishers = append(mgr.publishers, statsd)
+		}
+	}
+
+	// setup dogstatsd endpoints
+	if len(conf.DogStatsdServers) > 0 {
+		dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
+
+		for _, server := range dogStatsdServers {
+			dogStatsd, err := newDogStatsd(server, conf.ServiceName)
+			if err != nil {
+				return nil, err
+			}
+			mgr.publishers = append(mgr.publishers, dogStatsd)
+		}
+	}
+
+	mgr.wg.Add(1)
+	go mgr.addForwarder(addDataChan)
+
+	return mgr, nil
+}
+
+func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) {
+	// In case of Authentication failures, url parsing ...
+	if action == "" {
+		action = ActionUndetected
+	}
+
+	a := ActionMap[action]
+	reqTags := []Tag{
+		{Key: "method", Value: ctx.Method()},
+		{Key: "api", Value: a.Service},
+		{Key: "action", Value: a.Name},
+	}
+
+	reqStatus := status
+
+	if err != nil {
+		var apierr s3err.APIError
+		if errors.As(err, &apierr) {
+			reqStatus = apierr.HTTPStatusCode
+		} else {
+			reqStatus = http.StatusInternalServerError
+		}
+	}
+	if reqStatus == 0 {
+		reqStatus = http.StatusOK
+	}
+
+	reqTags = append(reqTags, Tag{
+		Key:   "status",
+		Value: fmt.Sprintf("%v", reqStatus),
+	})
+
+	if err != nil {
+		m.increment("failed_count", reqTags...)
+	} else {
+		m.increment("success_count", reqTags...)
+	}
+
+	switch action {
+	case ActionPutObject:
+		m.add("bytes_written", count, reqTags...)
+		m.increment("object_created_count", reqTags...)
+	case ActionCompleteMultipartUpload:
+		m.increment("object_created_count", reqTags...)
+	case ActionUploadPart:
+		m.add("bytes_written", count, reqTags...)
+	case ActionGetObject:
+		m.add("bytes_read", count, reqTags...)
+	case ActionDeleteObject:
+		m.increment("object_removed_count", reqTags...)
+	case ActionDeleteObjects:
+		m.add("object_removed_count", count, reqTags...)
+	}
+}
+
+// increment increments the key by one
+func (m *Manager) increment(key string, tags ...Tag) {
+	m.add(key, 1, tags...)
+}
+
+// add adds value to key
+func (m *Manager) add(key string, value int64, tags ...Tag) {
+	if m.ctx.Err() != nil {
+		return
+	}
+
+	d := datapoint{
+		key:   key,
+		value: value,
+		tags:  tags,
+	}
+
+	select {
+	case m.addDataChan <- d:
+	default:
+		// channel full, drop the updates
+	}
+}
+
+// Close closes metrics channels, waits for data to complete, closes all plugins
+func (m *Manager) Close() {
+	// drain the datapoint channels
+	close(m.addDataChan)
+	m.wg.Wait()
+
+	// close all publishers
+	for _, p := range m.publishers {
+		p.Close()
+	}
+}
+
+// publisher is the interface for interacting with the metrics plugins
+type publisher interface {
+	Add(key string, value int64, tags ...Tag)
+	Close()
+}
+
+func (m *Manager) addForwarder(addChan <-chan datapoint) {
+	for data := range addChan {
+		for _, s := range m.publishers {
+			s.Add(data.key, data.value, data.tags...)
+		}
+	}
+	m.wg.Done()
+}
+
+type datapoint struct {
+	key   string
+	tags  []Tag
+	value int64
+}

metrics/statsd.go

@@ -0,0 +1,51 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package metrics
+
+import (
+	"github.com/smira/go-statsd"
+)
+
+// vgwStatsd metrics type
+type vgwStatsd struct {
+	c *statsd.Client
+}
+
+// newStatsd takes a server address and returns a statsd merics
+// Supply service name to be used as a tag to identify the spcific
+// gateway instance, this may typically be the gateway hostname
+func newStatsd(server string, service string) (*vgwStatsd, error) {
+	c := statsd.NewClient(
+		server,
+		statsd.MetricPrefix("versitygw."),
+		statsd.TagStyle(statsd.TagFormatInfluxDB),
+		statsd.DefaultTags(statsd.StringTag("service", service)),
+	)
+	return &vgwStatsd{c: c}, nil
+}
+
+// Close closes statsd connections
+func (s *vgwStatsd) Close() {
+	s.c.Close()
+}
+
+// Add adds value to key
+func (s *vgwStatsd) Add(key string, value int64, tags ...Tag) {
+	stags := make([]statsd.Tag, len(tags))
+	for i, t := range tags {
+		stags[i] = statsd.StringTag(t.Key, t.Value)
+	}
+	s.c.Incr(key, value, stags...)
+}

runtests.sh

@@ -5,15 +5,19 @@ rm -rf /tmp/gw
 mkdir /tmp/gw
 rm -rf /tmp/covdata
 mkdir /tmp/covdata
+rm -rf /tmp/versioing.covdata
+mkdir /tmp/versioning.covdata
+rm -rf /tmp/versioningdir
+mkdir /tmp/versioningdir
 
-# run server in background
+# run server in background not versioning-enabled
+# port: 7070(default)
 GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
 GW_PID=$!
 
-# wait a second for server to start up
 sleep 1
 
-# check if server is still running
+# check if versioning-enabled gateway process is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -39,8 +43,39 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
 	exit 1
 fi
 
-# kill off server
 kill $GW_PID
+
+# run server in background versioning-enabled
+# port: 7071
+GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7071 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
+GW_VS_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if versioning-enabled gateway process is still running
+if ! kill -0 $GW_VS_PID; then
+	echo "versioning-enabled server no longer running"
+	exit 1
+fi
+
+# run tests
+# full flow tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 full-flow -vs; then
+	echo "versioning-enabled full-flow tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7071 posix -vs; then
+	echo "versiongin-enabled posix tests failed"
+	kill $GW_VS_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_VS_PID
+
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-router.go

@@ -19,12 +19,13 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminRouter struct{}
 
-func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
-	controller := controllers.NewAdminController(iam, be)
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
+	controller := controllers.NewAdminController(iam, be, logger)
 
 	// CreateUser admin api
 	app.Patch("/create-user", controller.CreateUser)
@@ -32,6 +33,9 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
 	// DeleteUsers admin api
 	app.Patch("/delete-user", controller.DeleteUser)
 
+	// UpdateUser admin api
+	app.Patch("/update-user", controller.UpdateUser)
+
 	// ListUsers admin api
 	app.Patch("/list-users", controller.ListUsers)
 

s3api/admin-server.go

@@ -22,17 +22,18 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminServer struct {
-	app     *fiber.App
 	backend backend.Backend
+	app     *fiber.App
 	router  *S3AdminRouter
-	port    string
 	cert    *tls.Certificate
+	port    string
 }
 
-func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
 	server := &S3AdminServer{
 		app:     app,
 		backend: be,
@@ -46,14 +47,16 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 
 	// Logging middlewares
 	app.Use(logger.New())
-	app.Use(middlewares.DecodeURL(nil))
+	app.Use(middlewares.DecodeURL(l, nil))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
-	app.Use(middlewares.VerifyMD5Body(nil))
-	app.Use(middlewares.AclParser(be, nil))
+	app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(l))
 
-	server.router.Init(app, be, iam)
+	// Admin role checker
+	app.Use(middlewares.IsAdmin(l))
+
+	server.router.Init(app, be, iam, l)
 
 	return server
 }

s3api/controllers/admin.go

@@ -16,107 +16,188 @@ package controllers
 
 import (
 	"encoding/json"
+	"encoding/xml"
 	"fmt"
+	"net/http"
+	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+	"github.com/versity/versitygw/s3response"
 )
 
 type AdminController struct {
 	iam auth.IAMService
 	be  backend.Backend
+	l   s3log.AuditLogger
 }
 
-func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
-	return AdminController{iam: iam, be: be}
+func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
+	return AdminController{iam: iam, be: be, l: l}
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
-	}
 	var usr auth.Account
-	err := json.Unmarshal(ctx.Body(), &usr)
+	err := xml.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return fmt.Errorf("failed to parse request body: %w", err)
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
+			})
 	}
 
-	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
+	if !usr.Role.IsValid() {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
+			})
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		return fmt.Errorf("failed to create user: %w", err)
+		if strings.Contains(err.Error(), "user already exists") {
+			err = s3err.GetAPIError(s3err.ErrAdminUserExists)
+		}
+
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminCreateUser,
+			})
 	}
 
-	return ctx.SendString("The user has been created successfully")
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminCreateUser,
+			Status: http.StatusCreated,
+		})
+}
+
+func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
+	access := ctx.Query("access")
+	if access == "" {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
+	}
+
+	var props auth.MutableProps
+	if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
+	}
+
+	err := c.iam.UpdateUserAccount(access, props)
+	if err != nil {
+		if strings.Contains(err.Error(), "user not found") {
+			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
+		}
+
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminUpdateUser,
+			})
+	}
+
+	return SendResponse(ctx, nil,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminUpdateUser,
+		})
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
-	}
 
 	err := c.iam.DeleteUserAccount(access)
-	if err != nil {
-		return err
-	}
-
-	return ctx.SendString("The user has been deleted successfully")
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminDeleteUser,
+		})
 }
 
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
-	}
 	accs, err := c.iam.ListUserAccounts()
-	if err != nil {
-		return err
-	}
-
-	return ctx.JSON(accs)
+	return SendXMLResponse(ctx,
+		auth.ListUserAccountsResult{
+			Accounts: accs,
+		}, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListUsers,
+		})
 }
 
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
-	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
 
 	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
 	if err != nil {
-		return err
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
+			})
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user specified as the new bucket owner does not exist")
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
+			})
 	}
 
-	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
+	acl := auth.ACL{
+		Owner: owner,
+		Grantees: []auth.Grantee{
+			{
+				Permission: types.PermissionFullControl,
+				Access:     owner,
+				Type:       types.TypeCanonicalUser,
+			},
+		},
+	}
+
+	aclParsed, err := json.Marshal(acl)
 	if err != nil {
-		return err
+		return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
+			&MetaOpts{
+				Logger: c.l,
+				Action: metrics.ActionAdminChangeBucketOwner,
+			})
 	}
 
-	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
+	return SendResponse(ctx, err,
+		&MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminChangeBucketOwner,
+		})
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
-	acct := ctx.Locals("account").(auth.Account)
-	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
-	}
-
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
-	if err != nil {
-		return err
-	}
-
-	return ctx.JSON(buckets)
+	return SendXMLResponse(ctx,
+		s3response.ListBucketsResult{
+			Buckets: buckets,
+		}, err, &MetaOpts{
+			Logger: c.l,
+			Action: metrics.ActionAdminListBuckets,
+		})
 }

s3api/controllers/admin_test.go

@@ -15,12 +15,11 @@
 package controllers
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
@@ -43,67 +42,60 @@ func TestAdminController_CreateUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/create-user", adminController.CreateUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	usr := auth.Account{
-		Access: "access",
-		Secret: "secret",
-		Role:   "invalid role",
-	}
-
-	user, _ := json.Marshal(&usr)
-
-	usr.Role = "admin"
-
-	succUsr, _ := json.Marshal(&usr)
-
-	appErr.Patch("/create-user", adminController.CreateUser)
+	succUser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>admin</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+	invuser := `
+		<Account>
+			<Access>access</Access>
+			<Secret>secret</Secret>
+			<Role>invalid_role</Role>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
-			name: "Admin-create-user-success",
+			name: "Admin-create-user-malformed-body",
 			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
-		{
-			name: "Admin-create-user-invalid-user-role",
-			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
-			},
-			wantErr:    false,
-			statusCode: 500,
-		},
-		{
-			name: "Admin-create-user-invalid-requester-role",
-			app:  appErr,
 			args: args{
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-create-user-invalid-requester-role",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-create-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
+			},
+			wantErr:    false,
+			statusCode: 201,
 		},
 	}
 	for _, tt := range tests {
@@ -119,6 +111,100 @@ func TestAdminController_CreateUser(t *testing.T) {
 	}
 }
 
+func TestAdminController_UpdateUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Patch("/update-user", adminController.UpdateUser)
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	appNotFound := fiber.New()
+
+	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
+
+	succUser := `
+		<Account>
+			<Secret>secret</Secret>
+			<UserID>0</UserID>
+			<GroupID>0</GroupID>
+		</Account>
+	`
+
+	tests := []struct {
+		app        *fiber.App
+		args       args
+		name       string
+		statusCode int
+		wantErr    bool
+	}{
+		{
+			name: "Admin-update-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-update-user-missing-access",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
+			},
+			wantErr:    false,
+			statusCode: 404,
+		},
+		{
+			name: "Admin-update-user-invalid-request-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-not-found",
+			app:  appNotFound,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
+			},
+			wantErr:    false,
+			statusCode: 404,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
 func TestAdminController_DeleteUser(t *testing.T) {
 	type args struct {
 		req *http.Request
@@ -134,28 +220,14 @@ func TestAdminController_DeleteUser(t *testing.T) {
 
 	app := fiber.New()
 
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/delete-user", adminController.DeleteUser)
 
-	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appErr.Patch("/delete-user", adminController.DeleteUser)
-
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Admin-delete-user-success",
@@ -166,15 +238,6 @@ func TestAdminController_DeleteUser(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
-		{
-			name: "Admin-delete-user-invalid-requester-role",
-			app:  appErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
-			},
-			wantErr:    false,
-			statusCode: 500,
-		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -211,48 +274,18 @@ func TestAdminController_ListUsers(t *testing.T) {
 	}
 
 	appErr := fiber.New()
-
-	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appErr.Patch("/list-users", adminControllerErr.ListUsers)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-users", adminController.ListUsers)
-
 	appSucc := fiber.New()
-
-	appSucc.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
-		{
-			name: "Admin-list-users-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
-			},
-			wantErr:    false,
-			statusCode: 500,
-		},
 		{
 			name: "Admin-list-users-iam-error",
 			app:  appErr,
@@ -291,7 +324,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 	adminController := AdminController{
 		be: &BackendMock{
-			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
+			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
 				return nil
 			},
 		},
@@ -319,57 +352,21 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
-
 	appIamErr := fiber.New()
-
-	appIamErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
 
 	appIamNoSuchUser := fiber.New()
-
-	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
-		{
-			name: "Change-bucket-owner-access-denied",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
-			},
-			wantErr:    false,
-			statusCode: 500,
-		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
 			app:  appIamErr,
@@ -386,7 +383,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 404,
 		},
 		{
 			name: "Change-bucket-owner-success",
@@ -395,7 +392,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 201,
+			statusCode: 200,
 		},
 	}
 	for _, tt := range tests {
@@ -424,39 +421,15 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	}
 
 	app := fiber.New()
-
-	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
-		return ctx.Next()
-	})
-
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
-	appRoleErr := fiber.New()
-
-	appRoleErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
-		return ctx.Next()
-	})
-
-	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
-
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
-		{
-			name: "List-buckets-incorrect-role",
-			app:  appRoleErr,
-			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
-			},
-			wantErr:    false,
-			statusCode: 500,
-		},
 		{
 			name: "List-buckets-success",
 			app:  app,

s3api/controllers/base_test.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -77,7 +76,7 @@ func TestNew(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := New(tt.args.be, tt.args.iam, nil, nil, false)
+			got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false)
 			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("New() = %v, want %v", got, tt.want)
 			}
@@ -93,7 +92,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, nil
 			},
 		},
@@ -110,7 +109,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr := fiber.New()
 	s3ApiControllerErr := S3ApiController{
 		be: &BackendMock{
-			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
 			},
 		},
@@ -124,11 +123,11 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
 
 	tests := []struct {
-		name       string
 		args       args
 		app        *fiber.App
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "List-bucket-method-not-allowed",
@@ -188,10 +187,10 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 				return &s3.GetObjectAclOutput{}, nil
 			},
-			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
-				return &s3.GetObjectAttributesOutput{}, nil
+			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+				return s3response.GetObjectAttributesResponse{}, nil
 			},
-			GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+			GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
 					Metadata:        map[string]string{"hello": "world"},
 					ContentType:     getPtr("application/xml"),
@@ -205,6 +204,19 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
 				return map[string]string{"hello": "world"}, nil
 			},
+			GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
+				result, err := json.Marshal(types.ObjectLockRetention{
+					Mode: types.ObjectLockRetentionModeCompliance,
+				})
+				if err != nil {
+					return nil, err
+				}
+				return result, nil
+			},
+			GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
+				result := true
+				return &result, nil
+			},
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
@@ -221,11 +233,11 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Get-actions-get-tags-success",
@@ -236,6 +248,24 @@ func TestS3ApiController_GetActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Get-actions-get-object-retention-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?retention", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Get-actions-get-object-legal-hold-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?legal-hold", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-actions-invalid-max-parts-string",
 			app:  app,
@@ -329,6 +359,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 		req *http.Request
 	}
 
+	objectLockResult, err := json.Marshal(auth.BucketLockConfig{})
+	if err != nil {
+		t.Errorf("failed to parse object lock result %v", err)
+	}
+
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
@@ -338,24 +373,30 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 				return s3response.ListMultipartUploadsResult{}, nil
 			},
-			ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-				return &s3.ListObjectsV2Output{}, nil
+			ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+				return s3response.ListObjectsV2Result{}, nil
 			},
-			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-				return &s3.ListObjectsOutput{}, nil
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+				return s3response.ListObjectsResult{}, nil
 			},
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return map[string]string{}, nil
 			},
-			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
-				return &s3.GetBucketVersioningOutput{}, nil
+			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+				return s3response.GetBucketVersioningOutput{}, nil
 			},
-			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
-				return &s3.ListObjectVersionsOutput{}, nil
+			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+				return s3response.ListVersionsResult{}, nil
 			},
 			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
 				return []byte{}, nil
 			},
+			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+				return objectLockResult, nil
+			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerEnforced, nil
+			},
 		},
 	}
 
@@ -369,14 +410,14 @@ func TestS3ApiController_ListActions(t *testing.T) {
 
 	app.Get("/:bucket", s3ApiController.ListActions)
 
-	//Error case
+	// Error case
 	s3ApiControllerError := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+				return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 			},
 			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 				return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -394,11 +435,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Get-bucket-tagging-non-existing-bucket",
@@ -409,6 +450,15 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 404,
 		},
+		{
+			name: "Get-bucket-ownership-control-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-bucket-tagging-success",
 			app:  app,
@@ -418,6 +468,15 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Get-object-lock-configuration-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?object-lock", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-bucket-acl-success",
 			app:  app,
@@ -514,7 +573,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	app := fiber.New()
 
 	// Mock valid acl
-	acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
+	acl := auth.ACL{Owner: "valid access"}
 	acldata, err := json.Marshal(acl)
 	if err != nil {
 		t.Errorf("Failed to parse the params: %v", err.Error())
@@ -545,14 +604,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</AccessControlPolicy>
 	`
 
-	succBody := `
-	<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-		<Owner>
-			<ID>valid access</ID>
-		</Owner>
-	</AccessControlPolicy>
-	`
-
 	tagBody := `
 	<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
 		<TagSet>
@@ -584,6 +635,34 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	}
 	`
 
+	objectLockBody := `
+	<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<ObjectLockEnabled>Enabled</ObjectLockEnabled>
+		<Rule>
+			<DefaultRetention>
+				<Mode>GOVERNANCE</Mode>
+				<Years>2</Years>
+			</DefaultRetention>
+		</Rule>
+	</ObjectLockConfiguration>
+	`
+
+	ownershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
+	invalidOwnershipBody := `
+	<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Rule>
+			<ObjectOwnership>invalid_value</ObjectOwnership>
+		</Rule>
+	</OwnershipControls>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -598,12 +677,21 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
 				return nil
 			},
-			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+			PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
 				return nil
 			},
 			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
 				return nil
 			},
+			PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
+				return nil
+			},
+			PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
+				return nil
+			},
+			GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
+				return types.ObjectOwnershipBucketOwnerPreferred, nil
+			},
 		},
 	}
 	// Mock ctx.Locals
@@ -627,22 +715,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 
 	// PutBucketAcl incorrect bucket owner case
 	incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
-	incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
 
 	// PutBucketAcl acl success
-	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
+	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
 	aclSuccReq.Header.Set("X-Amz-Acl", "private")
 
 	// Invalid acl body case
 	errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
 	errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
 
+	invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
+	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
+
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Put-bucket-tagging-invalid-body",
@@ -660,6 +750,42 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
 			},
 			wantErr:    false,
+			statusCode: 204,
+		},
+		{
+			name: "Put-bucket-ownership-controls-invalid-ownership",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Put-object-lock-configuration-invalid-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-object-lock-configuration-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", strings.NewReader(objectLockBody)),
+			},
+			wantErr:    false,
 			statusCode: 200,
 		},
 		{
@@ -732,7 +858,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 				req: incorrectBucketOwner,
 			},
 			wantErr:    false,
-			statusCode: 403,
+			statusCode: 400,
 		},
 		{
 			name: "Put-bucket-acl-success",
@@ -744,7 +870,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 200,
 		},
 		{
-			name: "Put-bucket-invalid-bucket-name",
+			name: "Create-bucket-invalid-acl-ownership-combination",
+			app:  app,
+			args: args{
+				req: invAclOwnershipReq,
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Create-bucket-invalid-bucket-name",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
@@ -753,7 +888,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 400,
 		},
 		{
-			name: "Put-bucket-success",
+			name: "Create-bucket-success",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
@@ -806,6 +941,19 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	</Tagging>
 	`
 
+	retentionBody := `
+	<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Mode>GOVERNANCE</Mode>
+		<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
+	</Retention>
+	`
+
+	legalHoldBody := `
+	<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Status>ON</Status>
+	</LegalHold>
+	`
+
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
@@ -820,8 +968,8 @@ func TestS3ApiController_PutActions(t *testing.T) {
 					CopyObjectResult: &types.CopyObjectResult{},
 				}, nil
 			},
-			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
-				return "ETag", nil
+			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
+				return s3response.PutObjectOutput{}, nil
 			},
 			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
 				return "hello", nil
@@ -832,6 +980,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
 				return s3response.CopyObjectResult{}, nil
 			},
+			PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
+				return nil
+			},
+			PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
+				return nil
+			},
+			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+				return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
+			},
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
@@ -877,11 +1034,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Put-object-part-error-case",
@@ -910,6 +1067,42 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "put-object-retention-invalid-request",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "put-object-retention-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "put-legal-hold-invalid-request",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "put-legal-hold-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", strings.NewReader(legalHoldBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Put-object-acl-invalid-acl",
 			app:  app,
@@ -1024,12 +1217,18 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
+			DeleteBucketFunc: func(_ context.Context, bucket string) error {
 				return nil
 			},
 			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 				return nil
 			},
+			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
+			DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
 		},
 	}
 
@@ -1044,11 +1243,11 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app.Delete("/:bucket", s3ApiController.DeleteBucket)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Delete-bucket-success",
@@ -1068,6 +1267,23 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			wantErr:    false,
 			statusCode: 204,
 		},
+		{
+			name: "Delete-bucket-ownership-controls-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		}, {
+			name: "Delete-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -1096,6 +1312,9 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 			DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 				return s3response.DeleteResult{}, nil
 			},
+			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+				return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
+			},
 		},
 	}
 
@@ -1115,11 +1334,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	request.Header.Set("Content-Type", "application/xml")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Delete-Objects-success",
@@ -1164,8 +1383,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
-				return nil
+			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+				return &s3.DeleteObjectOutput{}, nil
 			},
 			AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
 				return nil
@@ -1173,6 +1392,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 			DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error {
 				return nil
 			},
+			GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+				return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
+			},
 		},
 	}
 
@@ -1192,8 +1414,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 		GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 			return acldata, nil
 		},
-		DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
-			return s3err.GetAPIError(7)
+		DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		},
+		GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+			return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
 		},
 	}}
 
@@ -1207,11 +1432,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Abort-multipart-upload-success",
@@ -1285,6 +1510,7 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
+		ctx.Locals("region", "us-east-1")
 		return ctx.Next()
 	})
 
@@ -1308,17 +1534,18 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
+		ctx.Locals("region", "us-east-1")
 		return ctx.Next()
 	})
 
 	appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Head-bucket-success",
@@ -1401,7 +1628,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 				return acldata, nil
 			},
 			HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-				return nil, s3err.GetAPIError(42)
+				return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 			},
 		},
 	}
@@ -1416,11 +1643,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Head-object-success",
@@ -1470,8 +1697,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 				return &s3.CompleteMultipartUploadOutput{}, nil
 			},
-			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-				return &s3.CreateMultipartUploadOutput{}, nil
+			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+				return s3response.InitiateMultipartUploadResult{}, nil
 			},
 			SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
 				return func(w *bufio.Writer) {}
@@ -1496,29 +1723,20 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Restore-object-success",
 			app:  app,
-			args: args{
-				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
-			},
-			wantErr:    false,
-			statusCode: 200,
-		},
-		{
-			name: "Restore-object-error",
-			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 200,
 		},
 		{
 			name: "Select-object-content-invalid-body",
@@ -1590,10 +1808,10 @@ func Test_XMLresponse(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		name       string
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Internal-server-error",
@@ -1665,10 +1883,10 @@ func Test_response(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		name       string
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Internal-server-error",

s3api/controllers/iam_moq_test.go

@@ -33,6 +33,9 @@ var _ auth.IAMService = &IAMServiceMock{}
 //			ShutdownFunc: func() error {
 //				panic("mock out the Shutdown method")
 //			},
+//			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+//				panic("mock out the UpdateUserAccount method")
+//			},
 //		}
 //
 //		// use mockedIAMService in code that requires auth.IAMService
@@ -55,6 +58,9 @@ type IAMServiceMock struct {
 	// ShutdownFunc mocks the Shutdown method.
 	ShutdownFunc func() error
 
+	// UpdateUserAccountFunc mocks the UpdateUserAccount method.
+	UpdateUserAccountFunc func(access string, props auth.MutableProps) error
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// CreateAccount holds details about calls to the CreateAccount method.
@@ -78,12 +84,20 @@ type IAMServiceMock struct {
 		// Shutdown holds details about calls to the Shutdown method.
 		Shutdown []struct {
 		}
+		// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
+		UpdateUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+			// Props is the props argument value.
+			Props auth.MutableProps
+		}
 	}
 	lockCreateAccount     sync.RWMutex
 	lockDeleteUserAccount sync.RWMutex
 	lockGetUserAccount    sync.RWMutex
 	lockListUserAccounts  sync.RWMutex
 	lockShutdown          sync.RWMutex
+	lockUpdateUserAccount sync.RWMutex
 }
 
 // CreateAccount calls CreateAccountFunc.
@@ -235,3 +249,39 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
 	mock.lockShutdown.RUnlock()
 	return calls
 }
+
+// UpdateUserAccount calls UpdateUserAccountFunc.
+func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
+	if mock.UpdateUserAccountFunc == nil {
+		panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+		Props  auth.MutableProps
+	}{
+		Access: access,
+		Props:  props,
+	}
+	mock.lockUpdateUserAccount.Lock()
+	mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
+	mock.lockUpdateUserAccount.Unlock()
+	return mock.UpdateUserAccountFunc(access, props)
+}
+
+// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.UpdateUserAccountCalls())
+func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
+	Access string
+	Props  auth.MutableProps
+} {
+	var calls []struct {
+		Access string
+		Props  auth.MutableProps
+	}
+	mock.lockUpdateUserAccount.RLock()
+	calls = mock.calls.UpdateUserAccount
+	mock.lockUpdateUserAccount.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -24,6 +24,7 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
@@ -31,7 +32,7 @@ var (
 	singlePath = regexp.MustCompile(`^/[^/]+/?$`)
 )
 
-func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
+func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
 		path := ctx.Path()
@@ -48,10 +49,19 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 			!ctx.Request().URI().QueryArgs().Has("acl") &&
 			!ctx.Request().URI().QueryArgs().Has("tagging") &&
 			!ctx.Request().URI().QueryArgs().Has("versioning") &&
-			!ctx.Request().URI().QueryArgs().Has("policy") {
+			!ctx.Request().URI().QueryArgs().Has("policy") &&
+			!ctx.Request().URI().QueryArgs().Has("object-lock") &&
+			!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
+			if readonly {
+				return controllers.SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrAccessDenied),
+					&controllers.MetaOpts{
+						Logger: logger,
+						Action: "CreateBucket",
+					})
+			}
 			return ctx.Next()
 		}
 		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})

s3api/middlewares/admin.go

@@ -0,0 +1,59 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		acct := ctx.Locals("account").(auth.Account)
+		if acct.Role != auth.RoleAdmin {
+			path := ctx.Path()
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
+				&controllers.MetaOpts{
+					Logger: logger,
+					Action: detectAction(path),
+				})
+		}
+
+		return ctx.Next()
+	}
+}
+
+func detectAction(path string) (action string) {
+	if strings.Contains(path, "create-user") {
+		action = metrics.ActionAdminCreateUser
+	} else if strings.Contains(path, "update-user") {
+		action = metrics.ActionAdminUpdateUser
+	} else if strings.Contains(path, "delete-user") {
+		action = metrics.ActionAdminDeleteUser
+	} else if strings.Contains(path, "list-user") {
+		action = metrics.ActionAdminListUsers
+	} else if strings.Contains(path, "list-buckets") {
+		action = metrics.ActionAdminListBuckets
+	} else if strings.Contains(path, "change-bucket-owner") {
+		action = metrics.ActionAdminChangeBucketOwner
+	}
+	return action
+}

s3api/middlewares/authentication.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
@@ -40,7 +41,7 @@ type RootUserConfig struct {
 	Secret string
 }
 
-func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
+func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
@@ -54,16 +55,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
 		if authorization == "" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
 		}
 
 		authData, err := utils.ParseAuthorization(authorization)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		if authData.Algorithm != "AWS4-HMAC-SHA256" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger, mm)
 		}
 
 		if authData.Region != region {
@@ -71,40 +72,40 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 				Code:           "SignatureDoesNotMatch",
 				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
 				HTTPStatusCode: http.StatusForbidden,
-			}, logger)
+			}, logger, mm)
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
 
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
 		}
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 		ctx.Locals("account", account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
 		if date == "" {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm)
 		}
 
 		// Parse the date and check the date validity
 		tdate, err := time.Parse(iso8601Format, date)
 		if err != nil {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm)
 		}
 
 		if date[:8] != authData.Date {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm)
 		}
 
 		// Validate the dates difference
 		err = utils.ValidateDate(tdate)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		if utils.IsBigDataAction(ctx) {
@@ -125,7 +126,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 
 			// Compare the calculated hash with the hash provided
 			if hashPayload != hexPayload {
-				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm)
 			}
 		}
 
@@ -134,13 +135,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if contentLengthStr != "" {
 			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
 			if err != nil {
-				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
 			}
 		}
 
 		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		return ctx.Next()
@@ -148,8 +149,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 }
 
 type accounts struct {
-	root RootUserConfig
 	iam  auth.IAMService
+	root RootUserConfig
 }
 
 func (a accounts) getAccount(access string) (auth.Account, error) {
@@ -164,6 +165,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
 	return a.iam.GetUserAccount(access)
 }
 
-func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
-	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error {
+	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 }

s3api/middlewares/chunk.go

@@ -20,13 +20,14 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3log"
 )
 
 // ProcessChunkedBody initializes the chunked upload stream if the
 // request appears to be a chunked upload
-func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
+func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
 		if decodedLength == "" {
@@ -36,7 +37,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
 
 		authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		acct := ctx.Locals("account").(auth.Account)
@@ -51,7 +52,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
 				return cr
 			})
 			if err != nil {
-				return sendResponse(ctx, err, logger)
+				return sendResponse(ctx, err, logger, mm)
 			}
 			return ctx.Next()
 		}

s3api/middlewares/presign-auth.go

@@ -20,12 +20,13 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
-func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
+func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
@@ -38,16 +39,16 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 
 		authData, err := utils.ParsePresignedURIParts(ctx)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		ctx.Locals("isRoot", authData.Access == root.Access)
 		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
-			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
 		}
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 		ctx.Locals("account", account)
 
@@ -61,7 +62,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
 
 		err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
 		if err != nil {
-			return sendResponse(ctx, err, logger)
+			return sendResponse(ctx, err, logger, mm)
 		}
 
 		return ctx.Next()

s3api/middlewares/url-decoder.go

@@ -18,19 +18,19 @@ import (
 	"net/url"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
 
-func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
+func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		reqURL := ctx.Request().URI().String()
-		decoded, err := url.Parse(reqURL)
+		unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
 		if err != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
 		}
-		ctx.Path(decoded.Path)
+		ctx.Path(unescp)
 		return ctx.Next()
 	}
 }

s3api/router.go

@@ -18,7 +18,9 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
 )
@@ -27,26 +29,29 @@ type S3ApiRouter struct {
 	WithAdmSrv bool
 }
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
-	s3ApiController := controllers.New(be, iam, logger, evs, debug)
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
+	s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
 
 	if sa.WithAdmSrv {
-		adminController := controllers.NewAdminController(iam, be)
+		adminController := controllers.NewAdminController(iam, be, aLogger)
 
 		// CreateUser admin api
-		app.Patch("/create-user", adminController.CreateUser)
+		app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
 
 		// DeleteUsers admin api
-		app.Patch("/delete-user", adminController.DeleteUser)
+		app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
+
+		// UpdateUser admin api
+		app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
 
 		// ListUsers admin api
-		app.Patch("/list-users", adminController.ListUsers)
+		app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
 
 		// ChangeBucketOwner admin api
-		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+		app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
 
 		// ListBucketsAndOwners admin api
-		app.Patch("/list-buckets", adminController.ListBuckets)
+		app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
 	}
 
 	// ListBuckets action

s3api/router_test.go

@@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) {
 		iam auth.IAMService
 	}
 	tests := []struct {
-		name string
-		sa   *S3ApiRouter
 		args args
+		sa   *S3ApiRouter
+		name string
 	}{
 		{
 			name: "Initialize S3 api router",
@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false)
 		})
 	}
 }

s3api/server.go

@@ -22,23 +22,36 @@ import (
 	"github.com/gofiber/fiber/v2/middleware/logger"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
 )
 
 type S3ApiServer struct {
-	app     *fiber.App
-	backend backend.Backend
-	router  *S3ApiRouter
-	port    string
-	cert    *tls.Certificate
-	quiet   bool
-	debug   bool
-	health  string
+	backend  backend.Backend
+	app      *fiber.App
+	router   *S3ApiRouter
+	cert     *tls.Certificate
+	port     string
+	health   string
+	quiet    bool
+	debug    bool
+	readonly bool
 }
 
-func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
+func New(
+	app *fiber.App,
+	be backend.Backend,
+	root middlewares.RootUserConfig,
+	port, region string,
+	iam auth.IAMService,
+	l s3log.AuditLogger,
+	adminLogger s3log.AuditLogger,
+	evs s3event.S3EventSender,
+	mm *metrics.Manager,
+	opts ...Option,
+) (*S3ApiServer, error) {
 	server := &S3ApiServer{
 		app:     app,
 		backend: be,
@@ -52,7 +65,9 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 
 	// Logging middlewares
 	if !server.quiet {
-		app.Use(logger.New())
+		app.Use(logger.New(logger.Config{
+			Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
+		}))
 	}
 	// Set up health endpoint if specified
 	if server.health != "" {
@@ -60,17 +75,17 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 			return ctx.SendStatus(http.StatusOK)
 		})
 	}
-	app.Use(middlewares.DecodeURL(l))
+	app.Use(middlewares.DecodeURL(l, mm))
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
-	app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
-	app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
+	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
+	app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
+	app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
 	app.Use(middlewares.VerifyMD5Body(l))
-	app.Use(middlewares.AclParser(be, l))
+	app.Use(middlewares.AclParser(be, l, server.readonly))
 
-	server.router.Init(app, be, iam, l, evs, server.debug)
+	server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly)
 
 	return server, nil
 }
@@ -103,6 +118,10 @@ func WithHealth(health string) Option {
 	return func(s *S3ApiServer) { s.health = health }
 }
 
+func WithReadOnly() Option {
+	return func(s *S3ApiServer) { s.readonly = true }
+}
+
 func (sa *S3ApiServer) Serve() (err error) {
 	if sa.cert != nil {
 		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

s3api/server_test.go

@@ -39,9 +39,9 @@ func TestNew(t *testing.T) {
 	port := ":7070"
 
 	tests := []struct {
-		name            string
-		args            args
 		wantS3ApiServer *S3ApiServer
+		args            args
+		name            string
 		wantErr         bool
 	}{
 		{
@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
-				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
+				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -78,8 +78,8 @@ func TestNew(t *testing.T) {
 
 func TestS3ApiServer_Serve(t *testing.T) {
 	tests := []struct {
-		name    string
 		sa      *S3ApiServer
+		name    string
 		wantErr bool
 	}{
 		{

s3api/utils/auth-reader.go

@@ -41,10 +41,10 @@ const (
 // the data is completely read.
 type AuthReader struct {
 	ctx    *fiber.Ctx
+	r      *HashReader
 	auth   AuthData
 	secret string
 	size   int
-	r      *HashReader
 	debug  bool
 }
 

s3api/utils/auth_test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package utils
 
 import (

s3api/utils/chunk-reader.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"hash"
 	"io"
+	"math"
 	"strconv"
 	"time"
 
@@ -47,15 +48,15 @@ const (
 // object data stream
 type ChunkReader struct {
 	r                io.Reader
-	signingKey       []byte
+	chunkHash        hash.Hash
 	prevSig          string
 	parsedSig        string
+	strToSignPrefix  string
+	signingKey       []byte
+	stash            []byte
 	currentChunkSize int64
 	chunkDataLeft    int64
 	trailerExpected  int
-	stash            []byte
-	chunkHash        hash.Hash
-	strToSignPrefix  string
 	skipcheck        bool
 }
 
@@ -192,12 +193,15 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		cr.chunkDataLeft = 0
 		cr.chunkHash.Write(p[:chunkSize])
 		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		if (chunkSize + int64(n)) > math.MaxInt {
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
 		return n + int(chunkSize), err
-	} else {
-		cr.chunkDataLeft = chunkSize - int64(n)
-		cr.chunkHash.Write(p[:n])
 	}
 
+	cr.chunkDataLeft = chunkSize - int64(n)
+	cr.chunkHash.Write(p[:n])
+
 	return n, nil
 }
 
@@ -231,6 +235,7 @@ const (
 // error if any. See the AWS documentation for the chunk header format. The
 // header[0] byte is expected to be the first byte of the chunk size here.
 func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
+	stashLen := len(cr.stash)
 	if cr.stash != nil {
 		tmp := make([]byte, maxHeaderSize)
 		copy(tmp, cr.stash)
@@ -265,5 +270,5 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
 	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
 	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
 
-	return chunkSize, signature, dataStartOffset, nil
+	return chunkSize, signature, dataStartOffset - stashLen, nil
 }

s3api/utils/presign-auth-reader.go

@@ -41,10 +41,10 @@ const (
 // data requests where the data size is not known until
 // the data is completely read.
 type PresignedAuthReader struct {
+	r      io.Reader
 	ctx    *fiber.Ctx
 	auth   AuthData
 	secret string
-	r      io.Reader
 	debug  bool
 }
 

s3api/utils/presign-auth-reader_test.go

@@ -23,13 +23,13 @@ import (
 
 func Test_validateExpiration(t *testing.T) {
 	type args struct {
-		str  string
 		date time.Time
+		str  string
 	}
 	tests := []struct {
-		name string
 		args args
 		err  error
+		name string
 	}{
 		{
 			name: "empty-expiration",

s3api/utils/utils.go

@@ -26,10 +26,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/smithy-go/encoding/httpbinding"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
 var (
@@ -37,6 +39,10 @@ var (
 	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
+const (
+	upperhex = "0123456789ABCDEF"
+)
+
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
 	headers.DisableNormalizing()
@@ -62,7 +68,9 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		body = bytes.NewReader(req.Body())
 	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
+	escapedURI := escapeOriginalURI(ctx)
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), escapedURI, body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -172,7 +180,7 @@ func ParseUint(str string) (int32, error) {
 	}
 	num, err := strconv.ParseUint(str, 10, 16)
 	if err != nil {
-		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+		return 1000, fmt.Errorf("invalid uint: %w", err)
 	}
 	return int32(num), nil
 }
@@ -222,25 +230,205 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
 	return false
 }
 
+// expiration time window
+// https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationTimeStamp
+const timeExpirationSec = 15 * 60
+
 func ValidateDate(date time.Time) error {
 	now := time.Now().UTC()
 	diff := date.Unix() - now.Unix()
 
-	// Checks the dates difference to be less than a minute
-	if diff > 60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
-	}
-	if diff < -60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
+	// Checks the dates difference to be within allotted window
+	if diff > timeExpirationSec || diff < -timeExpirationSec {
+		return s3err.GetAPIError(s3err.ErrRequestTimeTooSkewed)
 	}
 
 	return nil
 }
+
+func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
+	for _, obj := range objs {
+		result = append(result, *obj.Key)
+	}
+
+	return
+}
+
+func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResponse) s3response.GetObjectAttributesResponse {
+	// These properties shouldn't appear in the final response body
+	output.LastModified = nil
+	output.VersionId = nil
+	output.DeleteMarker = nil
+
+	if _, ok := attrs[s3response.ObjectAttributesEtag]; !ok {
+		output.ETag = nil
+	}
+	if _, ok := attrs[s3response.ObjectAttributesObjectParts]; !ok {
+		output.ObjectParts = nil
+	}
+	if _, ok := attrs[s3response.ObjectAttributesObjectSize]; !ok {
+		output.ObjectSize = nil
+	}
+	if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
+		output.StorageClass = ""
+	}
+	fmt.Printf("%+v\n", output)
+
+	return output
+}
+
+func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
+	attrs := map[s3response.ObjectAttributes]struct{}{}
+	var err error
+	ctx.Request().Header.VisitAll(func(key, value []byte) {
+		if string(key) == "X-Amz-Object-Attributes" {
+			oattrs := strings.Split(string(value), ",")
+			for _, a := range oattrs {
+				attr := s3response.ObjectAttributes(a)
+				if !attr.IsValid() {
+					err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
+					break
+				}
+				attrs[attr] = struct{}{}
+			}
+		}
+	})
+
+	if len(attrs) == 0 {
+		return nil, s3err.GetAPIError(s3err.ErrObjectAttributesInvalidHeader)
+	}
+
+	return attrs, err
+}
+
+type objLockCfg struct {
+	RetainUntilDate time.Time
+	ObjectLockMode  types.ObjectLockMode
+	LegalHoldStatus types.ObjectLockLegalHoldStatus
+}
+
+func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
+	legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold")
+	objLockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode")
+	objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")
+
+	if (objLockDate != "" && objLockModeHdr == "") || (objLockDate == "" && objLockModeHdr != "") {
+		return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)
+	}
+
+	var retainUntilDate time.Time
+	if objLockDate != "" {
+		rDate, err := time.Parse(time.RFC3339, objLockDate)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		}
+		if rDate.Before(time.Now()) {
+			return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
+		}
+		retainUntilDate = rDate
+	}
+
+	objLockMode := types.ObjectLockMode(objLockModeHdr)
+
+	if objLockMode != "" &&
+		objLockMode != types.ObjectLockModeCompliance &&
+		objLockMode != types.ObjectLockModeGovernance {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)
+
+	if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	return &objLockCfg{
+		RetainUntilDate: retainUntilDate,
+		ObjectLockMode:  objLockMode,
+		LegalHoldStatus: legalHold,
+	}, nil
+}
+
+func IsValidOwnership(val types.ObjectOwnership) bool {
+	switch val {
+	case types.ObjectOwnershipBucketOwnerEnforced:
+		return true
+	case types.ObjectOwnershipBucketOwnerPreferred:
+		return true
+	case types.ObjectOwnershipObjectWriter:
+		return true
+	default:
+		return false
+	}
+}
+
+func escapeOriginalURI(ctx *fiber.Ctx) string {
+	path := ctx.Path()
+
+	// Escape the URI original path
+	escapedURI := escapePath(path)
+
+	// Add the URI query params
+	query := string(ctx.Request().URI().QueryArgs().QueryString())
+	if query != "" {
+		escapedURI = escapedURI + "?" + query
+	}
+
+	return escapedURI
+}
+
+// Escapes the path string
+// Most of the parts copied from std url
+func escapePath(s string) string {
+	hexCount := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			hexCount++
+		}
+	}
+
+	if hexCount == 0 {
+		return s
+	}
+
+	var buf [64]byte
+	var t []byte
+
+	required := len(s) + 2*hexCount
+	if required <= len(buf) {
+		t = buf[:required]
+	} else {
+		t = make([]byte, required)
+	}
+
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case shouldEscape(c):
+			t[j] = '%'
+			t[j+1] = upperhex[c>>4]
+			t[j+2] = upperhex[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+
+	return string(t)
+}
+
+// Checks if the character needs to be escaped
+func shouldEscape(c byte) bool {
+	if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '_', '.', '~', '/':
+		return false
+	}
+
+	return true
+}

s3api/utils/utils_test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package utils
 
 import (
@@ -5,9 +19,13 @@ import (
 	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3response"
 )
 
 func TestCreateHttpRequestFromCtx(t *testing.T) {
@@ -31,11 +49,11 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 	request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
 
 	tests := []struct {
-		name    string
 		args    args
 		want    *http.Request
-		wantErr bool
+		name    string
 		hdrs    []string
+		wantErr bool
 	}{
 		{
 			name: "Success-response",
@@ -83,9 +101,9 @@ func TestGetUserMetaData(t *testing.T) {
 	req := ctx.Request()
 
 	tests := []struct {
-		name         string
 		args         args
 		wantMetadata map[string]string
+		name         string
 	}{
 		{
 			name: "Success-empty-response",
@@ -264,3 +282,248 @@ func TestParseUint(t *testing.T) {
 		})
 	}
 }
+
+func TestFilterObjectAttributes(t *testing.T) {
+	type args struct {
+		attrs  map[s3response.ObjectAttributes]struct{}
+		output s3response.GetObjectAttributesResponse
+	}
+
+	etag, objSize := "etag", int64(3222)
+	delMarker := true
+
+	tests := []struct {
+		name string
+		args args
+		want s3response.GetObjectAttributesResponse
+	}{
+		{
+			name: "keep only ETag",
+			args: args{
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag: {},
+				},
+				output: s3response.GetObjectAttributesResponse{
+					ObjectSize: &objSize,
+					ETag:       &etag,
+				},
+			},
+			want: s3response.GetObjectAttributesResponse{ETag: &etag},
+		},
+		{
+			name: "keep multiple props",
+			args: args{
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag:         {},
+					s3response.ObjectAttributesObjectSize:   {},
+					s3response.ObjectAttributesStorageClass: {},
+				},
+				output: s3response.GetObjectAttributesResponse{
+					ObjectSize:  &objSize,
+					ETag:        &etag,
+					ObjectParts: &s3response.ObjectParts{},
+					VersionId:   &etag,
+				},
+			},
+			want: s3response.GetObjectAttributesResponse{
+				ETag:       &etag,
+				ObjectSize: &objSize,
+			},
+		},
+		{
+			name: "make sure LastModified, DeleteMarker and VersionId are removed",
+			args: args{
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag: {},
+				},
+				output: s3response.GetObjectAttributesResponse{
+					ObjectSize:   &objSize,
+					ETag:         &etag,
+					ObjectParts:  &s3response.ObjectParts{},
+					VersionId:    &etag,
+					LastModified: backend.GetTimePtr(time.Now()),
+					DeleteMarker: &delMarker,
+				},
+			},
+			want: s3response.GetObjectAttributesResponse{
+				ETag: &etag,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := FilterObjectAttributes(tt.args.attrs, tt.args.output); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("FilterObjectAttributes() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsValidOwnership(t *testing.T) {
+	type args struct {
+		val types.ObjectOwnership
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "valid-BucketOwnerEnforced",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerEnforced,
+			},
+			want: true,
+		},
+		{
+			name: "valid-BucketOwnerPreferred",
+			args: args{
+				val: types.ObjectOwnershipBucketOwnerPreferred,
+			},
+			want: true,
+		},
+		{
+			name: "valid-ObjectWriter",
+			args: args{
+				val: types.ObjectOwnershipObjectWriter,
+			},
+			want: true,
+		},
+		{
+			name: "invalid_value",
+			args: args{
+				val: types.ObjectOwnership("invalid_value"),
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidOwnership(tt.args.val); got != tt.want {
+				t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_shouldEscape(t *testing.T) {
+	type args struct {
+		c byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "shouldn't-escape-alphanum",
+			args: args{
+				c: 'h',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-char",
+			args: args{
+				c: '_',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-unreserved-number",
+			args: args{
+				c: '0',
+			},
+			want: false,
+		},
+		{
+			name: "shouldn't-escape-path-separator",
+			args: args{
+				c: '/',
+			},
+			want: false,
+		},
+		{
+			name: "should-escape-special-char-1",
+			args: args{
+				c: '&',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-2",
+			args: args{
+				c: '*',
+			},
+			want: true,
+		},
+		{
+			name: "should-escape-special-char-3",
+			args: args{
+				c: '(',
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldEscape(tt.args.c); got != tt.want {
+				t.Errorf("shouldEscape() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_escapePath(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "empty-string",
+			args: args{
+				s: "",
+			},
+			want: "",
+		},
+		{
+			name: "alphanum-path",
+			args: args{
+				s: "/test-bucket/test-key",
+			},
+			want: "/test-bucket/test-key",
+		},
+		{
+			name: "path-with-unescapable-chars",
+			args: args{
+				s: "/test~bucket/test.key",
+			},
+			want: "/test~bucket/test.key",
+		},
+		{
+			name: "path-with-escapable-chars",
+			args: args{
+				s: "/bucket-*(/test=key&",
+			},
+			want: "/bucket-%2A%28/test%3Dkey%26",
+		},
+		{
+			name: "path-with-space",
+			args: args{
+				s: "/test-bucket/my key",
+			},
+			want: "/test-bucket/my%20key",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := escapePath(tt.args.s); got != tt.want {
+				t.Errorf("escapePath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -57,6 +57,7 @@ const (
 	ErrAccessDenied
 	ErrMethodNotAllowed
 	ErrBucketNotEmpty
+	ErrVersionedBucketNotEmpty
 	ErrBucketAlreadyExists
 	ErrBucketAlreadyOwnedByYou
 	ErrNoSuchBucket
@@ -65,10 +66,13 @@ const (
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidMaxKeys
+	ErrInvalidMaxBuckets
 	ErrInvalidMaxUploads
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
+	ErrInvalidObjectAttributes
 	ErrInvalidPart
+	ErrInvalidPartNumber
 	ErrInternalError
 	ErrInvalidCopyDest
 	ErrInvalidCopySource
@@ -111,12 +115,44 @@ const (
 	ErrInvalidObjectState
 	ErrInvalidRange
 	ErrInvalidURI
+	ErrObjectLockConfigurationNotFound
+	ErrNoSuchObjectLockConfiguration
+	ErrInvalidBucketObjectLockConfiguration
+	ErrObjectLockConfigurationNotAllowed
+	ErrObjectLocked
+	ErrPastObjectLockRetainDate
+	ErrObjectLockInvalidRetentionPeriod
+	ErrNoSuchBucketPolicy
+	ErrBucketTaggingNotFound
+	ErrObjectLockInvalidHeaders
+	ErrObjectAttributesInvalidHeader
+	ErrRequestTimeTooSkewed
+	ErrInvalidBucketAclWithObjectOwnership
+	ErrBothCannedAndHeaderGrants
+	ErrOwnershipControlsNotFound
+	ErrAclNotSupported
+	ErrMalformedACL
+	ErrUnexpectedContent
+	ErrMissingSecurityHeader
+	ErrInvalidMetadataDirective
+	ErrKeyTooLong
+	ErrInvalidVersionId
+	ErrNoSuchVersion
+	ErrSuspendedVersioningNotAllowed
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
 	ErrQuotaExceeded
+	ErrVersioningNotConfigured
+
+	// Admin api errors
+	ErrAdminAccessDenied
+	ErrAdminUserNotFound
+	ErrAdminUserExists
+	ErrAdminInvalidUserRole
+	ErrAdminMissingUserAcess
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -132,7 +168,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
-		Description:    "The bucket you tried to delete is not empty",
+		Description:    "The bucket you tried to delete is not empty.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrVersionedBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty. You must delete all versions in the bucket.",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrBucketAlreadyExists: {
@@ -155,19 +196,24 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The Content-Md5 you specified is not valid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidMaxBuckets: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-buckets must be an integer between 1 and 10000.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidMaxUploads: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxKeys: {
 		Code:           "InvalidArgument",
-		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartNumberMarker: {
@@ -175,9 +221,14 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Argument partNumberMarker must be an integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidObjectAttributes: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid attribute name specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchBucket: {
 		Code:           "NoSuchBucket",
-		Description:    "The specified bucket does not exist",
+		Description:    "The specified bucket does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchKey: {
@@ -200,6 +251,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidPartNumber: {
+		Code:           "InvalidArgument",
+		Description:    "Part number must be an integer between 1 and 10000, inclusive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -242,7 +298,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
-		Description:    "Invalid according to Policy: Policy Condition failed",
+		Description:    "Invalid according to Policy: Policy Condition failed.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrEntityTooSmall: {
@@ -277,7 +333,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrMalformedPresignedDate: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
+		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSignHeadersTag: {
@@ -292,7 +348,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
-		Description:    "There were headers present in the request which were not signed",
+		Description:    "There were headers present in the request which were not signed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidQueryParams: {
@@ -307,32 +363,32 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrExpiredPresignRequest: {
 		Code:           "AccessDenied",
-		Description:    "Request has expired",
+		Description:    "Request has expired.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMalformedExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires should be a number",
+		Description:    "X-Amz-Expires should be a number.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNegativeExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires must be non-negative",
+		Description:    "X-Amz-Expires must be non-negative.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
+		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
-		Description:    "The access key ID you provided does not exist in our records.",
+		Description:    "The AWS Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrRequestNotReadyYet: {
 		Code:           "AccessDenied",
-		Description:    "Request is not valid yet",
+		Description:    "Request is not valid yet.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureDoesNotMatch: {
@@ -342,17 +398,17 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrSignatureDateDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
+		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureTerminationStr: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'",
+		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrSignatureIncorrService: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "Credential should be scoped to correct service: s3",
+		Description:    "Credential should be scoped to correct service: s3.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrContentSHA256Mismatch: {
@@ -362,32 +418,32 @@ var errorCodeResponse = map[ErrorCode]APIError{
 	},
 	ErrMissingDateHeader: {
 		Code:           "AccessDenied",
-		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidRequest: {
 		Code:           "InvalidRequest",
-		Description:    "Invalid Request",
+		Description:    "Invalid Request.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAuthNotSetup: {
 		Code:           "InvalidRequest",
-		Description:    "Signed request requires setting up SeaweedFS S3 authentication",
+		Description:    "Signed request requires setting up SeaweedFS S3 authentication.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
-		Description:    "A header you provided implies functionality that is not implemented",
+		Description:    "A header you provided implies functionality that is not implemented.",
 		HTTPStatusCode: http.StatusNotImplemented,
 	},
 	ErrPreconditionFailed: {
 		Code:           "PreconditionFailed",
-		Description:    "At least one of the pre-conditions you specified did not hold",
+		Description:    "At least one of the pre-conditions you specified did not hold.",
 		HTTPStatusCode: http.StatusPreconditionFailed,
 	},
 	ErrInvalidObjectState: {
 		Code:           "InvalidObjectState",
-		Description:    "The operation is not valid for the current state of the object",
+		Description:    "The operation is not valid for the current state of the object.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrInvalidRange: {
@@ -400,6 +456,128 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The specified URI couldn't be parsed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectLockConfigurationNotFound: {
+		Code:           "ObjectLockConfigurationNotFoundError",
+		Description:    "Object Lock configuration does not exist for this bucket.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchObjectLockConfiguration: {
+		Code:           "NoSuchObjectLockConfiguration",
+		Description:    "The specified object does not have a ObjectLock configuration.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidBucketObjectLockConfiguration: {
+		Code:           "InvalidRequest",
+		Description:    "Bucket is missing Object Lock Configuration.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectLockConfigurationNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrObjectLocked: {
+		Code:           "InvalidRequest",
+		Description:    "Object is WORM protected and cannot be overwritten.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPastObjectLockRetainDate: {
+		Code:           "InvalidRequest",
+		Description:    "the retain until date must be in the future.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectLockInvalidRetentionPeriod: {
+		Code:           "InvalidRetentionPeriod",
+		Description:    "the retention days/years must be positive integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNoSuchBucketPolicy: {
+		Code:           "NoSuchBucketPolicy",
+		Description:    "The bucket policy does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrBucketTaggingNotFound: {
+		Code:           "NoSuchTagSet",
+		Description:    "The TagSet does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrObjectLockInvalidHeaders: {
+		Code:           "InvalidRequest",
+		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectAttributesInvalidHeader: {
+		Code:           "InvalidRequest",
+		Description:    "The x-amz-object-attributes header specifying the attributes to be retrieved is either missing or empty",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrRequestTimeTooSkewed: {
+		Code:           "RequestTimeTooSkewed",
+		Description:    "The difference between the request time and the server's time is too large.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidBucketAclWithObjectOwnership: {
+		Code:           "ErrInvalidBucketAclWithObjectOwnership",
+		Description:    "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBothCannedAndHeaderGrants: {
+		Code:           "InvalidRequest",
+		Description:    "Specifying both Canned ACLs and Header Grants is not allowed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOwnershipControlsNotFound: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAclNotSupported: {
+		Code:           "AccessControlListNotSupported",
+		Description:    "The bucket does not allow ACLs.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedACL: {
+		Code:           "MalformedACLError",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrUnexpectedContent: {
+		Code:           "UnexpectedContent",
+		Description:    "This request does not support content.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSecurityHeader: {
+		Code:           "MissingSecurityHeader",
+		Description:    "Your request was missing a required header.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrInvalidMetadataDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown metadata directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidVersionId: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid version id specified",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrKeyTooLong: {
+		Code:           "KeyTooLongError",
+		Description:    "Your key is too long.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNoSuchVersion: {
+		Code:           "NoSuchVersion",
+		Description:    "The specified version does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrSuspendedVersioningNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	// non aws errors
 	ErrExistingObjectIsDirectory: {
 		Code:           "ExistingObjectIsDirectory",
 		Description:    "Existing Object is a directory.",
@@ -420,6 +598,38 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Your request was denied due to quota exceeded.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrVersioningNotConfigured: {
+		Code:           "VersioningNotConfigured",
+		Description:    "Versioning has not been configured for the gateway.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+
+	// Admin api errors
+	ErrAdminAccessDenied: {
+		Code:           "XAdminAccessDenied",
+		Description:    "Only admin users have access to this resource.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAdminUserNotFound: {
+		Code:           "XAdminUserNotFound",
+		Description:    "No user exists with the provided access key ID.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminUserExists: {
+		Code:           "XAdminUserExists",
+		Description:    "A user with the provided access key ID already exists.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAdminInvalidUserRole: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User role has to be one of the following: 'user', 'admin', 'userplus'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminMissingUserAcess: {
+		Code:           "XAdminInvalidArgument",
+		Description:    "User access key ID is missing.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.