test: more test conversions/direct checks

fix: fixes PutObjectRetention error cases and object lock error code/message.

.dockerignore

@@ -0,0 +1,46 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+cmd/versitygw/versitygw
+/versitygw
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# ignore IntelliJ directories
+.idea
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+/versitygw.spec.in
+*.tar
+*.tar.gz
+**/rand.data
+/profile.txt
+
+dist/
+
+# Release config files
+/.github
+
+# Docker configuration files
+*Dockerfile
+/docker-compose.yml
+
+# read files
+/LICENSE
+/NOTICE
+/CODE_OF_CONDUCT.md
+/README.md

.env.dev

@@ -0,0 +1,8 @@
+POSIX_PORT=7071
+PROXY_PORT=7070
+ACCESS_KEY_ID=user
+SECRET_ACCESS_KEY=pass
+IAM_DIR=.
+SETUP_DIR=.
+AZ_ACCOUNT_NAME=devstoreaccount1
+AZ_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,23 @@
+---
+name: Bug Report
+about: Create a report to help us improve
+title: '[Bug] - <Short Description>'
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior. -->
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Server Version**
+<!-- output of: './versitygw -version && uname -a'  -->
+
+**Additional context**
+<!-- Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature Request
+about: Suggest an idea for this project
+title: '[Feature] - <Short Description>'
+labels: enhancement
+assignees: ''
+
+---
+
+**Describe the solution you'd like**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/test_case.md

@@ -0,0 +1,33 @@
+---
+name: Test Case Request
+about: Request new test cases or additional test coverage
+title: '[Test Case] - <Short Description>'
+labels: 'testcase'
+assignees: ''
+
+---
+
+## Description
+<!-- Please provide a detailed description of the test case or test coverage request. -->
+
+## Purpose
+<!-- Explain why this test case is important and what it aims to achieve. -->
+
+## Scope
+<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
+
+## Acceptance Criteria
+<!-- List the criteria that must be met for the test case to be considered complete. -->
+
+1. 
+2. 
+3. 
+
+## Additional Context
+<!-- Add any other context or screenshots about the feature request here. -->
+
+## Resources
+<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
+
+
+**Thank you for contributing to our project!**

.github/SECURITY.md

@@ -0,0 +1,25 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+If you discover a security vulnerability in `versitygw`, we strongly encourage you to report it privately and responsibly.
+
+Please do **not** create public issues or pull requests that contain details about the vulnerability.
+
+Instead, report the issue using GitHub's private **Security Advisories** feature:
+
+- Go to [versitygw's Security Advisories page](https://github.com/versity/versitygw/security/advisories)
+- Click on **"Report a vulnerability"**
+
+We aim to respond within **2 business days** and work with you to quickly resolve the issue.
+
+## Supported Versions
+
+| Version         | Supported |
+| --------------- | --------- |
+| Latest (v1.x.x) | ✅        |
+| Older versions  | ❌        |
+
+## Responsible Disclosure
+
+We appreciate responsible disclosures and are committed to fixing vulnerabilities in a timely manner. Thank you for helping keep `versitygw` secure.

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      dev-dependencies:
+        patterns:
+          - "*"
+    allow:
+      # Allow both direct and indirect updates for all packages
+      - dependency-type: "all"
+

.github/workflows/azurite.yml

@@ -0,0 +1,37 @@
+name: azurite functional tests
+permissions: {}
+on: pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+              
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Set up Docker Compose
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
+
+      - name: Wait for Azurite to be ready
+        run: sleep 40
+
+      - name: Get Dependencies
+        run: |
+          go mod download
+  
+      - name: Build and Run
+        run: |
+          make
+          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
+  
+      - name: Shut down services
+        run: |
+          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/docker-bats.yml

@@ -0,0 +1,29 @@
+name: docker bats tests
+permissions: {}
+on: pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        run: |
+          cp tests/.env.docker.default tests/.env.docker
+          cp tests/.secrets.default tests/.secrets
+          # see https://github.com/versity/versitygw/issues/1034
+          docker build \
+            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
+            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
+            --build-arg="MC_FOLDER=linux-amd64" \
+            --progress=plain \
+            -f tests/Dockerfile_test_bats \
+            -t bats_test .
+
+      - name: Run Docker Container
+        run: |
+          docker compose -f tests/docker-compose-bats.yml --project-directory . \
+          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/docker.yml

@@ -0,0 +1,56 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registries:
+    name: Push Docker image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            versity/versitygw
+            ghcr.io/${{ github.repository }}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            VERSION=${{ github.event.release.tag_name }}
+            TIME=${{ github.event.release.published_at }}
+            BUILD=${{ github.sha }}

.github/workflows/functional.yml

@@ -0,0 +1,31 @@
+name: functional tests
+permissions: {}
+on: pull_request
+
+jobs:
+  build:
+    name: RunTests
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 'stable'
+      id: go
+
+    - name: Get Dependencies
+      run: |
+        go mod download
+
+    - name: Build and Run
+      run: |
+        make testbin
+        ./runtests.sh
+
+    - name: Coverage Report
+      run: |
+        go tool covdata percent -i=/tmp/covdata

.github/workflows/go.yml

@@ -0,0 +1,58 @@
+name: general
+permissions: {}
+on: pull_request
+jobs:
+
+  build:
+    name: Go Basic Checks
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 'stable'
+      id: go
+
+    - name: Verify all files pass gofmt formatting
+      run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Test
+      run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
+
+    - name: Install govulncheck
+      run: go install golang.org/x/vuln/cmd/govulncheck@latest
+      shell: bash
+
+    - name: Run govulncheck
+      run: govulncheck ./...
+      shell: bash
+
+  verify-build:
+    name: Verify Build Targets
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [darwin, freebsd, linux]
+        arch: [amd64, arm64]
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+
+      - name: Build for ${{ matrix.os }}/${{ matrix.arch }}
+        run: |
+          GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o versitygw-${{ matrix.os }}-${{ matrix.arch }} cmd/versitygw/*.go

.github/workflows/goreleaser.yml

@@ -0,0 +1,34 @@
+name: goreleaser
+permissions:
+  contents: write
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Fetch tags
+        run: git fetch --force --tags
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
+      - name: Run Releaser
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          distribution: goreleaser
+          version: '~> v2'
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/host-style-tests.yml

@@ -0,0 +1,13 @@
+name: host style tests
+permissions: {}
+on: pull_request
+
+jobs:
+  build-and-run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: run host-style tests
+        run: make test-host-style

.github/workflows/shellcheck.yml

@@ -0,0 +1,17 @@
+name: shellcheck
+permissions: {}
+on: pull_request
+jobs:
+
+  build:
+    name: Run shellcheck
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Run checks
+      run: |
+        shellcheck --version
+        shellcheck -e SC1091 tests/*.sh tests/*/*.sh

.github/workflows/static.yml

@@ -0,0 +1,25 @@
+name: staticcheck
+permissions: {}
+on: pull_request
+jobs: 
+
+  build: 
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 'stable'
+      id: go
+ 
+    - name: "staticcheck"
+      uses: dominikh/staticcheck-action@v1
+      with:
+        version: "latest"

.github/workflows/system.yml

@@ -0,0 +1,263 @@
+name: system tests
+permissions: {}
+on: pull_request
+jobs:
+  build:
+    name: RunTests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - set: "mc, posix, non-file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "mc-non-file-count"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "mc, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "mc-file-count"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "REST, posix, non-static, base|acl|multipart|put-object, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "REST, posix, non-static, chunked|checksum|versioning|bucket, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "REST, posix, non-static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "REST, posix, static, base|acl|multipart|put-object, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          - set: "REST, posix, static, chunked|checksum|versioning|bucket, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          - set: "REST, posix, static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          - set: "s3, posix, non-file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-non-file-count"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "s3, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-file-count"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, non-static, s3 IAM"
+            IAM_TYPE: s3
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, multipart, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-multipart"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, object, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-object"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "false"
+            DELETE_BUCKETS_AFTER_TEST: "false"
+            BACKEND: "posix"
+          # TODO fix/debug s3 gateway
+          #- set: "s3api, s3, multipart|object, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
+          #- set: "s3api, s3, policy|user, non-static, folder IAM"
+          #  IAM_TYPE: folder
+          #  RUN_SET: "s3api-policy,s3api-user"
+          #  RECREATE_BUCKETS: "true"
+          #  BACKEND: "s3"
+          - set: "s3cmd, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-file-count"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, non-user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-non-user"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-user"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+          - set: "setup/remove static buckets scripts"
+            IAM_TYPE: folder
+            RUN_SET: "setup-remove-static"
+            RECREATE_BUCKETS: "true"
+            DELETE_BUCKETS_AFTER_TEST: "true"
+            BACKEND: "posix"
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "stable"
+        id: go
+
+      - name: Get Dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Install BATS
+        run: |
+          git clone https://github.com/bats-core/bats-core.git
+          cd bats-core && ./install.sh $HOME
+          git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
+          git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
+
+      - name: Install s3cmd
+        run: |
+          sudo apt-get update
+          sudo apt-get install s3cmd
+
+      - name: Install mc
+        run: |
+          curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
+          chmod 755 /usr/local/bin/mc
+
+      - name: Install xml libraries (for rest)
+        run: |
+          sudo apt-get update
+          sudo apt-get install libxml2-utils xmlstarlet
+
+      # see https://github.com/versity/versitygw/issues/1034
+      - name: Install AWS cli
+        run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" 
+          unzip -o awscliv2.zip 
+          ./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+      - name: Build and run
+        env:
+          IAM_TYPE: ${{ matrix.IAM_TYPE }}
+          RUN_SET: ${{ matrix.RUN_SET }}
+          AWS_PROFILE: versity
+          VERSITY_EXE: ${{ github.workspace }}/versitygw
+          RUN_VERSITYGW: true
+          BACKEND: ${{ matrix.BACKEND }}
+          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
+          DELETE_BUCKETS_AFTER_TEST: ${{ matrix.DELETE_BUCKETS_AFTER_TEST }}
+          CERT: ${{ github.workspace }}/cert.pem
+          KEY: ${{ github.workspace }}/versitygw.pem
+          LOCAL_FOLDER: /tmp/gw
+          BUCKET_ONE_NAME: versity-gwtest-bucket-one
+          BUCKET_TWO_NAME: versity-gwtest-bucket-two
+          USERS_FOLDER: /tmp/iam
+          USERS_BUCKET: versity-gwtest-iam
+          AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          PORT: 7070
+          S3CMD_CONFIG: tests/s3cfg.local.default
+          MC_ALIAS: versity
+          LOG_LEVEL: 4
+          GOCOVERDIR: ${{ github.workspace }}/cover
+          USERNAME_ONE: ABCDEFG
+          PASSWORD_ONE: 1234567
+          USERNAME_TWO: HIJKLMN
+          PASSWORD_TWO: 8901234
+          TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
+          REMOVE_TEST_FILE_FOLDER: true
+          VERSIONING_DIR: ${{ github.workspace }}/versioning
+          COMMAND_LOG: command.log
+          TIME_LOG: time.log
+          PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
+          AUTOGENERATE_USERS: true
+          USER_AUTOGENERATION_PREFIX: github-actions-test-
+        run: |
+          make testbin
+          export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+          export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+          export AWS_REGION=us-east-1
+          export AWS_ACCESS_KEY_ID_TWO=user
+          export AWS_SECRET_ACCESS_KEY_TWO=pass
+          export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
+          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
+          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
+          aws configure set aws_region $AWS_REGION --profile versity
+          mkdir $LOCAL_FOLDER
+          export WORKSPACE=$GITHUB_WORKSPACE
+          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          mkdir $GOCOVERDIR $USERS_FOLDER
+          if [[ $RECREATE_BUCKETS == "false" ]]; then
+            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
+          fi
+          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
+
+      - name: Time report
+        run: | 
+          if [ -e ${{ github.workspace }}/time.log ]; then
+            cat ${{ github.workspace }}/time.log
+          fi
+
+      - name: Coverage report
+        run: |
+          go tool covdata percent -i=cover

.gitignore

@@ -7,6 +7,8 @@
 *.dll
 *.so
 *.dylib
+cmd/versitygw/versitygw
+/versitygw
 
 # Test binary, built with `go test -c`
 *.test
@@ -19,3 +21,49 @@
 
 # Go workspace file
 go.work
+
+# ignore IntelliJ directories
+.idea
+
+# ignore VS code directories
+.vscode
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+*.tar
+*.tar.gz
+**/rand.data
+/profile.txt
+
+dist/
+
+# secrets file for local github-actions testing
+tests/.secrets*
+
+# IAM users files often created in testing
+users.json
+users.json.backup
+
+# env files for testing
+**/.env*
+**/!.env.default
+
+# s3cmd config files (testing)
+tests/s3cfg.local*
+tests/!s3cfg.local.default
+
+# keys
+*.pem
+
+# patches
+*.patch
+
+# grafana's local database (kept on filesystem for survival between instantiations)
+metrics-exploration/grafana_data/**
+
+# bats tools
+/tests/bats-assert
+/tests/bats-support

.goreleaser.yaml

@@ -0,0 +1,128 @@
+version: 2
+
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - goos:
+      - linux
+      - darwin
+      - freebsd
+      # windows is untested, we can start doing windows releases
+      # if someone is interested in taking on testing
+      # - windows
+    env:
+      # disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
+      # once we need to enable this, we will need to do per distro releases
+      - CGO_ENABLED=0
+    main: ./cmd/versitygw
+    binary: versitygw
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
+
+archives:
+  - formats: [ 'tar.gz' ]
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_v{{ .Version }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+
+    # Set this to true if you want all files in the archive to be in a single directory.
+    # If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
+    # you'll get a folder 'goreleaser_Linux_arm64'.
+    # If set to false, all files are extracted separately.
+    # You can also set it to a custom folder name (templating is supported).
+    wrap_in_directory: true
+
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      formats: [ 'zip' ]
+
+    # Additional files/globs you want to add to the archive.
+    #
+    # Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
+    # Templates: allowed
+    files:
+      - README.md
+      - LICENSE
+      - NOTICE
+
+checksum:
+  name_template: 'checksums.txt'
+
+snapshot:
+  version_template: "{{ incpatch .Version }}-{{.ShortCommit}}"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge '
+
+nfpms:
+  - id: packages
+    package_name: versitygw
+    vendor: Versity Software
+    homepage: https://github.com/versity/versitygw
+    maintainer: Ben McClelland <ben.mcclelland@versity.com>
+
+    description: |-
+      The Versity S3 Gateway.
+      A high-performance tool facilitating translation between AWS S3 API
+      requests and various backend storage systems, including POSIX file
+      backend storage. Its stateless architecture enables deployment in
+      clusters for increased throughput, distributing requests across gateways
+      for optimal performance. With a focus on modularity, it supports future
+      extensions for additional backend systems.
+
+    license: Apache 2.0
+
+    ids:
+      - versitygw
+
+    formats:
+      - deb
+      - rpm
+
+    umask: 0o002
+    bindir: /usr/bin
+    epoch: "1"
+    release: "1"
+
+    rpm:
+      group: "System Environment/Daemons"
+      # RPM specific scripts.
+      scripts:
+        # The pretrans script runs before all RPM package transactions / stages.
+        #pretrans: ./extra/pretrans.sh
+        # The posttrans script runs after all RPM package transactions / stages.
+        posttrans: ./extra/posttrans.sh
+
+    contents:
+      - src: extra/versitygw@.service
+        dst: /lib/systemd/system/versitygw@.service
+
+      - src: extra/example.conf
+        dst: /etc/versitygw.d/example.conf
+        type: config
+  
+      - dst: /etc/versitygw.d
+        type: dir
+        file_info:
+          mode: 0700
+
+
+
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+versitygw@versity.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

Dockerfile

@@ -0,0 +1,35 @@
+FROM golang:latest
+
+# Set build arguments with default values
+ARG VERSION="none"
+ARG BUILD="none"
+ARG TIME="none"
+
+# Set environment variables
+ENV VERSION=${VERSION}
+ENV BUILD=${BUILD}
+ENV TIME=${TIME}
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY ./ ./
+
+WORKDIR /app/cmd/versitygw
+ENV CGO_ENABLED=0
+RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
+
+FROM alpine:latest
+
+# These arguments can be overriden when building the image
+ARG IAM_DIR=/tmp/vgw
+ARG SETUP_DIR=/tmp/vgw
+
+RUN mkdir -p $IAM_DIR
+RUN mkdir -p $SETUP_DIR
+
+COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
+
+ENTRYPOINT [ "/app/versitygw" ]

Makefile

@@ -0,0 +1,104 @@
+# Copyright 2023 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOCLEAN=$(GOCMD) clean
+GOTEST=$(GOCMD) test
+
+# docker-compose
+DCCMD=docker-compose
+DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
+
+BIN=versitygw
+
+VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
+BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
+TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
+
+LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
+
+all: build
+
+build: $(BIN)
+
+.PHONY: $(BIN)
+$(BIN):
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
+
+testbin:
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
+
+.PHONY: test
+test: 
+	$(GOTEST) ./...
+
+.PHONY: check
+check:
+# note this requires staticcheck be in your PATH:
+# export PATH=$PATH:~/go/bin
+# go install honnef.co/go/tools/cmd/staticcheck@latest
+	staticcheck ./...
+	golint ./...
+	gofmt -s -l .
+
+.PHONY: clean
+clean: 
+	$(GOCLEAN)
+
+.PHONY: cleanall
+cleanall: clean
+	rm -f $(BIN)
+	rm -f versitygw-*.tar
+	rm -f versitygw-*.tar.gz
+
+TARFILE = $(BIN)-$(VERSION).tar
+
+dist:
+	echo $(VERSION) >VERSION
+	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
+	rm -f VERSION
+	gzip -f $(TARFILE)
+
+.PHONY: snapshot
+snapshot:
+# brew install goreleaser/tap/goreleaser
+	goreleaser release --snapshot --skip publish --clean
+
+# Creates and runs S3 gateway instance in a docker container
+.PHONY: up-posix
+up-posix:
+	$(DOCKERCOMPOSE) up posix
+
+# Creates and runs S3 gateway proxy instance in a docker container
+.PHONY: up-proxy
+up-proxy:
+	$(DOCKERCOMPOSE) up proxy
+
+# Creates and runs S3 gateway to azurite instance in a docker container
+.PHONY: up-azurite
+up-azurite:
+	$(DOCKERCOMPOSE) up azurite azuritegw
+
+# Creates and runs both S3 gateway and proxy server instances in docker containers
+.PHONY: up-app
+up-app:
+	$(DOCKERCOMPOSE) up
+
+# Run the host-style tests in docker containers
+.PHONY: test-host-style
+test-host-style:
+	docker compose -f tests/host-style-tests/docker-compose.yml up --build --abort-on-container-exit --exit-code-from test
+

NOTICE

@@ -0,0 +1,2 @@
+versitygw - Versity S3 Gateway
+Copyright 2023 Versity Software

README.md

@@ -0,0 +1,87 @@
+# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
+  <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
+</picture>
+
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
+
+### Binary release builds
+Download [latest release](https://github.com/versity/versitygw/releases)
+ | Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
+ |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
+ |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
+ 
+### Use Cases
+* Turn your local filesystem into an S3 server with a single command!
+* Proxy S3 requests to S3 storage
+* Simple to deploy S3 server with a single command
+* Protocol compatibility in `posix` allows common access to files via posix or S3
+* Simplified interface for adding new storage system support
+
+### News
+Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
+
+### Mailing List
+Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
+
+### Documentation
+See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
+
+### Need help?
+Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
+<br>
+Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
+
+### Overview
+Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
+
+The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
+
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versity’s open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.  
+
+The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
+
+The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework.  This framework is actively developed with a focus on performance.  S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3. 
+
+## Getting Started
+See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
+
+### Run the gateway with posix backend:
+
+```
+mkdir /tmp/vgw
+ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
+```
+This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
+
+To get the usage output, run the following:
+
+```
+./versitygw --help
+```
+
+The command format is
+
+```
+versitygw [global options] command [command options] [arguments...]
+```
+The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
+
+***
+
+#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
+
+### Contact
+![versity logo](https://www.versity.com/wp-content/uploads/2022/12/cropped-android-chrome-512x512-1-32x32.png)
+info@versity.com <br />
++1 844 726 8826
+
+### @versitysoftware 
+[![linkedin](https://github.com/versity/versitygw/blob/assets/assets/linkedin.jpg)](https://www.linkedin.com/company/versity/) &nbsp; 
+[![twitter](https://github.com/versity/versitygw/blob/assets/assets/twitter.jpg)](https://twitter.com/VersitySoftware) &nbsp;
+[![facebook](https://github.com/versity/versitygw/blob/assets/assets/facebook.jpg)](https://www.facebook.com/versitysoftware) &nbsp;
+[![instagram](https://github.com/versity/versitygw/blob/assets/assets/instagram.jpg)](https://www.instagram.com/versitysoftware/) &nbsp;
+

auth/access-control.go

@@ -0,0 +1,189 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+)
+
+func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	// Verify destination bucket access
+	if err := VerifyAccess(ctx, be, opts); err != nil {
+		return err
+	}
+	// Verify source bucket access
+	srcBucket, srcObject, found := strings.Cut(copySource, "/")
+	if !found {
+		return s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
+	}
+
+	// Get source bucket ACL
+	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+	if err != nil {
+		return err
+	}
+
+	var srcBucketAcl ACL
+	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+		return err
+	}
+
+	if err := VerifyAccess(ctx, be, AccessOptions{
+		Acl:           srcBucketAcl,
+		AclPermission: PermissionRead,
+		IsRoot:        opts.IsRoot,
+		Acc:           opts.Acc,
+		Bucket:        srcBucket,
+		Object:        srcObject,
+		Action:        GetObjectAction,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type AccessOptions struct {
+	Acl             ACL
+	AclPermission   Permission
+	IsRoot          bool
+	Acc             Account
+	Bucket          string
+	Object          string
+	Action          Action
+	Readonly        bool
+	IsPublicRequest bool
+}
+
+func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	// Skip the access check for public bucket requests
+	if opts.IsPublicRequest {
+		return nil
+	}
+	if opts.Readonly {
+		if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+	}
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
+	if policyErr != nil {
+		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+			return policyErr
+		}
+	} else {
+		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
+	}
+
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Detects if the action is policy related
+// e.g.
+// 'GetBucketPolicy', 'PutBucketPolicy'
+func isPolicyAction(action Action) bool {
+	return action == GetBucketPolicyAction || action == PutBucketPolicyAction
+}
+
+// VerifyPublicAccess checks if the bucket is publically accessible by ACL or Policy
+func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, permission Permission, bucket, object string) error {
+	// ACL disabled
+	policy, err := be.GetBucketPolicy(ctx, bucket)
+	if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+		return err
+	}
+	if err == nil {
+		err = VerifyPublicBucketPolicy(policy, bucket, object, action)
+		if err == nil {
+			// if ACLs are disabled, and the bucket grants public access,
+			// policy actions should return 'MethodNotAllowed'
+			if isPolicyAction(action) {
+				return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+			}
+
+			return nil
+		}
+	}
+
+	// if the action is not in the ACL whitelist the access is denied
+	_, ok := publicACLAllowedActions[action]
+	if !ok {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	err = VerifyPublicBucketACL(ctx, be, bucket, action, permission)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
+	// Owner check
+	if acct.Access == acl.Owner {
+		return nil
+	}
+
+	// Root user has access over almost everything
+	if isRoot {
+		return nil
+	}
+
+	// Admin user case
+	if acct.Role == RoleAdmin {
+		return nil
+	}
+
+	// Return access denied in all other cases
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}
+
+type PublicACLAllowedActions map[Action]struct{}
+
+var publicACLAllowedActions PublicACLAllowedActions = PublicACLAllowedActions{
+	ListBucketAction:                 struct{}{},
+	PutObjectAction:                  struct{}{},
+	ListBucketMultipartUploadsAction: struct{}{},
+	DeleteObjectAction:               struct{}{},
+	ListBucketVersionsAction:         struct{}{},
+	GetObjectAction:                  struct{}{},
+	GetObjectAttributesAction:        struct{}{},
+	GetObjectAclAction:               struct{}{},
+}

auth/acl.go

@@ -0,0 +1,495 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+)
+
+type ACL struct {
+	Owner    string
+	Grantees []Grantee
+}
+
+// IsPublic specifies if the acl grants public read access
+func (acl *ACL) IsPublic(permission Permission) bool {
+	for _, grt := range acl.Grantees {
+		if grt.Permission == permission && grt.Type == types.TypeGroup && grt.Access == "all-users" {
+			return true
+		}
+	}
+
+	return false
+}
+
+type Grantee struct {
+	Permission Permission
+	Access     string
+	Type       types.Type
+}
+
+type GetBucketAclOutput struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
+	Owner             *types.Owner
+	AccessControlList AccessControlList
+}
+
+type PutBucketAclInput struct {
+	Bucket              *string
+	ACL                 types.BucketCannedACL
+	AccessControlPolicy *AccessControlPolicy
+	GrantFullControl    *string
+	GrantRead           *string
+	GrantReadACP        *string
+	GrantWrite          *string
+	GrantWriteACP       *string
+}
+
+type AccessControlPolicy struct {
+	AccessControlList AccessControlList `xml:"AccessControlList"`
+	Owner             *types.Owner
+}
+
+func (acp *AccessControlPolicy) Validate() error {
+	if !acp.AccessControlList.isValid() {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner can't be nil
+	if acp.Owner == nil {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	// The Owner ID can't be empty
+	if acp.Owner.ID == nil || *acp.Owner.ID == "" {
+		return s3err.GetAPIError(s3err.ErrMalformedACL)
+	}
+
+	return nil
+}
+
+type AccessControlList struct {
+	Grants []Grant `xml:"Grant"`
+}
+
+// Validates the AccessControlList
+func (acl *AccessControlList) isValid() bool {
+	for _, el := range acl.Grants {
+		if !el.isValid() {
+			return false
+		}
+	}
+
+	return true
+}
+
+type Permission string
+
+const (
+	PermissionFullControl Permission = "FULL_CONTROL"
+	PermissionWrite       Permission = "WRITE"
+	PermissionWriteAcp    Permission = "WRITE_ACP"
+	PermissionRead        Permission = "READ"
+	PermissionReadAcp     Permission = "READ_ACP"
+)
+
+// Check if the permission is valid
+func (p Permission) isValid() bool {
+	return p == PermissionFullControl ||
+		p == PermissionRead ||
+		p == PermissionReadAcp ||
+		p == PermissionWrite ||
+		p == PermissionWriteAcp
+}
+
+type Grant struct {
+	Grantee    *Grt       `xml:"Grantee"`
+	Permission Permission `xml:"Permission"`
+}
+
+// Checks if Grant is valid
+func (g *Grant) isValid() bool {
+	return g.Permission.isValid() && g.Grantee.isValid()
+}
+
+type Grt struct {
+	XMLNS string     `xml:"xmlns:xsi,attr"`
+	Type  types.Type `xml:"xsi:type,attr"`
+	ID    string     `xml:"ID"`
+}
+
+// Custom Unmarshalling for Grt to parse xsi:type properly
+func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Iterate through the XML tokens to process the attributes
+	for _, attr := range start.Attr {
+		// Check if the attribute is xsi:type and belongs to the xsi namespace
+		if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
+			g.Type = types.Type(attr.Value)
+		}
+		// Handle xmlns:xsi
+		if attr.Name.Local == "xmlns:xsi" {
+			g.XMLNS = attr.Value
+		}
+	}
+
+	// Decode the inner XML elements like ID
+	for {
+		t, err := d.Token()
+		if err != nil {
+			return err
+		}
+
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "ID" {
+				if err := d.DecodeElement(&g.ID, &se); err != nil {
+					return err
+				}
+			}
+		case xml.EndElement:
+			if se.Name.Local == start.Name.Local {
+				return nil
+			}
+		}
+	}
+}
+
+// Validates Grt
+func (g *Grt) isValid() bool {
+	// Validate the Type
+	// Only these 2 types are supported in the gateway
+	if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
+		return false
+	}
+
+	// The ID prop shouldn't be empty
+	if g.ID == "" {
+		return false
+	}
+
+	return true
+}
+
+func ParseACL(data []byte) (ACL, error) {
+	if len(data) == 0 {
+		return ACL{}, nil
+	}
+
+	var acl ACL
+	if err := json.Unmarshal(data, &acl); err != nil {
+		return acl, fmt.Errorf("parse acl: %w", err)
+	}
+	return acl, nil
+}
+
+func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
+	grants := []Grant{}
+
+	if len(data) == 0 {
+		return GetBucketAclOutput{
+			Owner: &types.Owner{
+				ID: &owner,
+			},
+			AccessControlList: AccessControlList{
+				Grants: grants,
+			},
+		}, nil
+	}
+
+	var acl ACL
+	if err := json.Unmarshal(data, &acl); err != nil {
+		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
+	}
+
+	for _, elem := range acl.Grantees {
+		acs := elem.Access
+		grants = append(grants, Grant{
+			Grantee: &Grt{
+				XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
+				ID:    acs,
+				Type:  elem.Type,
+			},
+			Permission: elem.Permission,
+		})
+	}
+
+	return GetBucketAclOutput{
+		Owner: &types.Owner{
+			ID: &acl.Owner,
+		},
+		AccessControlList: AccessControlList{
+			Grants: grants,
+		},
+	}, nil
+}
+
+func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
+	if input == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	defaultGrantees := []Grantee{
+		{
+			Permission: PermissionFullControl,
+			Access:     acl.Owner,
+			Type:       types.TypeCanonicalUser,
+		},
+	}
+
+	// if the ACL is specified, set the ACL, else replace the grantees
+	if input.ACL != "" {
+		switch input.ACL {
+		case types.BucketCannedACLPublicRead:
+			defaultGrantees = append(defaultGrantees, Grantee{
+				Permission: PermissionRead,
+				Access:     "all-users",
+				Type:       types.TypeGroup,
+			})
+		case types.BucketCannedACLPublicReadWrite:
+			defaultGrantees = append(defaultGrantees, []Grantee{
+				{
+					Permission: PermissionRead,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+				{
+					Permission: PermissionWrite,
+					Access:     "all-users",
+					Type:       types.TypeGroup,
+				},
+			}...)
+		}
+	} else {
+		accs := []string{}
+
+		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
+			fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
+
+			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
+				fullControlList = splitUnique(*input.GrantFullControl, ",")
+				for _, str := range fullControlList {
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionFullControl,
+						Type:       types.TypeCanonicalUser,
+					})
+				}
+			}
+			if input.GrantRead != nil && *input.GrantRead != "" {
+				readList = splitUnique(*input.GrantRead, ",")
+				for _, str := range readList {
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionRead,
+						Type:       types.TypeCanonicalUser,
+					})
+				}
+			}
+			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
+				readACPList = splitUnique(*input.GrantReadACP, ",")
+				for _, str := range readACPList {
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionReadAcp,
+						Type:       types.TypeCanonicalUser,
+					})
+				}
+			}
+			if input.GrantWrite != nil && *input.GrantWrite != "" {
+				writeList = splitUnique(*input.GrantWrite, ",")
+				for _, str := range writeList {
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionWrite,
+						Type:       types.TypeCanonicalUser,
+					})
+				}
+			}
+			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
+				writeACPList = splitUnique(*input.GrantWriteACP, ",")
+				for _, str := range writeACPList {
+					defaultGrantees = append(defaultGrantees, Grantee{
+						Access:     str,
+						Permission: PermissionWriteAcp,
+						Type:       types.TypeCanonicalUser,
+					})
+				}
+			}
+
+			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
+		} else {
+			cache := make(map[string]bool)
+			for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
+				if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
+					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+				}
+
+				access := grt.Grantee.ID
+				defaultGrantees = append(defaultGrantees, Grantee{
+					Access:     access,
+					Permission: grt.Permission,
+					Type:       types.TypeCanonicalUser,
+				})
+				if _, ok := cache[access]; !ok {
+					cache[access] = true
+					accs = append(accs, access)
+				}
+			}
+		}
+
+		// Check if the specified accounts exist
+		accList, err := CheckIfAccountsExist(accs, iam)
+		if err != nil {
+			return nil, err
+		}
+		if len(accList) > 0 {
+			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
+		}
+	}
+
+	acl.Grantees = defaultGrantees
+
+	result, err := json.Marshal(acl)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
+	result := []string{}
+
+	for _, acc := range accs {
+		_, err := iam.GetUserAccount(acc)
+		if err != nil {
+			if err == ErrNoSuchUser || err == s3err.GetAPIError(s3err.ErrAdminUserNotFound) {
+				result = append(result, acc)
+				continue
+			}
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
+				return nil, err
+			}
+			return nil, fmt.Errorf("check user account: %w", err)
+		}
+	}
+	return result, nil
+}
+
+func splitUnique(s, divider string) []string {
+	elements := strings.Split(s, divider)
+	uniqueElements := make(map[string]bool)
+	result := make([]string, 0, len(elements))
+
+	for _, element := range elements {
+		if _, ok := uniqueElements[element]; !ok {
+			result = append(result, element)
+			uniqueElements[element] = true
+		}
+	}
+
+	return result
+}
+
+func verifyACL(acl ACL, access string, permission Permission) error {
+	grantee := Grantee{
+		Access:     access,
+		Permission: permission,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeFullCtrl := Grantee{
+		Access:     access,
+		Permission: PermissionFullControl,
+		Type:       types.TypeCanonicalUser,
+	}
+	granteeAllUsers := Grantee{
+		Access:     "all-users",
+		Permission: permission,
+		Type:       types.TypeGroup,
+	}
+
+	isFound := false
+
+	for _, grt := range acl.Grantees {
+		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
+			isFound = true
+			break
+		}
+	}
+
+	if isFound {
+		return nil
+	}
+
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}
+
+// Verifies if the bucket acl grants public access
+func VerifyPublicBucketACL(ctx context.Context, be backend.Backend, bucket string, action Action, permission Permission) error {
+	aclBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return err
+	}
+
+	acl, err := ParseACL(aclBytes)
+	if err != nil {
+		return err
+	}
+
+	if !acl.IsPublic(permission) {
+		return ErrAccessDenied
+	}
+
+	return nil
+}
+
+// UpdateBucketACLOwner sets default ACL with new owner and removes
+// any previous bucket policy that was in place
+func UpdateBucketACLOwner(ctx context.Context, be backend.Backend, bucket, newOwner string) error {
+	acl := ACL{
+		Owner: newOwner,
+		Grantees: []Grantee{
+			{
+				Permission: PermissionFullControl,
+				Access:     newOwner,
+				Type:       types.TypeCanonicalUser,
+			},
+		},
+	}
+
+	result, err := json.Marshal(acl)
+	if err != nil {
+		return fmt.Errorf("marshal ACL: %w", err)
+	}
+
+	err = be.PutBucketAcl(ctx, bucket, result)
+	if err != nil {
+		return err
+	}
+
+	return be.DeleteBucketPolicy(ctx, bucket)
+}

auth/bucket_cors.go

@@ -0,0 +1,338 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/versity/versitygw/debuglogger"
+	"github.com/versity/versitygw/s3err"
+)
+
+// headerRegex is the regexp to validate http header names
+var headerRegex = regexp.MustCompile(`^[!#$%&'*+\-.^_` + "`" + `|~0-9A-Za-z]+$`)
+
+type CORSHeader string
+type CORSHTTPMethod string
+
+// IsValid validates the CORS http header
+// the rules are based on http RFC
+// https://datatracker.ietf.org/doc/html/rfc7230#section-3.2
+//
+// Empty values are considered as valid
+func (ch CORSHeader) IsValid() bool {
+	return ch == "" || headerRegex.MatchString(ch.String())
+}
+
+// String converts the header value to 'string'
+func (ch CORSHeader) String() string {
+	return string(ch)
+}
+
+// ToLower converts the header to lower case
+func (ch CORSHeader) ToLower() string {
+	return strings.ToLower(string(ch))
+}
+
+// IsValid validates the cors http request method:
+// the methods are case sensitive
+func (cm CORSHTTPMethod) IsValid() bool {
+	return cm.IsEmpty() || cm == http.MethodGet || cm == http.MethodHead || cm == http.MethodPut ||
+		cm == http.MethodPost || cm == http.MethodDelete
+}
+
+// IsEmpty checks if the cors method is an empty string
+func (cm CORSHTTPMethod) IsEmpty() bool {
+	return cm == ""
+}
+
+// String converts the method value to 'string'
+func (cm CORSHTTPMethod) String() string {
+	return string(cm)
+}
+
+type CORSConfiguration struct {
+	Rules []CORSRule `xml:"CORSRule"`
+}
+
+// Validate validates the cors configuration rules
+func (cc *CORSConfiguration) Validate() error {
+	if cc == nil || cc.Rules == nil {
+		debuglogger.Logf("invalid CORS configuration")
+		return s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	if len(cc.Rules) == 0 {
+		debuglogger.Logf("empty CORS config rules")
+		return s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	// validate each CORS rule
+	for _, rule := range cc.Rules {
+		if err := rule.Validate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type CORSAllowanceConfig struct {
+	Origin           string
+	Methods          string
+	ExposedHeaders   string
+	AllowCredentials string
+	AllowHeaders     string
+	MaxAge           *int32
+}
+
+// IsAllowed walks through the CORS rules and finds the first one allowing access.
+// If no rule grants access, returns 'AccessForbidden'
+func (cc *CORSConfiguration) IsAllowed(origin string, method CORSHTTPMethod, headers []CORSHeader) (*CORSAllowanceConfig, error) {
+	// if method is empty, anyways cors is forbidden
+	// skip, without going through the rules
+	if method.IsEmpty() {
+		debuglogger.Logf("empty Access-Control-Request-Method")
+		return nil, s3err.GetAPIError(s3err.ErrCORSForbidden)
+	}
+	for _, rule := range cc.Rules {
+		// find the first rule granting access
+		if isAllowed, wilcardOrigin := rule.Match(origin, method, headers); isAllowed {
+			o := origin
+			allowCredentials := "true"
+			if wilcardOrigin {
+				o = "*"
+				allowCredentials = "false"
+			}
+
+			return &CORSAllowanceConfig{
+				Origin:           o,
+				AllowCredentials: allowCredentials,
+				Methods:          rule.GetAllowedMethods(),
+				ExposedHeaders:   rule.GetExposeHeaders(),
+				AllowHeaders:     buildAllowedHeaders(headers),
+				MaxAge:           rule.MaxAgeSeconds,
+			}, nil
+		}
+	}
+
+	// if no matching rule is found, return AccessForbidden
+	return nil, s3err.GetAPIError(s3err.ErrCORSForbidden)
+}
+
+type CORSRule struct {
+	AllowedMethods []CORSHTTPMethod `xml:"AllowedMethod"`
+	AllowedHeaders []CORSHeader     `xml:"AllowedHeader"`
+	ExposeHeaders  []CORSHeader     `xml:"ExposeHeader"`
+	AllowedOrigins []string         `xml:"AllowedOrigin"`
+	ID             *string
+	MaxAgeSeconds  *int32
+}
+
+// Validate validates and returns error if CORS configuration has invalid rule
+func (cr *CORSRule) Validate() error {
+	// validate CORS allowed headers
+	for _, header := range cr.AllowedHeaders {
+		if !header.IsValid() {
+			debuglogger.Logf("invalid CORS allowed header: %s", header)
+			return s3err.GetInvalidCORSHeaderErr(header.String())
+		}
+	}
+	// validate CORS allowed methods
+	for _, method := range cr.AllowedMethods {
+		if !method.IsValid() {
+			debuglogger.Logf("invalid CORS allowed method: %s", method)
+			return s3err.GetUnsopportedCORSMethodErr(method.String())
+		}
+	}
+	// validate CORS expose headers
+	for _, header := range cr.ExposeHeaders {
+		if !header.IsValid() {
+			debuglogger.Logf("invalid CORS exposed header: %s", header)
+			return s3err.GetInvalidCORSHeaderErr(header.String())
+		}
+	}
+
+	return nil
+}
+
+// Match matches the provided origin, method and headers with the
+// CORS configuration rule
+// if the matching origin is "*", it returns true as the first argument
+func (cr *CORSRule) Match(origin string, method CORSHTTPMethod, headers []CORSHeader) (bool, bool) {
+	wildcardOrigin := false
+	originFound := false
+
+	// check if the provided origin exists in CORS AllowedOrigins
+	for _, or := range cr.AllowedOrigins {
+		if wildcardMatch(or, origin) {
+			originFound = true
+			if or == "*" {
+				// mark wildcardOrigin as true, if "*" is found in AllowedOrigins
+				wildcardOrigin = true
+			}
+			break
+		}
+	}
+
+	if !originFound {
+		return false, false
+	}
+
+	// cache the CORS AllowedMethods in a map
+	allowedMethods := cacheCORSMethods(cr.AllowedMethods)
+	// check if the provided method exists in CORS AllowedMethods
+	if _, ok := allowedMethods[method]; !ok {
+		return false, false
+	}
+
+	// check is CORS rule allowed headers match
+	// with the requested allowed headers
+	for _, reqHeader := range headers {
+		match := false
+		for _, header := range cr.AllowedHeaders {
+			if wildcardMatch(header.ToLower(), reqHeader.ToLower()) {
+				match = true
+				break
+			}
+		}
+
+		if !match {
+			return false, false
+		}
+	}
+
+	return true, wildcardOrigin
+}
+
+// GetExposeHeaders returns comma separated CORS expose headers
+func (cr *CORSRule) GetExposeHeaders() string {
+	var result strings.Builder
+
+	for i, h := range cr.ExposeHeaders {
+		if i > 0 {
+			result.WriteString(", ")
+		}
+		result.WriteString(h.String())
+	}
+
+	return result.String()
+}
+
+// buildAllowedHeaders builds a comma separated string from []CORSHeader
+func buildAllowedHeaders(headers []CORSHeader) string {
+	var result strings.Builder
+
+	for i, h := range headers {
+		if i > 0 {
+			result.WriteString(", ")
+		}
+		result.WriteString(h.ToLower())
+	}
+
+	return result.String()
+}
+
+// GetAllowedMethods returns comma separated CORS allowed methods
+func (cr *CORSRule) GetAllowedMethods() string {
+	var result strings.Builder
+
+	for i, m := range cr.AllowedMethods {
+		if i > 0 {
+			result.WriteString(", ")
+		}
+		result.WriteString(m.String())
+	}
+
+	return result.String()
+}
+
+// ParseCORSOutput parses raw bytes to 'CORSConfiguration'
+func ParseCORSOutput(data []byte) (*CORSConfiguration, error) {
+	var config CORSConfiguration
+	err := xml.Unmarshal(data, &config)
+	if err != nil {
+		debuglogger.Logf("unmarshal cors output: %v", err)
+		return nil, fmt.Errorf("failed to parse cors config: %w", err)
+	}
+
+	return &config, nil
+}
+
+func cacheCORSMethods(input []CORSHTTPMethod) map[CORSHTTPMethod]struct{} {
+	result := make(map[CORSHTTPMethod]struct{}, len(input))
+	for _, el := range input {
+		result[el] = struct{}{}
+	}
+
+	return result
+}
+
+// ParseCORSHeaders parses/validates Access-Control-Request-Headers
+// and returns []CORSHeaders
+func ParseCORSHeaders(headers string) ([]CORSHeader, error) {
+	result := []CORSHeader{}
+	if headers == "" {
+		return result, nil
+	}
+
+	headersSplitted := strings.Split(headers, ",")
+	for _, h := range headersSplitted {
+		corsHeader := CORSHeader(strings.TrimSpace(h))
+		if corsHeader == "" || !corsHeader.IsValid() {
+			debuglogger.Logf("invalid access control header: %s", h)
+			return nil, s3err.GetInvalidCORSRequestHeaderErr(h)
+		}
+		result = append(result, corsHeader)
+	}
+
+	return result, nil
+}
+
+func wildcardMatch(pattern, input string) bool {
+	pIdx, sIdx := 0, 0
+	starIdx, matchIdx := -1, 0
+
+	for sIdx < len(input) {
+		if pIdx < len(pattern) && pattern[pIdx] == input[sIdx] {
+			// exact match of current char
+			sIdx++
+			pIdx++
+		} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
+			// remember star position
+			starIdx = pIdx
+			matchIdx = sIdx
+			pIdx++
+		} else if starIdx != -1 {
+			// backtrack: try to match more characters with '*'
+			pIdx = starIdx + 1
+			matchIdx++
+			sIdx = matchIdx
+		} else {
+			return false
+		}
+	}
+
+	// skip trailing stars
+	for pIdx < len(pattern) && pattern[pIdx] == '*' {
+		pIdx++
+	}
+
+	return pIdx == len(pattern)
+}

auth/bucket_cors_test.go

@@ -0,0 +1,736 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/versity/versitygw/s3err"
+)
+
+func TestCORSHeader_IsValid(t *testing.T) {
+	tests := []struct {
+		name   string
+		header CORSHeader
+		want   bool
+	}{
+		{"empty", "", true},
+		{"valid", "X-Custom-Header", true},
+		{"invalid_1", "Invalid Header", false},
+		{"invalid_2", "invalid/header", false},
+		{"invalid_3", "Invalid\tHeader", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.header.IsValid(); got != tt.want {
+				t.Errorf("IsValid() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCORSHTTPMethod_IsValid(t *testing.T) {
+	tests := []struct {
+		name   string
+		method CORSHTTPMethod
+		want   bool
+	}{
+		{"empty valid", "", true},
+		{"GET valid", http.MethodGet, true},
+		{"HEAD valid", http.MethodHead, true},
+		{"PUT valid", http.MethodPut, true},
+		{"POST valid", http.MethodPost, true},
+		{"DELETE valid", http.MethodDelete, true},
+		{"get valid", "get", false},
+		{"put valid", "put", false},
+		{"post valid", "post", false},
+		{"head valid", "head", false},
+		{"invalid", "FOO", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.method.IsValid(); got != tt.want {
+				t.Errorf("IsValid() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCORSHeader_ToLower(t *testing.T) {
+	tests := []struct {
+		name   string
+		header CORSHeader
+		want   string
+	}{
+		{
+			name:   "already lowercase",
+			header: CORSHeader("content-type"),
+			want:   "content-type",
+		},
+		{
+			name:   "mixed case",
+			header: CORSHeader("X-CuStOm-HeAdEr"),
+			want:   "x-custom-header",
+		},
+		{
+			name:   "uppercase",
+			header: CORSHeader("AUTHORIZATION"),
+			want:   "authorization",
+		},
+		{
+			name:   "empty string",
+			header: CORSHeader(""),
+			want:   "",
+		},
+		{
+			name:   "numeric and symbols",
+			header: CORSHeader("X-123-HEADER"),
+			want:   "x-123-header",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.header.ToLower()
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestCORSHTTPMethod_IsEmpty(t *testing.T) {
+	tests := []struct {
+		name   string
+		method CORSHTTPMethod
+		want   bool
+	}{
+		{
+			name:   "empty string is empty",
+			method: CORSHTTPMethod(""),
+			want:   true,
+		},
+		{
+			name:   "GET method is not empty",
+			method: CORSHTTPMethod("GET"),
+			want:   false,
+		},
+		{
+			name:   "random string is not empty",
+			method: CORSHTTPMethod("FOO"),
+			want:   false,
+		},
+		{
+			name:   "lowercase get is not empty (case sensitive)",
+			method: CORSHTTPMethod("get"),
+			want:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.method.IsEmpty()
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestCORSConfiguration_Validate(t *testing.T) {
+	tests := []struct {
+		name string
+		cfg  *CORSConfiguration
+		want error
+	}{
+		{"nil config", nil, s3err.GetAPIError(s3err.ErrMalformedXML)},
+		{"nil rules", &CORSConfiguration{}, s3err.GetAPIError(s3err.ErrMalformedXML)},
+		{"empty rules", &CORSConfiguration{Rules: []CORSRule{}}, s3err.GetAPIError(s3err.ErrMalformedXML)},
+		{"invalid rule", &CORSConfiguration{Rules: []CORSRule{{AllowedHeaders: []CORSHeader{"Invalid Header"}}}}, s3err.GetInvalidCORSHeaderErr("Invalid Header")},
+		{"valid rule", &CORSConfiguration{Rules: []CORSRule{{
+			AllowedOrigins: []string{"origin"},
+			AllowedHeaders: []CORSHeader{"X-Test"},
+			AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+			ExposeHeaders:  []CORSHeader{"X-Expose"},
+		}}}, nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			assert.EqualValues(t, tt.want, err)
+		})
+	}
+}
+
+func TestCORSConfiguration_IsAllowed(t *testing.T) {
+	type input struct {
+		cfg     *CORSConfiguration
+		origin  string
+		method  CORSHTTPMethod
+		headers []CORSHeader
+	}
+	type output struct {
+		result *CORSAllowanceConfig
+		err    error
+	}
+	tests := []struct {
+		name   string
+		input  input
+		output output
+	}{
+		{
+			name: "allowed exact origin",
+			input: input{
+				cfg: &CORSConfiguration{Rules: []CORSRule{{
+					AllowedOrigins: []string{"http://allowed.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				}}},
+				origin:  "http://allowed.com",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{
+				result: &CORSAllowanceConfig{
+					Origin:           "http://allowed.com",
+					AllowCredentials: "true",
+					Methods:          http.MethodGet,
+					AllowHeaders:     "x-test",
+					ExposedHeaders:   "",
+					MaxAge:           nil,
+				},
+				err: nil,
+			},
+		},
+		{
+			name: "allowed wildcard origin",
+			input: input{
+				cfg: &CORSConfiguration{Rules: []CORSRule{{
+					AllowedOrigins: []string{"*"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				}}},
+				origin:  "anything",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{
+				result: &CORSAllowanceConfig{
+					Origin:           "*",
+					AllowCredentials: "false",
+					AllowHeaders:     "x-test",
+					Methods:          http.MethodGet,
+					ExposedHeaders:   "",
+					MaxAge:           nil,
+				},
+				err: nil,
+			},
+		},
+		{
+			name: "forbidden no matching origin",
+			input: input{
+				cfg: &CORSConfiguration{Rules: []CORSRule{{
+					AllowedOrigins: []string{"http://nope.com"},
+				}}},
+				origin: "http://not-allowed.com",
+				method: http.MethodGet,
+			},
+			output: output{
+				result: nil,
+				err:    s3err.GetAPIError(s3err.ErrCORSForbidden),
+			},
+		},
+		{
+			name: "forbidden method not allowed",
+			input: input{
+				cfg: &CORSConfiguration{Rules: []CORSRule{{
+					AllowedOrigins: []string{"http://allowed.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodPost},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				}}},
+				origin:  "http://allowed.com",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{
+				result: nil,
+				err:    s3err.GetAPIError(s3err.ErrCORSForbidden),
+			},
+		},
+		{
+			name: "forbidden header not allowed",
+			input: input{
+				cfg: &CORSConfiguration{Rules: []CORSRule{{
+					AllowedOrigins: []string{"http://allowed.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				}}},
+				origin:  "http://allowed.com",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Nope"},
+			},
+			output: output{
+				result: nil,
+				err:    s3err.GetAPIError(s3err.ErrCORSForbidden),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.input.cfg.IsAllowed(tt.input.origin, tt.input.method, tt.input.headers)
+			assert.EqualValues(t, tt.output.err, err)
+			assert.EqualValues(t, tt.output.result, got)
+		})
+	}
+}
+
+func TestCORSRule_Validate(t *testing.T) {
+	tests := []struct {
+		name string
+		rule CORSRule
+		want error
+	}{
+		{
+			name: "valid rule",
+			rule: CORSRule{
+				AllowedOrigins: []string{"http://allowed.com"},
+				AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+				AllowedHeaders: []CORSHeader{"X-Test"},
+			},
+			want: nil,
+		},
+		{
+			name: "invalid allowed methods",
+			rule: CORSRule{
+				AllowedOrigins: []string{"http://allowed.com"},
+				AllowedMethods: []CORSHTTPMethod{"invalid_method"},
+				AllowedHeaders: []CORSHeader{"X-Test"},
+			},
+			want: s3err.GetUnsopportedCORSMethodErr("invalid_method"),
+		},
+		{
+			name: "invalid allowed header",
+			rule: CORSRule{
+				AllowedOrigins: []string{"http://allowed.com"},
+				AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+				AllowedHeaders: []CORSHeader{"Invalid Header"},
+			},
+			want: s3err.GetInvalidCORSHeaderErr("Invalid Header"),
+		},
+		{
+			name: "invalid allowed header",
+			rule: CORSRule{
+				AllowedOrigins: []string{"http://allowed.com"},
+				AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+				AllowedHeaders: []CORSHeader{"Content-Length"},
+				ExposeHeaders:  []CORSHeader{"Content-Encoding", "invalid header"},
+			},
+			want: s3err.GetInvalidCORSHeaderErr("invalid header"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.rule.Validate()
+			assert.EqualValues(t, tt.want, err)
+		})
+	}
+}
+
+func TestCORSRule_Match(t *testing.T) {
+	type input struct {
+		rule    CORSRule
+		origin  string
+		method  CORSHTTPMethod
+		headers []CORSHeader
+	}
+	type output struct {
+		isAllowed  bool
+		isWildcard bool
+	}
+	tests := []struct {
+		name   string
+		input  input
+		output output
+	}{
+		{
+			name: "exact origin and method match",
+			input: input{
+				rule: CORSRule{
+					AllowedOrigins: []string{"http://allowed.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				},
+				origin:  "http://allowed.com",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{isAllowed: true, isWildcard: false},
+		},
+		{
+			name: "wildcard origin match",
+			input: input{
+				rule: CORSRule{
+					AllowedOrigins: []string{"*"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodPost},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				},
+				origin:  "http://random.com",
+				method:  http.MethodPost,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{isAllowed: true, isWildcard: true},
+		},
+		{
+			name: "wildcard containing origin match",
+			input: input{
+				rule: CORSRule{
+					AllowedOrigins: []string{"http://random*"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodPost},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				},
+				origin:  "http://random.com",
+				method:  http.MethodPost,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{isAllowed: true, isWildcard: false},
+		},
+		{
+			name: "wildcard allowed headers match",
+			input: input{
+				rule: CORSRule{
+					AllowedOrigins: []string{"http://something.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodPost},
+					AllowedHeaders: []CORSHeader{"X-*"},
+				},
+				origin:  "http://something.com",
+				method:  http.MethodPost,
+				headers: []CORSHeader{"X-Test", "X-Something", "X-Anyting"},
+			},
+			output: output{isAllowed: true, isWildcard: false},
+		},
+		{
+			name: "origin mismatch",
+			input: input{
+				rule: CORSRule{
+					AllowedOrigins: []string{"http://allowed.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				},
+				origin:  "http://notallowed.com",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{isAllowed: false, isWildcard: false},
+		},
+		{
+			name: "method mismatch",
+			input: input{
+				rule: CORSRule{
+					AllowedOrigins: []string{"http://allowed.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodPost},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				},
+				origin:  "http://allowed.com",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Test"},
+			},
+			output: output{isAllowed: false, isWildcard: false},
+		},
+		{
+			name: "header mismatch",
+			input: input{
+				rule: CORSRule{
+					AllowedOrigins: []string{"http://allowed.com"},
+					AllowedMethods: []CORSHTTPMethod{http.MethodGet},
+					AllowedHeaders: []CORSHeader{"X-Test"},
+				},
+				origin:  "http://allowed.com",
+				method:  http.MethodGet,
+				headers: []CORSHeader{"X-Other"},
+			},
+			output: output{isAllowed: false, isWildcard: false},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			isAllowed, wild := tt.input.rule.Match(tt.input.origin, tt.input.method, tt.input.headers)
+			assert.Equal(t, tt.output.isAllowed, isAllowed)
+			assert.Equal(t, tt.output.isWildcard, wild)
+		})
+	}
+}
+
+func TestGetExposeHeaders(t *testing.T) {
+	tests := []struct {
+		name string
+		rule CORSRule
+		want string
+	}{
+		{"multiple headers", CORSRule{ExposeHeaders: []CORSHeader{"Content-Length", "Content-Type", "Content-Encoding"}}, "Content-Length, Content-Type, Content-Encoding"},
+		{"single header", CORSRule{ExposeHeaders: []CORSHeader{"Authorization"}}, "Authorization"},
+		{"no headers", CORSRule{}, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.rule.GetExposeHeaders()
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestBuildAllowedHeaders(t *testing.T) {
+	tests := []struct {
+		name    string
+		headers []CORSHeader
+		want    string
+	}{
+		{
+			name:    "empty slice returns empty string",
+			headers: []CORSHeader{},
+			want:    "",
+		},
+		{
+			name:    "single header lowercase",
+			headers: []CORSHeader{"Content-Type"},
+			want:    "content-type",
+		},
+		{
+			name:    "multiple headers lowercased with commas",
+			headers: []CORSHeader{"Content-Type", "X-Custom-Header", "Authorization"},
+			want:    "content-type, x-custom-header, authorization",
+		},
+		{
+			name:    "already lowercase header",
+			headers: []CORSHeader{"accept"},
+			want:    "accept",
+		},
+		{
+			name:    "mixed case headers",
+			headers: []CORSHeader{"ACCEPT", "x-Powered-By"},
+			want:    "accept, x-powered-by",
+		},
+		{
+			name:    "empty header value",
+			headers: []CORSHeader{""},
+			want:    "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := buildAllowedHeaders(tt.headers)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestGetAllowedMethods(t *testing.T) {
+	tests := []struct {
+		name string
+		rule CORSRule
+		want string
+	}{
+		{"multiple methods", CORSRule{AllowedMethods: []CORSHTTPMethod{http.MethodGet, http.MethodPost, http.MethodPut}}, "GET, POST, PUT"},
+		{"single method", CORSRule{AllowedMethods: []CORSHTTPMethod{http.MethodGet}}, "GET"},
+		{"no methods", CORSRule{}, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.rule.GetAllowedMethods()
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestParseCORSOutput(t *testing.T) {
+	tests := []struct {
+		name string
+		data string
+		want bool
+	}{
+		{"valid", `<CORSConfiguration><CORSRule></CORSRule></CORSConfiguration>`, true},
+		{"invalid xml", `<CORSConfiguration><CORSRule>`, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg, err := ParseCORSOutput([]byte(tt.data))
+			if (err == nil) != tt.want {
+				t.Errorf("ParseCORSOutput() err = %v, want success=%v", err, tt.want)
+			}
+			if tt.want && cfg == nil {
+				t.Errorf("Expected non-nil config")
+			}
+		})
+	}
+}
+
+func TestCacheCORSProps(t *testing.T) {
+	tests := []struct {
+		name string
+		in   []CORSHTTPMethod
+		want map[string]struct{}
+	}{
+		{
+			name: "empty CORSHTTPMethod slice",
+			in:   []CORSHTTPMethod{},
+			want: map[string]struct{}{},
+		},
+		{
+			name: "single CORSHTTPMethod",
+			in:   []CORSHTTPMethod{http.MethodGet},
+			want: map[string]struct{}{http.MethodGet: {}},
+		},
+		{
+			name: "multiple CORSHTTPMethods",
+			in:   []CORSHTTPMethod{http.MethodGet, http.MethodPost, http.MethodPut},
+			want: map[string]struct{}{
+				http.MethodGet:  {},
+				http.MethodPost: {},
+				http.MethodPut:  {},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := cacheCORSMethods(tt.in)
+			assert.Equal(t, len(tt.want), len(got))
+			for key := range tt.want {
+				_, ok := got[CORSHTTPMethod(key)]
+				assert.True(t, ok)
+			}
+		})
+	}
+}
+
+func TestParseCORSHeaders(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want []CORSHeader
+		err  error
+	}{
+		{
+			name: "empty string",
+			in:   "",
+			want: []CORSHeader{},
+			err:  nil,
+		},
+		{
+			name: "single valid header",
+			in:   "X-Test",
+			want: []CORSHeader{"X-Test"},
+			err:  nil,
+		},
+		{
+			name: "multiple valid headers with spaces",
+			in:   "X-Test, Content-Type, Authorization",
+			want: []CORSHeader{"X-Test", "Content-Type", "Authorization"},
+			err:  nil,
+		},
+		{
+			name: "header with leading/trailing spaces",
+			in:   "   X-Test   ",
+			want: []CORSHeader{"X-Test"},
+			err:  nil,
+		},
+		{
+			name: "contains invalid header",
+			in:   "X-Test, Invalid Header, Content-Type",
+			want: nil,
+			err:  s3err.GetInvalidCORSRequestHeaderErr(" Invalid Header"),
+		},
+		{
+			name: "only invalid header",
+			in:   "Invalid Header",
+			want: nil,
+			err:  s3err.GetInvalidCORSRequestHeaderErr("Invalid Header"),
+		},
+		{
+			name: "multiple commas in a row",
+			in:   "X-Test,,Content-Type",
+			want: nil,
+			err:  s3err.GetInvalidCORSRequestHeaderErr(""),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseCORSHeaders(tt.in)
+			assert.EqualValues(t, tt.err, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestWildcardMatch(t *testing.T) {
+	tests := []struct {
+		name    string
+		pattern string
+		input   string
+		want    bool
+	}{
+		// Exact match, no wildcards
+		{"exact match", "hello", "hello", true},
+		{"exact mismatch", "hello", "hell", false},
+		// Single '*' matching zero chars
+		{"star matches zero chars", "he*lo", "helo", true},
+		// Single '*' matching multiple chars
+		{"star matches multiple chars", "he*o", "heyyyyyo", true},
+		// '*' at start
+		{"star at start", "*world", "hello world", true},
+		// '*' at end
+		{"star at end", "hello*", "hello there", true},
+		// '*' matches whole string
+		{"only star", "*", "anything", true},
+		{"only star empty", "*", "", true},
+		// Multiple '*'s
+		{"multiple stars", "a*b*c", "axxxbzzzzyc", true},
+		{"multiple stars no match", "a*b*c", "axxxbzzzzy", false},
+		// Backtracking needed
+		{"backtracking required", "a*b*c", "ab123c", true},
+		// No match with star present
+		{"star but mismatch", "he*world", "hey there", false},
+		// Trailing stars in pattern
+		{"trailing stars match", "abc**", "abc", true},
+		{"trailing stars match longer", "abc**", "abccc", true},
+		// Empty pattern cases
+		{"empty pattern and empty input", "", "", true},
+		{"empty pattern non-empty input", "", "a", false},
+		{"only stars pattern with empty input", "***", "", true},
+		// Pattern longer than input
+		{"pattern longer no star", "abcd", "abc", false},
+		// Input longer but no star
+		{"input longer no star", "abc", "abcd", false},
+		// Complex interleaved match
+		{"complex interleaved", "*a*b*cd*", "xxaYYbZZcd123", true},
+		// Star match at the end after mismatch
+		{"mismatch then star match", "ab*xyz", "abzzzxyz", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := wildcardMatch(tt.pattern, tt.input)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

auth/bucket_policy.go

@@ -0,0 +1,290 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+var ErrAccessDenied = errors.New("access denied")
+
+type policyErr string
+
+func (p policyErr) Error() string {
+	return string(p)
+}
+
+const (
+	policyErrResourceMismatch     = policyErr("Action does not apply to any resource(s) in statement")
+	policyErrInvalidResource      = policyErr("Policy has invalid resource")
+	policyErrInvalidPrincipal     = policyErr("Invalid principal in policy")
+	policyErrInvalidAction        = policyErr("Policy has invalid action")
+	policyErrInvalidPolicy        = policyErr("This policy contains invalid Json")
+	policyErrInvalidFirstChar     = policyErr("Policies must be valid JSON and the first byte must be '{'")
+	policyErrEmptyStatement       = policyErr("Could not parse the policy: Statement is empty!")
+	policyErrMissingStatmentField = policyErr("Missing required field Statement")
+)
+
+type BucketPolicy struct {
+	Statement []BucketPolicyItem `json:"Statement"`
+}
+
+func (bp *BucketPolicy) UnmarshalJSON(data []byte) error {
+	var tmp struct {
+		Statement *[]BucketPolicyItem `json:"Statement"`
+	}
+
+	if err := json.Unmarshal(data, &tmp); err != nil {
+		return err
+	}
+
+	// If Statement is nil (not present in JSON), return an error
+	if tmp.Statement == nil {
+		return policyErrMissingStatmentField
+	}
+
+	// Assign the parsed value to the actual struct
+	bp.Statement = *tmp.Statement
+	return nil
+}
+
+func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
+	for _, statement := range bp.Statement {
+		err := statement.Validate(bucket, iam)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	var isAllowed bool
+	for _, statement := range bp.Statement {
+		if statement.findMatch(principal, action, resource) {
+			switch statement.Effect {
+			case BucketPolicyAccessTypeAllow:
+				isAllowed = true
+			case BucketPolicyAccessTypeDeny:
+				return false
+			}
+		}
+	}
+
+	return isAllowed
+}
+
+// IsPublicFor checks if the bucket policy statements contain
+// an entity granting public access to the given resource and action
+func (bp *BucketPolicy) isPublicFor(resource string, action Action) bool {
+	var isAllowed bool
+	for _, statement := range bp.Statement {
+		if statement.isPublicFor(resource, action) {
+			switch statement.Effect {
+			case BucketPolicyAccessTypeAllow:
+				isAllowed = true
+			case BucketPolicyAccessTypeDeny:
+				return false
+			}
+		}
+	}
+
+	return isAllowed
+}
+
+// IsPublic checks if one of bucket policy statments grant
+// public access to ALL users
+func (bp *BucketPolicy) IsPublic() bool {
+	for _, statement := range bp.Statement {
+		if statement.isPublic() {
+			return true
+		}
+	}
+
+	return false
+}
+
+type BucketPolicyItem struct {
+	Effect     BucketPolicyAccessType `json:"Effect"`
+	Principals Principals             `json:"Principal"`
+	Actions    Actions                `json:"Action"`
+	Resources  Resources              `json:"Resource"`
+}
+
+func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
+	if err := bpi.Effect.Validate(); err != nil {
+		return err
+	}
+	if err := bpi.Principals.Validate(iam); err != nil {
+		return err
+	}
+	if err := bpi.Resources.Validate(bucket); err != nil {
+		return err
+	}
+
+	containsObjectAction := bpi.Resources.ContainsObjectPattern()
+	containsBucketAction := bpi.Resources.ContainsBucketPattern()
+
+	for action := range bpi.Actions {
+		isObjectAction := action.IsObjectAction()
+		if isObjectAction == nil {
+			break
+		}
+		if *isObjectAction && !containsObjectAction {
+			return policyErrResourceMismatch
+		}
+		if !*isObjectAction && !containsBucketAction {
+			return policyErrResourceMismatch
+		}
+	}
+
+	return nil
+}
+
+func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
+	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
+		return true
+	}
+
+	return false
+}
+
+// isPublicFor checks if the bucket policy statemant grants public access
+// for given resource and action
+func (bpi *BucketPolicyItem) isPublicFor(resource string, action Action) bool {
+	return bpi.Principals.isPublic() && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource)
+}
+
+// isPublic checks if the statement grants public access
+// to ALL users
+func (bpi *BucketPolicyItem) isPublic() bool {
+	return bpi.Principals.isPublic()
+}
+
+func getMalformedPolicyError(err error) error {
+	return s3err.APIError{
+		Code:           "MalformedPolicy",
+		Description:    err.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+// ParsePolicyDocument parses raw bytes to 'BucketPolicy'
+func ParsePolicyDocument(data []byte) (*BucketPolicy, error) {
+	var policy BucketPolicy
+	if err := json.Unmarshal(data, &policy); err != nil {
+		var pe policyErr
+		if errors.As(err, &pe) {
+			return nil, getMalformedPolicyError(err)
+		}
+		return nil, getMalformedPolicyError(policyErrInvalidPolicy)
+	}
+
+	return &policy, nil
+}
+
+func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
+	if len(policyBin) == 0 || policyBin[0] != '{' {
+		return getMalformedPolicyError(policyErrInvalidFirstChar)
+	}
+	policy, err := ParsePolicyDocument(policyBin)
+	if err != nil {
+		return err
+	}
+
+	if len(policy.Statement) == 0 {
+		return getMalformedPolicyError(policyErrEmptyStatement)
+	}
+
+	if err := policy.Validate(bucket, iam); err != nil {
+		return getMalformedPolicyError(err)
+	}
+
+	return nil
+}
+
+func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
+	var bucketPolicy BucketPolicy
+	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
+		return fmt.Errorf("failed to parse the bucket policy: %w", err)
+	}
+
+	resource := bucket
+	if object != "" {
+		resource += "/" + object
+	}
+
+	if !bucketPolicy.isAllowed(access, action, resource) {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+// Checks if the bucket policy grants public access
+func VerifyPublicBucketPolicy(policy []byte, bucket, object string, action Action) error {
+	var bucketPolicy BucketPolicy
+	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
+		return err
+	}
+
+	resource := bucket
+	if object != "" {
+		resource += "/" + object
+	}
+
+	if !bucketPolicy.isPublicFor(resource, action) {
+		return ErrAccessDenied
+	}
+
+	return nil
+}
+
+// matchPattern checks if the input string matches the given pattern with wildcard(`*`) and any character(`?`).
+// - `?` matches exactly one occurrence of any character.
+// - `*` matches arbitrary many (including zero) occurrences of any character.
+func matchPattern(pattern, input string) bool {
+	pIdx, sIdx := 0, 0
+	starIdx, matchIdx := -1, 0
+
+	for sIdx < len(input) {
+		if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
+			sIdx++
+			pIdx++
+		} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
+			starIdx = pIdx
+			matchIdx = sIdx
+			pIdx++
+		} else if starIdx != -1 {
+			pIdx = starIdx + 1
+			matchIdx++
+			sIdx = matchIdx
+		} else {
+			return false
+		}
+	}
+
+	for pIdx < len(pattern) && pattern[pIdx] == '*' {
+		pIdx++
+	}
+
+	return pIdx == len(pattern)
+}

auth/bucket_policy_actions.go

@@ -0,0 +1,313 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+type Action string
+
+const (
+	GetBucketAclAction                       Action = "s3:GetBucketAcl"
+	CreateBucketAction                       Action = "s3:CreateBucket"
+	PutBucketAclAction                       Action = "s3:PutBucketAcl"
+	DeleteBucketAction                       Action = "s3:DeleteBucket"
+	PutBucketVersioningAction                Action = "s3:PutBucketVersioning"
+	GetBucketVersioningAction                Action = "s3:GetBucketVersioning"
+	PutBucketPolicyAction                    Action = "s3:PutBucketPolicy"
+	GetBucketPolicyAction                    Action = "s3:GetBucketPolicy"
+	DeleteBucketPolicyAction                 Action = "s3:DeleteBucketPolicy"
+	AbortMultipartUploadAction               Action = "s3:AbortMultipartUpload"
+	ListMultipartUploadPartsAction           Action = "s3:ListMultipartUploadParts"
+	ListBucketMultipartUploadsAction         Action = "s3:ListBucketMultipartUploads"
+	PutObjectAction                          Action = "s3:PutObject"
+	GetObjectAction                          Action = "s3:GetObject"
+	GetObjectVersionAction                   Action = "s3:GetObjectVersion"
+	DeleteObjectAction                       Action = "s3:DeleteObject"
+	GetObjectAclAction                       Action = "s3:GetObjectAcl"
+	GetObjectAttributesAction                Action = "s3:GetObjectAttributes"
+	PutObjectAclAction                       Action = "s3:PutObjectAcl"
+	RestoreObjectAction                      Action = "s3:RestoreObject"
+	GetBucketTaggingAction                   Action = "s3:GetBucketTagging"
+	PutBucketTaggingAction                   Action = "s3:PutBucketTagging"
+	GetObjectTaggingAction                   Action = "s3:GetObjectTagging"
+	PutObjectTaggingAction                   Action = "s3:PutObjectTagging"
+	DeleteObjectTaggingAction                Action = "s3:DeleteObjectTagging"
+	ListBucketVersionsAction                 Action = "s3:ListBucketVersions"
+	ListBucketAction                         Action = "s3:ListBucket"
+	GetBucketObjectLockConfigurationAction   Action = "s3:GetBucketObjectLockConfiguration"
+	PutBucketObjectLockConfigurationAction   Action = "s3:PutBucketObjectLockConfiguration"
+	GetObjectLegalHoldAction                 Action = "s3:GetObjectLegalHold"
+	PutObjectLegalHoldAction                 Action = "s3:PutObjectLegalHold"
+	GetObjectRetentionAction                 Action = "s3:GetObjectRetention"
+	PutObjectRetentionAction                 Action = "s3:PutObjectRetention"
+	BypassGovernanceRetentionAction          Action = "s3:BypassGovernanceRetention"
+	PutBucketOwnershipControlsAction         Action = "s3:PutBucketOwnershipControls"
+	GetBucketOwnershipControlsAction         Action = "s3:GetBucketOwnershipControls"
+	PutBucketCorsAction                      Action = "s3:PutBucketCORS"
+	GetBucketCorsAction                      Action = "s3:GetBucketCORS"
+	PutAnalyticsConfigurationAction          Action = "s3:PutAnalyticsConfiguration"
+	GetAnalyticsConfigurationAction          Action = "s3:GetAnalyticsConfiguration"
+	PutEncryptionConfigurationAction         Action = "s3:PutEncryptionConfiguration"
+	GetEncryptionConfigurationAction         Action = "s3:GetEncryptionConfiguration"
+	PutIntelligentTieringConfigurationAction Action = "s3:PutIntelligentTieringConfiguration"
+	GetIntelligentTieringConfigurationAction Action = "s3:GetIntelligentTieringConfiguration"
+	PutInventoryConfigurationAction          Action = "s3:PutInventoryConfiguration"
+	GetInventoryConfigurationAction          Action = "s3:GetInventoryConfiguration"
+	PutLifecycleConfigurationAction          Action = "s3:PutLifecycleConfiguration"
+	GetLifecycleConfigurationAction          Action = "s3:GetLifecycleConfiguration"
+	PutBucketLoggingAction                   Action = "s3:PutBucketLogging"
+	GetBucketLoggingAction                   Action = "s3:GetBucketLogging"
+	PutBucketRequestPaymentAction            Action = "s3:PutBucketRequestPayment"
+	GetBucketRequestPaymentAction            Action = "s3:GetBucketRequestPayment"
+	PutMetricsConfigurationAction            Action = "s3:PutMetricsConfiguration"
+	GetMetricsConfigurationAction            Action = "s3:GetMetricsConfiguration"
+	PutReplicationConfigurationAction        Action = "s3:PutReplicationConfiguration"
+	GetReplicationConfigurationAction        Action = "s3:GetReplicationConfiguration"
+	PutBucketPublicAccessBlockAction         Action = "s3:PutBucketPublicAccessBlock"
+	GetBucketPublicAccessBlockAction         Action = "s3:GetBucketPublicAccessBlock"
+	PutBucketNotificationAction              Action = "s3:PutBucketNotification"
+	GetBucketNotificationAction              Action = "s3:GetBucketNotification"
+	PutAccelerateConfigurationAction         Action = "s3:PutAccelerateConfiguration"
+	GetAccelerateConfigurationAction         Action = "s3:GetAccelerateConfiguration"
+	PutBucketWebsiteAction                   Action = "s3:PutBucketWebsite"
+	GetBucketWebsiteAction                   Action = "s3:GetBucketWebsite"
+	GetBucketPolicyStatusAction              Action = "s3:GetBucketPolicyStatus"
+	GetBucketLocationAction                  Action = "s3:GetBucketLocation"
+
+	AllActions Action = "s3:*"
+)
+
+var supportedActionList = map[Action]struct{}{
+	GetBucketAclAction:                       {},
+	CreateBucketAction:                       {},
+	PutBucketAclAction:                       {},
+	DeleteBucketAction:                       {},
+	PutBucketVersioningAction:                {},
+	GetBucketVersioningAction:                {},
+	PutBucketPolicyAction:                    {},
+	GetBucketPolicyAction:                    {},
+	DeleteBucketPolicyAction:                 {},
+	AbortMultipartUploadAction:               {},
+	ListMultipartUploadPartsAction:           {},
+	ListBucketMultipartUploadsAction:         {},
+	PutObjectAction:                          {},
+	GetObjectAction:                          {},
+	GetObjectVersionAction:                   {},
+	DeleteObjectAction:                       {},
+	GetObjectAclAction:                       {},
+	GetObjectAttributesAction:                {},
+	PutObjectAclAction:                       {},
+	RestoreObjectAction:                      {},
+	GetBucketTaggingAction:                   {},
+	PutBucketTaggingAction:                   {},
+	GetObjectTaggingAction:                   {},
+	PutObjectTaggingAction:                   {},
+	DeleteObjectTaggingAction:                {},
+	ListBucketVersionsAction:                 {},
+	ListBucketAction:                         {},
+	GetBucketObjectLockConfigurationAction:   {},
+	PutBucketObjectLockConfigurationAction:   {},
+	GetObjectLegalHoldAction:                 {},
+	PutObjectLegalHoldAction:                 {},
+	GetObjectRetentionAction:                 {},
+	PutObjectRetentionAction:                 {},
+	BypassGovernanceRetentionAction:          {},
+	PutBucketOwnershipControlsAction:         {},
+	GetBucketOwnershipControlsAction:         {},
+	PutBucketCorsAction:                      {},
+	GetBucketCorsAction:                      {},
+	PutAnalyticsConfigurationAction:          {},
+	GetAnalyticsConfigurationAction:          {},
+	PutEncryptionConfigurationAction:         {},
+	GetEncryptionConfigurationAction:         {},
+	PutIntelligentTieringConfigurationAction: {},
+	GetIntelligentTieringConfigurationAction: {},
+	PutInventoryConfigurationAction:          {},
+	GetInventoryConfigurationAction:          {},
+	PutLifecycleConfigurationAction:          {},
+	GetLifecycleConfigurationAction:          {},
+	PutBucketLoggingAction:                   {},
+	GetBucketLoggingAction:                   {},
+	PutBucketRequestPaymentAction:            {},
+	GetBucketRequestPaymentAction:            {},
+	PutMetricsConfigurationAction:            {},
+	GetMetricsConfigurationAction:            {},
+	PutReplicationConfigurationAction:        {},
+	GetReplicationConfigurationAction:        {},
+	PutBucketPublicAccessBlockAction:         {},
+	GetBucketPublicAccessBlockAction:         {},
+	PutBucketNotificationAction:              {},
+	GetBucketNotificationAction:              {},
+	PutAccelerateConfigurationAction:         {},
+	GetAccelerateConfigurationAction:         {},
+	PutBucketWebsiteAction:                   {},
+	GetBucketWebsiteAction:                   {},
+	GetBucketPolicyStatusAction:              {},
+	GetBucketLocationAction:                  {},
+	AllActions:                               {},
+}
+
+var supportedObjectActionList = map[Action]struct{}{
+	AbortMultipartUploadAction:      {},
+	ListMultipartUploadPartsAction:  {},
+	PutObjectAction:                 {},
+	GetObjectAction:                 {},
+	GetObjectVersionAction:          {},
+	DeleteObjectAction:              {},
+	GetObjectAclAction:              {},
+	GetObjectAttributesAction:       {},
+	PutObjectAclAction:              {},
+	RestoreObjectAction:             {},
+	GetObjectTaggingAction:          {},
+	PutObjectTaggingAction:          {},
+	DeleteObjectTaggingAction:       {},
+	GetObjectLegalHoldAction:        {},
+	PutObjectLegalHoldAction:        {},
+	GetObjectRetentionAction:        {},
+	PutObjectRetentionAction:        {},
+	BypassGovernanceRetentionAction: {},
+	AllActions:                      {},
+}
+
+// Validates Action: it should either wildcard match with supported actions list or be in it
+func (a Action) IsValid() error {
+	if !strings.HasPrefix(string(a), "s3:") {
+		return policyErrInvalidAction
+	}
+
+	if a == AllActions {
+		return nil
+	}
+
+	// first check for an exact match
+	if _, ok := supportedActionList[a]; ok {
+		return nil
+	}
+
+	// walk through the supported actions and try wildcard match
+	for action := range supportedActionList {
+		if action.Match(a) {
+			return nil
+		}
+	}
+
+	return policyErrInvalidAction
+}
+
+func getBoolPtr(bl bool) *bool {
+	return &bl
+}
+
+// String converts the action to string
+func (a Action) String() string {
+	return string(a)
+}
+
+// Match wildcard matches the given pattern to the action
+func (a Action) Match(pattern Action) bool {
+	return matchPattern(pattern.String(), a.String())
+}
+
+// Checks if the action is object action
+// nil points to 's3:*'
+func (a Action) IsObjectAction() *bool {
+	if a == AllActions {
+		return nil
+	}
+
+	// first find an exact match
+	if _, ok := supportedObjectActionList[a]; ok {
+		return &ok
+	}
+
+	for action := range supportedObjectActionList {
+		if action.Match(a) {
+			return getBoolPtr(true)
+		}
+	}
+
+	return getBoolPtr(false)
+}
+
+type Actions map[Action]struct{}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (a *Actions) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return policyErrInvalidAction
+		}
+		*a = make(Actions)
+		for _, s := range ss {
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return policyErrInvalidAction
+			}
+			*a = make(Actions)
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Validates and adds a new Action to Actions map
+func (a Actions) Add(str string) error {
+	action := Action(str)
+	err := action.IsValid()
+	if err != nil {
+		return err
+	}
+
+	a[action] = struct{}{}
+	return nil
+}
+
+// FindMatch tries to match the given action to the actions list
+func (a Actions) FindMatch(action Action) bool {
+	_, ok := a[AllActions]
+	if ok {
+		return true
+	}
+	// First O(1) check for non wildcard actions
+	_, found := a[action]
+	if found {
+		return true
+	}
+
+	// search for a wildcard match
+	for act := range a {
+		if action.Match(act) {
+			return true
+		}
+	}
+
+	return false
+}

auth/bucket_policy_actions_test.go

@@ -0,0 +1,175 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAction_IsValid(t *testing.T) {
+	tests := []struct {
+		name    string
+		action  Action
+		wantErr bool
+	}{
+		{"valid exact action", GetObjectAction, false},
+		{"valid all actions", AllActions, false},
+		{"invalid prefix", "invalid:Action", true},
+		{"unsupported action 1", "s3:Unsupported", true},
+		{"unsupported action 2", "s3:HeadObject", true},
+		{"valid wildcard match 1", "s3:Get*", false},
+		{"valid wildcard match 2", "s3:*Object*", false},
+		{"valid wildcard match 3", "s3:*Multipart*", false},
+		{"any char match 1", "s3:Get?bject", false},
+		{"any char match 2", "s3:Get??bject", true},
+		{"any char match 3", "s3:???", true},
+		{"mixed match 1", "s3:Get?*", false},
+		{"mixed match 2", "s3:*Object?????", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.action.IsValid()
+			if tt.wantErr {
+				assert.EqualValues(t, policyErrInvalidAction, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestAction_String(t *testing.T) {
+	a := Action("s3:TestAction")
+	assert.Equal(t, "s3:TestAction", a.String())
+}
+
+func TestAction_Match(t *testing.T) {
+	tests := []struct {
+		name    string
+		action  Action
+		pattern Action
+		want    bool
+	}{
+		{"exact match", "s3:GetObject", "s3:GetObject", true},
+		{"wildcard match", "s3:GetObject", "s3:Get*", true},
+		{"wildcard mismatch", "s3:PutObject", "s3:Get*", false},
+		{"any character match", "s3:Get1", "s3:Get?", true},
+		{"any character mismatch", "s3:Get12", "s3:Get?", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.action.Match(tt.pattern)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestAction_IsObjectAction(t *testing.T) {
+	tests := []struct {
+		name   string
+		action Action
+		want   *bool
+	}{
+		{"all actions", AllActions, nil},
+		{"object action exact", GetObjectAction, getBoolPtr(true)},
+		{"object action wildcard", "s3:Get*", getBoolPtr(true)},
+		{"non object action", GetBucketAclAction, getBoolPtr(false)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.action.IsObjectAction()
+			if tt.want == nil {
+				assert.Nil(t, got)
+			} else {
+				assert.NotNil(t, got)
+				assert.Equal(t, *tt.want, *got)
+			}
+		})
+	}
+}
+
+func TestActions_UnmarshalJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   string
+		wantErr bool
+	}{
+		{"valid slice", `["s3:GetObject","s3:PutObject"]`, false},
+		{"empty slice", `[]`, true},
+		{"invalid action in slice", `["s3:Invalid"]`, true},
+		{"valid string", `"s3:GetObject"`, false},
+		{"empty string", `""`, true},
+		{"invalid string", `"s3:Invalid"`, true},
+		{"invalid json", `{}`, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var a Actions
+			err := json.Unmarshal([]byte(tt.input), &a)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestActions_Add(t *testing.T) {
+	tests := []struct {
+		name    string
+		action  string
+		wantErr bool
+	}{
+		{"valid add", "s3:GetObject", false},
+		{"invalid add", "s3:InvalidAction", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := make(Actions)
+			err := a.Add(tt.action)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				_, ok := a[Action(tt.action)]
+				assert.True(t, ok)
+			}
+		})
+	}
+}
+
+func TestActions_FindMatch(t *testing.T) {
+	tests := []struct {
+		name    string
+		actions Actions
+		check   Action
+		want    bool
+	}{
+		{"all actions present", Actions{AllActions: {}}, GetObjectAction, true},
+		{"exact match", Actions{GetObjectAction: {}}, GetObjectAction, true},
+		{"wildcard match", Actions{"s3:Get*": {}}, GetObjectAction, true},
+		{"no match", Actions{"s3:Put*": {}}, GetObjectAction, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.actions.FindMatch(tt.check)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

auth/bucket_policy_effect.go

@@ -0,0 +1,35 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import "fmt"
+
+type BucketPolicyAccessType string
+
+const (
+	BucketPolicyAccessTypeDeny  BucketPolicyAccessType = "Deny"
+	BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
+)
+
+// Checks policy statement Effect to be valid ("Deny", "Allow")
+func (bpat BucketPolicyAccessType) Validate() error {
+	switch bpat {
+	case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
+		return nil
+	}
+
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	return fmt.Errorf("Invalid effect: %v", bpat)
+}

auth/bucket_policy_effect_test.go

@@ -0,0 +1,57 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBucketPolicyAccessType_Validate(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   BucketPolicyAccessType
+		wantErr bool
+		errMsg  string
+	}{
+		{
+			name:    "valid allow",
+			input:   BucketPolicyAccessTypeAllow,
+			wantErr: false,
+		},
+		{
+			name:    "valid deny",
+			input:   BucketPolicyAccessTypeDeny,
+			wantErr: false,
+		},
+		{
+			name:    "invalid type",
+			input:   BucketPolicyAccessType("InvalidValue"),
+			wantErr: true,
+			errMsg:  "Invalid effect: InvalidValue",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.input.Validate()
+			if tt.wantErr {
+				assert.EqualError(t, err, tt.errMsg)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

auth/bucket_policy_principals.go

@@ -0,0 +1,130 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+)
+
+type Principals map[string]struct{}
+
+func (p Principals) Add(key string) {
+	p[key] = struct{}{}
+}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (p *Principals) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var s string
+	var k struct {
+		AWS string
+	}
+
+	var err error
+
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return policyErrInvalidPrincipal
+		}
+		*p = make(Principals)
+		for _, s := range ss {
+			p.Add(s)
+		}
+		return nil
+	} else if err = json.Unmarshal(data, &s); err == nil {
+		if s == "" {
+			return policyErrInvalidPrincipal
+		}
+		*p = make(Principals)
+		p.Add(s)
+
+		return nil
+	} else if err = json.Unmarshal(data, &k); err == nil {
+		if k.AWS == "" {
+			return policyErrInvalidPrincipal
+		}
+		*p = make(Principals)
+		p.Add(k.AWS)
+
+		return nil
+	} else {
+		var sk struct {
+			AWS []string
+		}
+		if err = json.Unmarshal(data, &sk); err == nil {
+			if len(sk.AWS) == 0 {
+				return policyErrInvalidPrincipal
+			}
+			*p = make(Principals)
+			for _, s := range sk.AWS {
+				p.Add(s)
+			}
+		}
+	}
+
+	return err
+}
+
+// Converts Principals map to a slice, by omitting "*"
+func (p Principals) ToSlice() []string {
+	principals := []string{}
+	for p := range p {
+		if p == "*" {
+			continue
+		}
+		principals = append(principals, p)
+	}
+
+	return principals
+}
+
+// Validates Principals by checking user account access keys existence
+func (p Principals) Validate(iam IAMService) error {
+	_, containsWildCard := p["*"]
+	if containsWildCard {
+		if len(p) == 1 {
+			return nil
+		}
+		return policyErrInvalidPrincipal
+	}
+
+	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return policyErrInvalidPrincipal
+	}
+
+	return nil
+}
+
+func (p Principals) Contains(userAccess string) bool {
+	// "*" means it matches for any user account
+	_, ok := p["*"]
+	if ok {
+		return true
+	}
+
+	_, found := p[userAccess]
+	return found
+}
+
+// Bucket policy grants public access, if it contains
+// a wildcard match to all the users
+func (p Principals) isPublic() bool {
+	_, ok := p["*"]
+	return ok
+}

auth/bucket_policy_principals_test.go

@@ -0,0 +1,106 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPrincipals_Add(t *testing.T) {
+	p := make(Principals)
+	p.Add("user1")
+	_, ok := p["user1"]
+	assert.True(t, ok)
+}
+
+func TestPrincipals_UnmarshalJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   string
+		want    Principals
+		wantErr bool
+	}{
+		{"valid slice", `["user1","user2"]`, Principals{"user1": {}, "user2": {}}, false},
+		{"empty slice", `[]`, nil, true},
+		{"valid string", `"user1"`, Principals{"user1": {}}, false},
+		{"empty string", `""`, nil, true},
+		{"valid AWS object", `{"AWS":"user1"}`, Principals{"user1": {}}, false},
+		{"empty AWS object", `{"AWS":""}`, nil, true},
+		{"valid AWS array", `{"AWS":["user1","user2"]}`, Principals{"user1": {}, "user2": {}}, false},
+		{"empty AWS array", `{"AWS":[]}`, nil, true},
+		{"invalid json", `{invalid}`, nil, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var p Principals
+			err := json.Unmarshal([]byte(tt.input), &p)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.want, p)
+			}
+		})
+	}
+}
+
+func TestPrincipals_ToSlice(t *testing.T) {
+	p := Principals{"user1": {}, "user2": {}, "*": {}}
+	got := p.ToSlice()
+	assert.Contains(t, got, "user1")
+	assert.Contains(t, got, "user2")
+	assert.NotContains(t, got, "*")
+}
+
+func TestPrincipals_Validate(t *testing.T) {
+	iamSingle := NewIAMServiceSingle(Account{
+		Access: "user1",
+	})
+	tests := []struct {
+		name       string
+		principals Principals
+		mockIAM    IAMService
+		err        error
+	}{
+		{"only wildcard", Principals{"*": {}}, iamSingle, nil},
+		{"wildcard and user", Principals{"*": {}, "user1": {}}, iamSingle, policyErrInvalidPrincipal},
+		{"accounts exist returns err", Principals{"user2": {}, "user3": {}}, iamSingle, policyErrInvalidPrincipal},
+		{"accounts exist non-empty", Principals{"user1": {}}, iamSingle, nil},
+		{"accounts valid", Principals{"user1": {}}, iamSingle, nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.principals.Validate(tt.mockIAM)
+			assert.EqualValues(t, tt.err, err)
+		})
+	}
+}
+
+func TestPrincipals_Contains(t *testing.T) {
+	p := Principals{"user1": {}}
+	assert.True(t, p.Contains("user1"))
+	assert.False(t, p.Contains("user2"))
+
+	p = Principals{"*": {}}
+	assert.True(t, p.Contains("anyuser"))
+}
+
+func TestPrincipals_isPublic(t *testing.T) {
+	assert.True(t, Principals{"*": {}}.isPublic())
+	assert.False(t, Principals{"user1": {}}.isPublic())
+}

auth/bucket_policy_resources.go

@@ -0,0 +1,134 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+type Resources map[string]struct{}
+
+const ResourceArnPrefix = "arn:aws:s3:::"
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (r *Resources) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return policyErrInvalidResource
+		}
+		*r = make(Resources)
+		for _, s := range ss {
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return policyErrInvalidResource
+			}
+			*r = make(Resources)
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Adds and validates a new resource to Resources map
+func (r Resources) Add(rc string) error {
+	ok, pattern := isValidResource(rc)
+	if !ok {
+		return policyErrInvalidResource
+	}
+
+	r[pattern] = struct{}{}
+
+	return nil
+}
+
+// Checks if the resources contain object pattern
+func (r Resources) ContainsObjectPattern() bool {
+	for resource := range r {
+		if resource == "*" || strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Checks if the resources contain bucket pattern
+func (r Resources) ContainsBucketPattern() bool {
+	for resource := range r {
+		if resource == "*" || !strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
+func (r Resources) Validate(bucket string) error {
+	for resource := range r {
+		if !strings.HasPrefix(resource, bucket) {
+			return policyErrInvalidResource
+		}
+	}
+
+	return nil
+}
+
+func (r Resources) FindMatch(resource string) bool {
+	for res := range r {
+		if r.Match(res, resource) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Match matches the given input resource with the pattern
+func (r Resources) Match(pattern, input string) bool {
+	return matchPattern(pattern, input)
+}
+
+// Checks the resource to have arn prefix and not starting with /
+func isValidResource(rc string) (isValid bool, pattern string) {
+	if !strings.HasPrefix(rc, ResourceArnPrefix) {
+		return false, ""
+	}
+
+	res := strings.TrimPrefix(rc, ResourceArnPrefix)
+	if res == "" {
+		return false, ""
+	}
+	// The resource can't start with / (bucket name comes first)
+	if strings.HasPrefix(res, "/") {
+		return false, ""
+	}
+
+	return true, res
+}

auth/bucket_policy_resources_test.go

@@ -0,0 +1,182 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestUnmarshalJSON(t *testing.T) {
+	var r Resources
+
+	cases := []struct {
+		input    string
+		expected int
+		wantErr  bool
+	}{
+		{`"arn:aws:s3:::my-bucket/*"`, 1, false},
+		{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
+		{`""`, 0, true},
+		{`[]`, 0, true},
+		{`["invalid-bucket"]`, 0, true},
+	}
+
+	for _, tc := range cases {
+		r = Resources{}
+		err := json.Unmarshal([]byte(tc.input), &r)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+		if len(r) != tc.expected {
+			t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
+		}
+	}
+}
+
+func TestAdd(t *testing.T) {
+	r := Resources{}
+
+	cases := []struct {
+		input   string
+		wantErr bool
+	}{
+		{"arn:aws:s3:::valid-bucket/*", false},
+		{"arn:aws:s3:::valid-bucket/object", false},
+		{"invalid-bucket/*", true},
+		{"/invalid-start", true},
+	}
+
+	for _, tc := range cases {
+		err := r.Add(tc.input)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
+		}
+	}
+}
+
+func TestContainsObjectPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, true},
+		{[]string{"arn:aws:s3:::my-bucket"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsObjectPattern() != tc.expected {
+			t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestContainsBucketPattern(t *testing.T) {
+	cases := []struct {
+		resources []string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket"}, true},
+		{[]string{"arn:aws:s3:::my-bucket/*"}, false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, false},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.ContainsBucketPattern() != tc.expected {
+			t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
+		}
+	}
+}
+
+func TestValidate(t *testing.T) {
+	cases := []struct {
+		resources []string
+		bucket    string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
+		{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
+		{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if (r.Validate(tc.bucket) == nil) != tc.expected {
+			t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
+		}
+	}
+}
+
+func TestFindMatch(t *testing.T) {
+	cases := []struct {
+		resources []string
+		input     string
+		expected  bool
+	}{
+		{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
+		{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
+		{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
+	}
+
+	for _, tc := range cases {
+		r := Resources{}
+		for _, res := range tc.resources {
+			r.Add(res)
+		}
+		if r.FindMatch(tc.input) != tc.expected {
+			t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
+		}
+	}
+}
+
+func TestMatch(t *testing.T) {
+	r := Resources{}
+	cases := []struct {
+		pattern  string
+		input    string
+		expected bool
+	}{
+		{"my-bucket/*", "my-bucket/object", true},
+		{"my-bucket/?bject", "my-bucket/object", true},
+		{"my-bucket/*", "other-bucket/object", false},
+		{"*", "any-bucket/object", true},
+		{"my-bucket/*", "my-bucket/subdir/object", true},
+		{"my-bucket/*", "other-bucket", false},
+		{"my-bucket/*/*", "my-bucket/hello", false},
+		{"my-bucket/*/*", "my-bucket/hello/world", true},
+		{"foo/???/bar", "foo/qux/bar", true},
+		{"foo/???/bar", "foo/quxx/bar", false},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
+		{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
+	}
+	for _, tc := range cases {
+		if r.Match(tc.pattern, tc.input) != tc.expected {
+			t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
+		}
+	}
+}

auth/iam.go

@@ -0,0 +1,191 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+type Role string
+
+const (
+	RoleUser     Role = "user"
+	RoleAdmin    Role = "admin"
+	RoleUserPlus Role = "userplus"
+)
+
+func (r Role) IsValid() bool {
+	switch r {
+	case RoleAdmin:
+		return true
+	case RoleUser:
+		return true
+	case RoleUserPlus:
+		return true
+	default:
+		return false
+	}
+}
+
+// Account is a gateway IAM account
+type Account struct {
+	Access  string `json:"access"`
+	Secret  string `json:"secret"`
+	Role    Role   `json:"role"`
+	UserID  int    `json:"userID"`
+	GroupID int    `json:"groupID"`
+}
+
+type ListUserAccountsResult struct {
+	Accounts []Account
+}
+
+// Mutable props, which could be changed when updating an IAM account
+type MutableProps struct {
+	Secret  *string `json:"secret"`
+	Role    Role    `json:"role"`
+	UserID  *int    `json:"userID"`
+	GroupID *int    `json:"groupID"`
+}
+
+func (m MutableProps) Validate() error {
+	if m.Role != "" && !m.Role.IsValid() {
+		return s3err.GetAPIError(s3err.ErrAdminInvalidUserRole)
+	}
+
+	return nil
+}
+
+func updateAcc(acc *Account, props MutableProps) {
+	if props.Secret != nil {
+		acc.Secret = *props.Secret
+	}
+	if props.GroupID != nil {
+		acc.GroupID = *props.GroupID
+	}
+	if props.UserID != nil {
+		acc.UserID = *props.UserID
+	}
+	if props.Role != "" {
+		acc.Role = props.Role
+	}
+}
+
+// IAMService is the interface for all IAM service implementations
+//
+//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
+type IAMService interface {
+	CreateAccount(account Account) error
+	GetUserAccount(access string) (Account, error)
+	UpdateUserAccount(access string, props MutableProps) error
+	DeleteUserAccount(access string) error
+	ListUserAccounts() ([]Account, error)
+	Shutdown() error
+}
+
+var (
+	// ErrUserExists is returned when the user already exists
+	ErrUserExists = errors.New("user already exists")
+	// ErrNoSuchUser is returned when the user does not exist
+	ErrNoSuchUser = errors.New("user not found")
+)
+
+type Opts struct {
+	RootAccount            Account
+	Dir                    string
+	LDAPServerURL          string
+	LDAPBindDN             string
+	LDAPPassword           string
+	LDAPQueryBase          string
+	LDAPObjClasses         string
+	LDAPAccessAtr          string
+	LDAPSecretAtr          string
+	LDAPRoleAtr            string
+	LDAPUserIdAtr          string
+	LDAPGroupIdAtr         string
+	VaultEndpointURL       string
+	VaultSecretStoragePath string
+	VaultAuthMethod        string
+	VaultMountPath         string
+	VaultRootToken         string
+	VaultRoleId            string
+	VaultRoleSecret        string
+	VaultServerCert        string
+	VaultClientCert        string
+	VaultClientCertKey     string
+	S3Access               string
+	S3Secret               string
+	S3Region               string
+	S3Bucket               string
+	S3Endpoint             string
+	S3DisableSSlVerfiy     bool
+	CacheDisable           bool
+	CacheTTL               int
+	CachePrune             int
+	IpaHost                string
+	IpaVaultName           string
+	IpaUser                string
+	IpaPassword            string
+	IpaInsecure            bool
+}
+
+func New(o *Opts) (IAMService, error) {
+	var svc IAMService
+	var err error
+
+	switch {
+	case o.Dir != "":
+		svc, err = NewInternal(o.RootAccount, o.Dir)
+		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
+	case o.LDAPServerURL != "":
+		svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
+			o.LDAPGroupIdAtr, o.LDAPObjClasses)
+		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
+	case o.S3Endpoint != "":
+		svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+			o.S3Endpoint, o.S3DisableSSlVerfiy)
+		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
+			o.S3Endpoint, o.S3Bucket)
+	case o.VaultEndpointURL != "":
+		svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
+			o.VaultAuthMethod, o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
+			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
+		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
+	case o.IpaHost != "":
+		svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure)
+		fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
+	default:
+		// if no iam options selected, default to the single user mode
+		fmt.Println("No IAM service configured, enabling single account mode")
+		return NewIAMServiceSingle(o.RootAccount), nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if o.CacheDisable {
+		return svc, nil
+	}
+
+	return NewCache(svc,
+		time.Duration(o.CacheTTL)*time.Second,
+		time.Duration(o.CachePrune)*time.Second), nil
+}

auth/iam_cache.go

@@ -0,0 +1,204 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+)
+
+// IAMCache is an in memory cache of the IAM accounts
+// with expiration. This helps to alleviate the load on
+// the real IAM service if the gateway is handling
+// many requests. This forwards account updates to the
+// underlying service, and returns cached results while
+// the in memory account is not expired.
+type IAMCache struct {
+	service  IAMService
+	iamcache *icache
+	cancel   context.CancelFunc
+}
+
+var _ IAMService = &IAMCache{}
+
+type item struct {
+	value Account
+	exp   time.Time
+}
+
+type icache struct {
+	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
+}
+
+func (i *icache) set(k string, v Account) {
+	cpy := v
+	i.Lock()
+	i.items[k] = item{
+		exp:   time.Now().Add(i.expire),
+		value: cpy,
+	}
+	i.Unlock()
+}
+
+func (i *icache) get(k string) (Account, bool) {
+	i.RLock()
+	v, ok := i.items[k]
+	i.RUnlock()
+	if !ok || !v.exp.After(time.Now()) {
+		return Account{}, false
+	}
+	return v.value, true
+}
+
+func (i *icache) update(k string, props MutableProps) {
+	i.Lock()
+	defer i.Unlock()
+
+	item, found := i.items[k]
+	if found {
+		updateAcc(&item.value, props)
+
+		// refresh the expiration date
+		item.exp = time.Now().Add(i.expire)
+
+		i.items[k] = item
+	}
+}
+
+func (i *icache) Delete(k string) {
+	i.Lock()
+	delete(i.items, k)
+	i.Unlock()
+}
+
+func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+
+		now := time.Now()
+
+		i.Lock()
+		// prune expired entries
+		for k, v := range i.items {
+			if now.After(v.exp) {
+				delete(i.items, k)
+			}
+		}
+		i.Unlock()
+
+		// sleep for the clean interval or context cancelation,
+		// whichever comes first
+		select {
+		case <-ctx.Done():
+		case <-time.After(interval):
+		}
+	}
+}
+
+// NewCache initializes an IAM cache for the provided service. The expireTime
+// is the duration a cache entry can be valid, and the cleanupInterval is
+// how often to scan cache and cleanup expired entries.
+func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
+	i := &IAMCache{
+		service: service,
+		iamcache: &icache{
+			items:  make(map[string]item),
+			expire: expireTime,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go i.iamcache.gcCache(ctx, cleanupInterval)
+	i.cancel = cancel
+
+	return i
+}
+
+// CreateAccount send create to IAM service and creates an account cache entry
+func (c *IAMCache) CreateAccount(account Account) error {
+	err := c.service.CreateAccount(account)
+	if err != nil {
+		return err
+	}
+
+	// we need a copy of account to be able to store beyond the
+	// lifetime of the request, otherwise Fiber will reuse and corrupt
+	// these entries
+	acct := Account{
+		Access: strings.Clone(account.Access),
+		Secret: strings.Clone(account.Secret),
+		Role:   Role(strings.Clone(string(account.Role))),
+	}
+
+	c.iamcache.set(acct.Access, acct)
+	return nil
+}
+
+// GetUserAccount retrieves the cache account if it is in the cache and not
+// expired. Otherwise retrieves from underlying IAM service and caches
+// result for the expire duration.
+func (c *IAMCache) GetUserAccount(access string) (Account, error) {
+	acct, found := c.iamcache.get(access)
+	if found {
+		return acct, nil
+	}
+
+	a, err := c.service.GetUserAccount(access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	c.iamcache.set(access, a)
+	return a, nil
+}
+
+// DeleteUserAccount deletes account from IAM service and cache
+func (c *IAMCache) DeleteUserAccount(access string) error {
+	err := c.service.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.Delete(access)
+	return nil
+}
+
+func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
+	err := c.service.UpdateUserAccount(access, props)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.update(access, props)
+	return nil
+}
+
+// ListUserAccounts is a passthrough to the underlying service and
+// does not make use of the cache
+func (c *IAMCache) ListUserAccounts() ([]Account, error) {
+	return c.service.ListUserAccounts()
+}
+
+// Shutdown graceful termination of service
+func (c *IAMCache) Shutdown() error {
+	c.cancel()
+	return nil
+}

auth/iam_internal.go

@@ -0,0 +1,354 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"time"
+)
+
+const (
+	iamFile       = "users.json"
+	iamBackupFile = "users.json.backup"
+)
+
+// IAMServiceInternal manages the internal IAM service
+type IAMServiceInternal struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+	dir     string
+	rootAcc Account
+}
+
+// UpdateAcctFunc accepts the current data and returns the new data to be stored
+type UpdateAcctFunc func([]byte) ([]byte, error)
+
+// iAMConfig stores all internal IAM accounts
+type iAMConfig struct {
+	AccessAccounts map[string]Account `json:"accessAccounts"`
+}
+
+var _ IAMService = &IAMServiceInternal{}
+
+// NewInternal creates a new instance for the Internal IAM service
+func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
+	i := &IAMServiceInternal{
+		dir:     dir,
+		rootAcc: rootAcc,
+	}
+
+	err := i.initIAM()
+	if err != nil {
+		return nil, fmt.Errorf("init iam: %w", err)
+	}
+
+	return i, nil
+}
+
+// CreateAccount creates a new IAM account. Returns an error if the account
+// already exists.
+func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	if account.Access == s.rootAcc.Access {
+		return ErrUserExists
+	}
+
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		_, ok := conf.AccessAccounts[account.Access]
+		if ok {
+			return nil, ErrUserExists
+		}
+		conf.AccessAccounts[account.Access] = account
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
+// GetUserAccount retrieves account info for the requested user. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
+	s.RLock()
+	defer s.RUnlock()
+
+	conf, err := s.getIAM()
+	if err != nil {
+		return Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+// UpdateUserAccount updates the specified user account fields. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		acc, found := conf.AccessAccounts[access]
+		if !found {
+			return nil, ErrNoSuchUser
+		}
+
+		updateAcc(&acc, props)
+		conf.AccessAccounts[access] = acc
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
+// DeleteUserAccount deletes the specified user account. Does not check if
+// account exists.
+func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		delete(conf.AccessAccounts, access)
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
+// ListUserAccounts lists all the user accounts stored.
+func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	conf, err := s.getIAM()
+	if err != nil {
+		return []Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:  k,
+			Secret:  conf.AccessAccounts[k].Secret,
+			Role:    conf.AccessAccounts[k].Role,
+			UserID:  conf.AccessAccounts[k].UserID,
+			GroupID: conf.AccessAccounts[k].GroupID,
+		})
+	}
+
+	return accs, nil
+}
+
+// Shutdown graceful termination of service
+func (s *IAMServiceInternal) Shutdown() error {
+	return nil
+}
+
+const (
+	iamMode = 0600
+)
+
+func (s *IAMServiceInternal) initIAM() error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	_, err := os.ReadFile(fname)
+	if errors.Is(err, fs.ErrNotExist) {
+		b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
+		if err != nil {
+			return fmt.Errorf("marshal default iam: %w", err)
+		}
+		err = os.WriteFile(fname, b, iamMode)
+		if err != nil {
+			return fmt.Errorf("write default iam: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
+	b, err := s.readIAMData()
+	if err != nil {
+		return iAMConfig{}, err
+	}
+
+	return parseIAM(b)
+}
+
+func parseIAM(b []byte) (iAMConfig, error) {
+	var conf iAMConfig
+	if err := json.Unmarshal(b, &conf); err != nil {
+		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
+	}
+
+	if conf.AccessAccounts == nil {
+		conf.AccessAccounts = make(map[string]Account)
+	}
+
+	return conf, nil
+}
+
+const (
+	backoff  = 100 * time.Millisecond
+	maxretry = 300
+)
+
+func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
+	// We are going to be racing with other running gateways without any
+	// coordination. So we might find the file does not exist at times.
+	// For this case we need to retry for a while assuming the other gateway
+	// will eventually write the file. If it doesn't after the max retries,
+	// then we will return the error.
+
+	retries := 0
+
+	for {
+		b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+			return nil, fmt.Errorf("read iam file: %w", err)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		return b, nil
+	}
+}
+
+func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
+	// We are going to be racing with other running gateways without any
+	// coordination. So the strategy here is to read the current file data,
+	// update the data, write back out to a temp file, then rename the
+	// temp file to the original file. This rename will replace the
+	// original file with the new file. This is atomic and should always
+	// allow for a consistent view of the data. There is a small
+	// window where the file could be read and then updated by
+	// another process. In this case any updates the other process did
+	// will be lost. This is a limitation of the internal IAM service.
+	// This should be rare, and even when it does happen should result
+	// in a valid IAM file, just without the other process's updates.
+
+	iamFname := filepath.Join(s.dir, iamFile)
+	backupFname := filepath.Join(s.dir, iamBackupFile)
+
+	b, err := os.ReadFile(iamFname)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("read iam file: %w", err)
+	}
+
+	// save copy of data
+	datacopy := make([]byte, len(b))
+	copy(datacopy, b)
+
+	// make a backup copy in case something happens
+	err = s.writeUsingTempFile(b, backupFname)
+	if err != nil {
+		return fmt.Errorf("write backup iam file: %w", err)
+	}
+
+	b, err = update(b)
+	if err != nil {
+		return fmt.Errorf("update iam data: %w", err)
+	}
+
+	err = s.writeUsingTempFile(b, iamFname)
+	if err != nil {
+		return fmt.Errorf("write iam file: %w", err)
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) writeUsingTempFile(b []byte, fname string) error {
+	f, err := os.CreateTemp(s.dir, iamFile)
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	_, err = f.Write(b)
+	f.Close()
+	if err != nil {
+		return fmt.Errorf("write temp file: %w", err)
+	}
+
+	err = os.Rename(f.Name(), fname)
+	if err != nil {
+		return fmt.Errorf("rename temp file: %w", err)
+	}
+
+	return nil
+}

auth/iam_ipa.go

@@ -0,0 +1,496 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/versity/versitygw/debuglogger"
+)
+
+const IpaVersion = "2.254"
+
+type IpaIAMService struct {
+	client          http.Client
+	id              int
+	version         string
+	host            string
+	vaultName       string
+	username        string
+	password        string
+	kraTransportKey *rsa.PublicKey
+	rootAcc         Account
+}
+
+var _ IAMService = &IpaIAMService{}
+
+func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure bool) (*IpaIAMService, error) {
+	ipa := IpaIAMService{
+		id:        0,
+		version:   IpaVersion,
+		host:      host,
+		vaultName: vaultName,
+		username:  username,
+		password:  password,
+		rootAcc:   rootAcc,
+	}
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		// this should never happen
+		return nil, fmt.Errorf("cookie jar creation: %w", err)
+	}
+
+	mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
+	tr := &http.Transport{
+		TLSClientConfig: mTLSConfig,
+		Proxy:           http.ProxyFromEnvironment,
+	}
+	ipa.client = http.Client{Jar: jar, Transport: tr}
+
+	err = ipa.login()
+	if err != nil {
+		return nil, fmt.Errorf("ipa login failed: %w", err)
+	}
+
+	req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
+	if err != nil {
+		return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
+	}
+	vaultConfig := struct {
+		Kra_Server_Server             []string
+		Transport_Cert                Base64EncodedWrapped
+		Wrapping_default_algorithm    string
+		Wrapping_supported_algorithms []string
+	}{}
+	err = ipa.rpc(req, &vaultConfig)
+	if err != nil {
+		return nil, fmt.Errorf("ipa vault config: %w", err)
+	}
+
+	cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
+	if err != nil {
+		return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
+	}
+
+	ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
+
+	isSupported := slices.Contains(vaultConfig.Wrapping_supported_algorithms, "aes-128-cbc")
+
+	if !isSupported {
+		return nil,
+			fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
+				vaultConfig.Wrapping_supported_algorithms)
+	}
+	return &ipa, nil
+}
+
+func (ipa *IpaIAMService) CreateAccount(account Account) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ipa.rootAcc.Access {
+		return ipa.rootAcc, nil
+	}
+
+	req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa user_show: %w", err)
+	}
+
+	userResult := struct {
+		Gidnumber []string
+		Uidnumber []string
+	}{}
+
+	err = ipa.rpc(req, &userResult)
+	if err != nil {
+		return Account{}, err
+	}
+
+	uid, err := strconv.Atoi(userResult.Uidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
+	}
+	gid, err := strconv.Atoi(userResult.Gidnumber[0])
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
+	}
+
+	account := Account{
+		Access:  access,
+		Role:    RoleUser,
+		UserID:  uid,
+		GroupID: gid,
+	}
+
+	session_key := make([]byte, 16)
+
+	_, err = rand.Read(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot generate session key: %w", err)
+	}
+
+	encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
+	}
+
+	req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
+		map[string]any{"username": access,
+			"session_key":   Base64EncodedWrapped(encryptedKey),
+			"wrapping_algo": "aes-128-cbc"})
+	if err != nil {
+		return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
+	}
+
+	data := struct {
+		Vault_data Base64EncodedWrapped
+		Nonce      Base64EncodedWrapped
+	}{}
+
+	err = ipa.rpc(req, &data)
+	if err != nil {
+		return account, err
+	}
+
+	aes, err := aes.NewCipher(session_key)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
+	}
+	cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
+	cbc.CryptBlocks(data.Vault_data, data.Vault_data)
+	secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
+	if err != nil {
+		return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
+	}
+
+	secret := struct {
+		Data Base64Encoded
+	}{}
+	json.Unmarshal(secretUnpaddedJson, &secret)
+	account.Secret = string(secret.Data)
+
+	return account, nil
+}
+
+func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
+	return []Account{}, fmt.Errorf("not implemented")
+}
+
+func (ipa *IpaIAMService) Shutdown() error {
+	return nil
+}
+
+// Implementation
+
+const requestRetries = 3
+
+func (ipa *IpaIAMService) login() error {
+	form := url.Values{}
+	form.Set("user", ipa.username)
+	form.Set("password", ipa.password)
+
+	req, err := http.NewRequest(
+		"POST",
+		fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
+		strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	var resp *http.Response
+	for i := range requestRetries {
+		resp, err = ipa.client.Do(req)
+		if err == nil {
+			break
+		}
+		// Check for transient network errors
+		if isRetryable(err) {
+			time.Sleep(time.Second * time.Duration(i+1))
+			continue
+		}
+		return fmt.Errorf("login POST to %s failed: %w", req.URL, err)
+	}
+	if err != nil {
+		return fmt.Errorf("login POST to %s failed after retries: %w",
+			req.URL, err)
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 401 {
+		return errors.New("cannot login to FreeIPA: invalid credentials")
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("cannot login to FreeIPA: status code %d",
+			resp.StatusCode)
+	}
+
+	return nil
+}
+
+type rpcRequest = string
+
+type rpcResponse struct {
+	Result    json.RawMessage
+	Principal string
+	Id        int
+	Version   string
+}
+
+func (p rpcResponse) String() string {
+	return string(p.Result)
+}
+
+var errRpc = errors.New("IPA RPC error")
+
+func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
+	err := ipa.login()
+	if err != nil {
+		return err
+	}
+
+	res, err := ipa.rpcInternal(req)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(res.Result, value)
+}
+
+func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
+	httpReq, err := http.NewRequest("POST",
+		fmt.Sprintf("%s/ipa/session/json", ipa.host),
+		strings.NewReader(req))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	debuglogger.IAMLogf("IPA request: %v", req)
+	httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	var httpResp *http.Response
+	for i := range requestRetries {
+		httpResp, err = ipa.client.Do(httpReq)
+		if err == nil {
+			break
+		}
+		// Check for transient network errors
+		if isRetryable(err) {
+			time.Sleep(time.Second * time.Duration(i+1))
+			continue
+		}
+		return rpcResponse{}, fmt.Errorf("ipa request to %s failed: %w",
+			httpReq.URL, err)
+	}
+	if err != nil {
+		return rpcResponse{},
+			fmt.Errorf("ipa request to %s failed after retries: %w",
+				httpReq.URL, err)
+	}
+
+	defer httpResp.Body.Close()
+
+	bytes, err := io.ReadAll(httpResp.Body)
+	debuglogger.IAMLogf("IPA response (%v): %v", err, string(bytes))
+	if err != nil {
+		return rpcResponse{}, err
+	}
+
+	result := struct {
+		Result struct {
+			Json    json.RawMessage `json:"result"`
+			Value   string          `json:"value"`
+			Summary any             `json:"summary"`
+		} `json:"result"`
+		Error     json.RawMessage `json:"error"`
+		Id        int             `json:"id"`
+		Principal string          `json:"principal"`
+		Version   string          `json:"version"`
+	}{}
+
+	err = json.Unmarshal(bytes, &result)
+	if err != nil {
+		return rpcResponse{}, err
+	}
+	if string(result.Error) != "null" {
+		return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
+	}
+
+	return rpcResponse{
+		Result:    result.Result.Json,
+		Principal: result.Principal,
+		Id:        result.Id,
+		Version:   result.Version,
+	}, nil
+}
+
+func isRetryable(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if errors.Is(err, io.EOF) {
+		return true
+	}
+
+	if err, ok := err.(net.Error); ok && err.Timeout() {
+		return true
+	}
+
+	if opErr, ok := err.(*net.OpError); ok {
+		if sysErr, ok := opErr.Err.(*syscall.Errno); ok {
+			if *sysErr == syscall.ECONNRESET {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
+
+	id := ipa.id
+	ipa.id++
+
+	dict["version"] = ipa.version
+
+	jmethod, errMethod := json.Marshal(method)
+	jargs, errArgs := json.Marshal(args)
+	jdict, errDict := json.Marshal(dict)
+
+	err := errors.Join(errMethod, errArgs, errDict)
+	if err != nil {
+		return "", fmt.Errorf("ipa request invalid: %w", err)
+	}
+
+	request := map[string]interface{}{
+		"id":     id,
+		"method": json.RawMessage(jmethod),
+		"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
+	}
+
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	return string(requestJSON), nil
+}
+
+// pkcs7Unpad validates and unpads data from the given bytes slice.
+// The returned value will be 1 to n bytes smaller depending on the
+// amount of padding, where n is the block size.
+func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
+	if blocksize <= 0 {
+		return nil, errors.New("invalid blocksize")
+	}
+	if len(b) == 0 {
+		return nil, errors.New("invalid PKCS7 data (empty or not padded)")
+	}
+	if len(b)%blocksize != 0 {
+		return nil, errors.New("invalid padding on input")
+	}
+	c := b[len(b)-1]
+	n := int(c)
+	if n == 0 || n > len(b) {
+		return nil, errors.New("invalid padding on input")
+	}
+	for i := 0; i < n; i++ {
+		if b[len(b)-n+i] != c {
+			return nil, errors.New("invalid padding on input")
+		}
+	}
+	return b[:len(b)-n], nil
+}
+
+/*
+e.g.
+
+	"value" {
+		"__base64__": "aGVsbG93b3JsZAo="
+	 }
+*/
+type Base64EncodedWrapped []byte
+
+func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{}
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
+	return err
+}
+
+func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
+	intermediate := struct {
+		Base64 string `json:"__base64__"`
+	}{Base64: base64.StdEncoding.EncodeToString(*b)}
+	return json.Marshal(intermediate)
+}
+
+/*
+e.g.
+
+	"value": "aGVsbG93b3JsZAo="
+*/
+type Base64Encoded []byte
+
+func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
+	var intermediate string
+	err := json.Unmarshal(data, &intermediate)
+	if err != nil {
+		return err
+	}
+	*b, err = base64.StdEncoding.DecodeString(intermediate)
+	return err
+}

auth/iam_ldap.go

@@ -0,0 +1,293 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/go-ldap/ldap/v3"
+	"github.com/versity/versitygw/debuglogger"
+)
+
+type LdapIAMService struct {
+	conn       *ldap.Conn
+	queryBase  string
+	objClasses []string
+	accessAtr  string
+	secretAtr  string
+	roleAtr    string
+	groupIdAtr string
+	userIdAtr  string
+	rootAcc    Account
+	url        string
+	bindDN     string
+	pass       string
+	mu         sync.Mutex
+}
+
+var _ IAMService = &LdapIAMService{}
+
+func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
+		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
+		return nil, fmt.Errorf("required parameters list not fully provided")
+	}
+	conn, err := ldap.DialURL(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
+	}
+
+	err = conn.Bind(bindDN, pass)
+	if err != nil {
+		return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
+	}
+	return &LdapIAMService{
+		conn:       conn,
+		queryBase:  queryBase,
+		objClasses: strings.Split(objClasses, ","),
+		accessAtr:  accAtr,
+		secretAtr:  secAtr,
+		roleAtr:    roleAtr,
+		userIdAtr:  userIdAtr,
+		groupIdAtr: groupIdAtr,
+		rootAcc:    rootAcc,
+		url:        url,
+		bindDN:     bindDN,
+		pass:       pass,
+	}, nil
+}
+
+func (ld *LdapIAMService) reconnect() error {
+	ld.conn.Close()
+
+	conn, err := ldap.DialURL(ld.url)
+	if err != nil {
+		return fmt.Errorf("failed to reconnect to LDAP server: %w", err)
+	}
+
+	err = conn.Bind(ld.bindDN, ld.pass)
+	if err != nil {
+		conn.Close()
+		return fmt.Errorf("failed to bind to LDAP server on reconnect: %w", err)
+	}
+	ld.conn = conn
+	return nil
+}
+
+func (ld *LdapIAMService) execute(f func(*ldap.Conn) error) error {
+	ld.mu.Lock()
+	defer ld.mu.Unlock()
+
+	err := f(ld.conn)
+	if err != nil {
+		if e, ok := err.(*ldap.Error); ok && e.ResultCode == ldap.ErrorNetwork {
+			if reconnErr := ld.reconnect(); reconnErr != nil {
+				return reconnErr
+			}
+			return f(ld.conn)
+		}
+	}
+	return err
+}
+
+func (ld *LdapIAMService) CreateAccount(account Account) error {
+	if ld.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry.Attribute("objectClass", ld.objClasses)
+	userEntry.Attribute(ld.accessAtr, []string{account.Access})
+	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
+	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
+	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
+	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
+
+	err := ld.execute(func(c *ldap.Conn) error {
+		return c.Add(userEntry)
+	})
+	if err != nil {
+		return fmt.Errorf("error adding an entry: %w", err)
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) buildSearchFilter(access string) string {
+	var searchFilter strings.Builder
+	for _, el := range ld.objClasses {
+		searchFilter.WriteString(fmt.Sprintf("(objectClass=%v)", el))
+	}
+	if access != "" {
+		searchFilter.WriteString(fmt.Sprintf("(%v=%v)", ld.accessAtr, access))
+	}
+	return fmt.Sprintf("(&%v)", searchFilter.String())
+}
+
+func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	if access == ld.rootAcc.Access {
+		return ld.rootAcc, nil
+	}
+	var result *ldap.SearchResult
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		ld.buildSearchFilter(access),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
+		nil,
+	)
+
+	if debuglogger.IsIAMDebugEnabled() {
+		debuglogger.IAMLogf("LDAP Search Request")
+		debuglogger.IAMLogf(spew.Sdump(searchRequest))
+	}
+
+	err := ld.execute(func(c *ldap.Conn) error {
+		var err error
+		result, err = c.Search(searchRequest)
+		return err
+	})
+
+	if debuglogger.IsIAMDebugEnabled() {
+		debuglogger.IAMLogf("LDAP Search Result")
+		debuglogger.IAMLogf(spew.Sdump(result))
+	}
+
+	if err != nil {
+		return Account{}, err
+	}
+
+	if len(result.Entries) == 0 {
+		return Account{}, ErrNoSuchUser
+	}
+
+	entry := result.Entries[0]
+	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id %q: %w",
+			entry.GetAttributeValue(ld.groupIdAtr), err)
+	}
+	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for user-id %q: %w",
+			entry.GetAttributeValue(ld.userIdAtr), err)
+	}
+	return Account{
+		Access:  entry.GetAttributeValue(ld.accessAtr),
+		Secret:  entry.GetAttributeValue(ld.secretAtr),
+		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
+		GroupID: groupId,
+		UserID:  userId,
+	}, nil
+}
+
+func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+	if props.Secret != nil {
+		req.Replace(ld.secretAtr, []string{*props.Secret})
+	}
+	if props.GroupID != nil {
+		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
+	}
+	if props.UserID != nil {
+		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
+	}
+	if props.Role != "" {
+		req.Replace(ld.roleAtr, []string{string(props.Role)})
+	}
+
+	err := ld.execute(func(c *ldap.Conn) error {
+		return c.Modify(req)
+	})
+	//TODO: Handle non existing user case
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ld *LdapIAMService) DeleteUserAccount(access string) error {
+	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+
+	err := ld.execute(func(c *ldap.Conn) error {
+		return c.Del(delReq)
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
+	var resp *ldap.SearchResult
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		ld.buildSearchFilter(""),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
+		nil,
+	)
+
+	err := ld.execute(func(c *ldap.Conn) error {
+		var err error
+		resp, err = c.Search(searchRequest)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	result := []Account{}
+	for _, el := range resp.Entries {
+		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id %q: %w",
+				el.GetAttributeValue(ld.groupIdAtr), err)
+		}
+		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for user-id %q: %w",
+				el.GetAttributeValue(ld.userIdAtr), err)
+		}
+		result = append(result, Account{
+			Access:  el.GetAttributeValue(ld.accessAtr),
+			Secret:  el.GetAttributeValue(ld.secretAtr),
+			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
+			GroupID: groupId,
+			UserID:  userId,
+		})
+	}
+
+	return result, nil
+}
+
+// Shutdown graceful termination of service
+func (ld *LdapIAMService) Shutdown() error {
+	ld.mu.Lock()
+	defer ld.mu.Unlock()
+	return ld.conn.Close()
+}

auth/iam_ldap_test.go

@@ -0,0 +1,56 @@
+package auth
+
+import "testing"
+
+func TestLdapIAMService_BuildSearchFilter(t *testing.T) {
+	tests := []struct {
+		name       string
+		objClasses []string
+		accessAtr  string
+		access     string
+		expected   string
+	}{
+		{
+			name:       "single object class with access",
+			objClasses: []string{"inetOrgPerson"},
+			accessAtr:  "uid",
+			access:     "testuser",
+			expected:   "(&(objectClass=inetOrgPerson)(uid=testuser))",
+		},
+		{
+			name:       "single object class without access",
+			objClasses: []string{"inetOrgPerson"},
+			accessAtr:  "uid",
+			access:     "",
+			expected:   "(&(objectClass=inetOrgPerson))",
+		},
+		{
+			name:       "multiple object classes with access",
+			objClasses: []string{"inetOrgPerson", "organizationalPerson"},
+			accessAtr:  "cn",
+			access:     "john.doe",
+			expected:   "(&(objectClass=inetOrgPerson)(objectClass=organizationalPerson)(cn=john.doe))",
+		},
+		{
+			name:       "multiple object classes without access",
+			objClasses: []string{"inetOrgPerson", "organizationalPerson", "person"},
+			accessAtr:  "cn",
+			access:     "",
+			expected:   "(&(objectClass=inetOrgPerson)(objectClass=organizationalPerson)(objectClass=person))",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ld := &LdapIAMService{
+				objClasses: tt.objClasses,
+				accessAtr:  tt.accessAtr,
+			}
+
+			result := ld.buildSearchFilter(tt.access)
+			if result != tt.expected {
+				t.Errorf("BuildSearchFilter() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}

auth/iam_s3_object.go

@@ -0,0 +1,304 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"sync"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/debuglogger"
+)
+
+// IAMServiceS3 stores user accounts in an S3 object
+// The endpoint, credentials, bucket, and region are provided
+// from cli configuration.
+// The object format and name is the same as the internal IAM service:
+// coming from iAMConfig and iamFile in iam_internal.
+
+type IAMServiceS3 struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
+	sslSkipVerify bool
+	rootAcc       Account
+	client        *s3.Client
+}
+
+var _ IAMService = &IAMServiceS3{}
+
+func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify bool) (*IAMServiceS3, error) {
+	if access == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service access key")
+	}
+	if secret == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service secret key")
+	}
+	if region == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service region")
+	}
+	if bucket == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service bucket")
+	}
+	if endpoint == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service endpoint")
+	}
+
+	i := &IAMServiceS3{
+		access:        access,
+		secret:        secret,
+		region:        region,
+		bucket:        bucket,
+		endpoint:      endpoint,
+		sslSkipVerify: sslSkipVerify,
+		rootAcc:       rootAcc,
+	}
+
+	cfg, err := i.getConfig()
+	if err != nil {
+		return nil, fmt.Errorf("init s3 IAM: %v", err)
+	}
+
+	if endpoint != "" {
+		i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &endpoint
+		})
+		return i, nil
+	}
+
+	i.client = s3.NewFromConfig(cfg)
+	return i, nil
+}
+
+func (s *IAMServiceS3) CreateAccount(account Account) error {
+	if s.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[account.Access]
+	if ok {
+		return ErrUserExists
+	}
+	conf.AccessAccounts[account.Access] = account
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	if access == s.rootAcc.Access {
+		return s.rootAcc, nil
+	}
+
+	s.RLock()
+	defer s.RUnlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	acc, ok := conf.AccessAccounts[access]
+	if !ok {
+		return ErrNoSuchUser
+	}
+
+	updateAcc(&acc, props)
+	conf.AccessAccounts[access] = acc
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[access]
+	if !ok {
+		return fmt.Errorf("account does not exist")
+	}
+	delete(conf.AccessAccounts, access)
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:  k,
+			Secret:  conf.AccessAccounts[k].Secret,
+			Role:    conf.AccessAccounts[k].Role,
+			UserID:  conf.AccessAccounts[k].UserID,
+			GroupID: conf.AccessAccounts[k].GroupID,
+		})
+	}
+
+	return accs, nil
+}
+
+func (s *IAMServiceS3) Shutdown() error {
+	return nil
+}
+
+func (s *IAMServiceS3) getConfig() (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.region),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if debuglogger.IsIAMDebugEnabled() {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(context.Background(), opts...)
+}
+
+func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
+	obj := iamFile
+
+	out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
+		Bucket: &s.bucket,
+		Key:    &obj,
+	})
+	if err != nil {
+		// if the error is object not exists,
+		// init empty accounts struct and return that
+		var nsk *types.NoSuchKey
+		if errors.As(err, &nsk) {
+			return iAMConfig{AccessAccounts: map[string]Account{}}, nil
+		}
+		var apiErr smithy.APIError
+		if errors.As(err, &apiErr) {
+			if apiErr.ErrorCode() == "NotFound" {
+				return iAMConfig{AccessAccounts: map[string]Account{}}, nil
+			}
+		}
+
+		// all other errors, return the error
+		return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
+	}
+
+	defer out.Body.Close()
+
+	b, err := io.ReadAll(out.Body)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
+	}
+
+	conf, err := parseIAM(b)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
+	}
+
+	return conf, nil
+}
+
+func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
+	b, err := json.Marshal(conf)
+	if err != nil {
+		return fmt.Errorf("failed to serialize iam: %w", err)
+	}
+
+	obj := iamFile
+	uploader := manager.NewUploader(s.client)
+	upinfo := &s3.PutObjectInput{
+		Body:   bytes.NewReader(b),
+		Bucket: &s.bucket,
+		Key:    &obj,
+	}
+	_, err = uploader.Upload(context.Background(), upinfo)
+	if err != nil {
+		return fmt.Errorf("store accounts in %v: %w", iamFile, err)
+	}
+
+	return nil
+}

auth/iam_single.go

@@ -0,0 +1,66 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"github.com/versity/versitygw/s3err"
+)
+
+// IAMServiceSingle manages the single tenant (root-only) IAM service
+type IAMServiceSingle struct {
+	root Account
+}
+
+var _ IAMService = &IAMServiceSingle{}
+
+func NewIAMServiceSingle(r Account) IAMService {
+	return &IAMServiceSingle{
+		root: r,
+	}
+}
+
+// CreateAccount not valid in single tenant mode
+func (IAMServiceSingle) CreateAccount(account Account) error {
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+}
+
+// GetUserAccount returns root account, if the root access key
+// is provided and "ErrAdminUserNotFound" otherwise
+func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	if access == s.root.Access {
+		return s.root, nil
+	}
+	return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound)
+}
+
+// UpdateUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+}
+
+// DeleteUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) DeleteUserAccount(access string) error {
+	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+}
+
+// ListUserAccounts no accounts in single tenant mode
+func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
+	return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+}
+
+// Shutdown graceful termination of service
+func (IAMServiceSingle) Shutdown() error {
+	return nil
+}

auth/iam_vault.go

@@ -0,0 +1,330 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	vault "github.com/hashicorp/vault-client-go"
+	"github.com/hashicorp/vault-client-go/schema"
+)
+
+const requestTimeout = 10 * time.Second
+
+type VaultIAMService struct {
+	client            *vault.Client
+	authReqOpts       []vault.RequestOption
+	kvReqOpts         []vault.RequestOption
+	secretStoragePath string
+	rootAcc           Account
+	creds             schema.AppRoleLoginRequest
+}
+
+var _ IAMService = &VaultIAMService{}
+
+func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath,
+	authMethod, mountPath, rootToken, roleID, roleSecret, serverCert,
+	clientCert, clientCertKey string) (IAMService, error) {
+	opts := []vault.ClientOption{
+		vault.WithAddress(endpoint),
+		vault.WithRequestTimeout(requestTimeout),
+	}
+	if serverCert != "" {
+		tls := vault.TLSConfiguration{}
+
+		tls.ServerCertificate.FromBytes = []byte(serverCert)
+		if clientCert != "" {
+			if clientCertKey == "" {
+				return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
+			}
+
+			tls.ClientCertificate.FromBytes = []byte(clientCert)
+			tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
+		}
+
+		opts = append(opts, vault.WithTLS(tls))
+	}
+
+	client, err := vault.New(opts...)
+	if err != nil {
+		return nil, fmt.Errorf("init vault client: %w", err)
+	}
+
+	authReqOpts := []vault.RequestOption{}
+	// if auth method path is not specified, it defaults to "approle"
+	if authMethod != "" {
+		authReqOpts = append(authReqOpts, vault.WithMountPath(authMethod))
+	}
+
+	kvReqOpts := []vault.RequestOption{}
+	// if mount path is not specified, it defaults to "kv-v2"
+	if mountPath != "" {
+		kvReqOpts = append(kvReqOpts, vault.WithMountPath(mountPath))
+	}
+
+	creds := schema.AppRoleLoginRequest{
+		RoleId:   roleID,
+		SecretId: roleSecret,
+	}
+
+	// Authentication
+	switch {
+	case rootToken != "":
+		err := client.SetToken(rootToken)
+		if err != nil {
+			return nil, fmt.Errorf("root token authentication failure: %w", err)
+		}
+	case roleID != "":
+		if roleSecret == "" {
+			return nil, fmt.Errorf("role id and role secret must both be specified")
+		}
+
+		resp, err := client.Auth.AppRoleLogin(context.Background(),
+			creds, authReqOpts...)
+		if err != nil {
+			return nil, fmt.Errorf("approle authentication failure: %w", err)
+		}
+
+		if err := client.SetToken(resp.Auth.ClientToken); err != nil {
+			return nil, fmt.Errorf("approle authentication set token failure: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
+	}
+
+	return &VaultIAMService{
+		client:            client,
+		authReqOpts:       authReqOpts,
+		kvReqOpts:         kvReqOpts,
+		secretStoragePath: secretStoragePath,
+		rootAcc:           rootAcc,
+		creds:             creds,
+	}, nil
+}
+
+func (vt *VaultIAMService) reAuthIfNeeded(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	// Vault returns 403 for expired/revoked tokens
+	// pass all other errors back unchanged
+	if !vault.IsErrorStatus(err, http.StatusForbidden) {
+		return err
+	}
+
+	resp, authErr := vt.client.Auth.AppRoleLogin(context.Background(),
+		vt.creds, vt.authReqOpts...)
+	if authErr != nil {
+		return fmt.Errorf("vault re-authentication failure: %w", authErr)
+	}
+	if err := vt.client.SetToken(resp.Auth.ClientToken); err != nil {
+		return fmt.Errorf("vault re-authentication set token failure: %w", err)
+	}
+
+	return nil
+}
+
+func (vt *VaultIAMService) CreateAccount(account Account) error {
+	if vt.rootAcc.Access == account.Access {
+		return ErrUserExists
+	}
+	_, err := vt.client.Secrets.KvV2Write(context.Background(),
+		vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
+			Data: map[string]any{
+				account.Access: account,
+			},
+			Options: map[string]any{
+				"cas": 0,
+			},
+		}, vt.kvReqOpts...)
+	if err != nil {
+		if strings.Contains(err.Error(), "check-and-set") {
+			return ErrUserExists
+		}
+
+		reauthErr := vt.reAuthIfNeeded(err)
+		if reauthErr != nil {
+			return reauthErr
+		}
+		// retry once after re-auth
+		_, err = vt.client.Secrets.KvV2Write(context.Background(),
+			vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
+				Data: map[string]any{
+					account.Access: account,
+				},
+				Options: map[string]any{
+					"cas": 0,
+				},
+			}, vt.kvReqOpts...)
+		if err != nil {
+			if strings.Contains(err.Error(), "check-and-set") {
+				return ErrUserExists
+			}
+			return err
+		}
+		return nil
+	}
+	return nil
+}
+
+func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
+	if vt.rootAcc.Access == access {
+		return vt.rootAcc, nil
+	}
+	resp, err := vt.client.Secrets.KvV2Read(context.Background(),
+		vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
+	if err != nil {
+		reauthErr := vt.reAuthIfNeeded(err)
+		if reauthErr != nil {
+			return Account{}, reauthErr
+		}
+		// retry once after re-auth
+		resp, err = vt.client.Secrets.KvV2Read(context.Background(),
+			vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
+		if err != nil {
+			return Account{}, err
+		}
+	}
+	acc, err := parseVaultUserAccount(resp.Data.Data, access)
+	if err != nil {
+		return Account{}, err
+	}
+	return acc, nil
+}
+
+func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	acc, err := vt.GetUserAccount(access)
+	if err != nil {
+		return err
+	}
+	updateAcc(&acc, props)
+	err = vt.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+	err = vt.CreateAccount(acc)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (vt *VaultIAMService) DeleteUserAccount(access string) error {
+	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(context.Background(),
+		vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
+	if err != nil {
+		reauthErr := vt.reAuthIfNeeded(err)
+		if reauthErr != nil {
+			return reauthErr
+		}
+		// retry once after re-auth
+		_, err = vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(context.Background(),
+			vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
+	resp, err := vt.client.Secrets.KvV2List(context.Background(),
+		vt.secretStoragePath, vt.kvReqOpts...)
+	if err != nil {
+		reauthErr := vt.reAuthIfNeeded(err)
+		if reauthErr != nil {
+			if vault.IsErrorStatus(err, http.StatusNotFound) {
+				return []Account{}, nil
+			}
+			return nil, reauthErr
+		}
+		// retry once after re-auth
+		resp, err = vt.client.Secrets.KvV2List(context.Background(),
+			vt.secretStoragePath, vt.kvReqOpts...)
+		if err != nil {
+			if vault.IsErrorStatus(err, http.StatusNotFound) {
+				return []Account{}, nil
+			}
+			return nil, err
+		}
+	}
+	accs := []Account{}
+	for _, acss := range resp.Data.Keys {
+		acc, err := vt.GetUserAccount(acss)
+		if err != nil {
+			return nil, err
+		}
+		accs = append(accs, acc)
+	}
+	return accs, nil
+}
+
+// the client doesn't have explicit shutdown, as it uses http.Client
+func (vt *VaultIAMService) Shutdown() error {
+	return nil
+}
+
+var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
+
+func parseVaultUserAccount(data map[string]any, access string) (acc Account, err error) {
+	usrAcc, ok := data[access].(map[string]any)
+	if !ok {
+		return acc, errInvalidUser
+	}
+
+	acss, ok := usrAcc["access"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	secret, ok := usrAcc["secret"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	role, ok := usrAcc["role"].(string)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userIdJson, ok := usrAcc["userID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	userId, err := userIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+	groupIdJson, ok := usrAcc["groupID"].(json.Number)
+	if !ok {
+		return acc, errInvalidUser
+	}
+	groupId, err := groupIdJson.Int64()
+	if err != nil {
+		return acc, errInvalidUser
+	}
+
+	return Account{
+		Access:  acss,
+		Secret:  secret,
+		Role:    Role(role),
+		UserID:  int(userId),
+		GroupID: int(groupId),
+	}, nil
+}

auth/object_lock.go

@@ -0,0 +1,351 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/debuglogger"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+type BucketLockConfig struct {
+	Enabled          bool
+	DefaultRetention *types.DefaultRetention
+	CreatedAt        *time.Time
+}
+
+func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
+	var lockConfig types.ObjectLockConfiguration
+	if err := xml.Unmarshal(input, &lockConfig); err != nil {
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	config := BucketLockConfig{
+		Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
+	}
+
+	if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
+		retention := lockConfig.Rule.DefaultRetention
+
+		if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
+			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+		}
+		if retention.Years != nil && retention.Days != nil {
+			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+		}
+
+		if retention.Days != nil && *retention.Days <= 0 {
+			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
+		}
+		if retention.Years != nil && *retention.Years <= 0 {
+			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
+		}
+
+		config.DefaultRetention = retention
+		now := time.Now()
+		config.CreatedAt = &now
+	}
+
+	return json.Marshal(config)
+}
+
+func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
+	var config BucketLockConfig
+	if err := json.Unmarshal(input, &config); err != nil {
+		return nil, fmt.Errorf("parse object lock config: %w", err)
+	}
+
+	result := &types.ObjectLockConfiguration{
+		Rule: &types.ObjectLockRule{
+			DefaultRetention: config.DefaultRetention,
+		},
+	}
+
+	if config.Enabled {
+		result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
+	}
+
+	return result, nil
+}
+
+func ParseObjectLockRetentionInput(input []byte) (*s3response.PutObjectRetentionInput, error) {
+	var retention s3response.PutObjectRetentionInput
+	if err := xml.Unmarshal(input, &retention); err != nil {
+		debuglogger.Logf("invalid object lock retention request body: %v", err)
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	if retention.RetainUntilDate.Before(time.Now()) {
+		debuglogger.Logf("object lock retain until date must be in the future")
+		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
+	}
+	switch retention.Mode {
+	case types.ObjectLockRetentionModeCompliance:
+	case types.ObjectLockRetentionModeGovernance:
+	default:
+		debuglogger.Logf("invalid object lock retention mode: %s", retention.Mode)
+		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
+	}
+
+	return &retention, nil
+}
+
+func ParseObjectLockRetentionInputToJSON(input *s3response.PutObjectRetentionInput) ([]byte, error) {
+	data, err := json.Marshal(input)
+	if err != nil {
+		debuglogger.Logf("parse object lock retention to JSON: %v", err)
+		return nil, fmt.Errorf("parse object lock retention: %w", err)
+	}
+
+	return data, nil
+}
+
+// IsObjectLockRetentionPutAllowed checks if the object lock retention PUT request
+// is allowed against the current state of the object lock
+func IsObjectLockRetentionPutAllowed(ctx context.Context, be backend.Backend, bucket, object, versionId, userAccess string, input *s3response.PutObjectRetentionInput, bypass bool) error {
+	ret, err := be.GetObjectRetention(ctx, bucket, object, versionId)
+	if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
+		// if object lock configuration is not set
+		// allow the retention modification without any checks
+		return nil
+	}
+	if err != nil {
+		debuglogger.Logf("failed to get object retention: %v", err)
+		return err
+	}
+
+	retention, err := ParseObjectLockRetentionOutput(ret)
+	if err != nil {
+		return err
+	}
+
+	if retention.Mode == input.Mode {
+		// if retention mode is the same
+		// the operation is allowed
+		return nil
+	}
+
+	if retention.Mode == types.ObjectLockRetentionModeCompliance {
+		// COMPLIANCE mode is by definition not allowed to modify
+		debuglogger.Logf("object lock retention change request from 'COMPLIANCE' to 'GOVERNANCE' is not allowed")
+		return s3err.GetAPIError(s3err.ErrObjectLocked)
+	}
+
+	if !bypass {
+		// if x-amz-bypass-governance-retention is not provided
+		// return error: object is locked
+		debuglogger.Logf("object lock retention mode change is not allowed and bypass governence is not forced")
+		return s3err.GetAPIError(s3err.ErrObjectLocked)
+	}
+
+	// the last case left, when user tries to chenge
+	// from 'GOVERNANCE' to 'COMPLIANCE' with
+	// 'x-amz-bypass-governance-retention' header
+	// first we need to check if user has 's3:BypassGovernanceRetention'
+	policy, err := be.GetBucketPolicy(ctx, bucket)
+	if err != nil {
+		// if it fails to get the policy, return object is locked
+		debuglogger.Logf("failed to get the bucket policy: %v", err)
+		return s3err.GetAPIError(s3err.ErrObjectLocked)
+	}
+	err = VerifyBucketPolicy(policy, userAccess, bucket, object, BypassGovernanceRetentionAction)
+	if err != nil {
+		// if user doesn't have "s3:BypassGovernanceRetention" permission
+		// return object is locked
+		debuglogger.Logf("the user is missing 's3:BypassGovernanceRetention' permission")
+		return s3err.GetAPIError(s3err.ErrObjectLocked)
+	}
+
+	return nil
+}
+
+func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
+	var retention types.ObjectLockRetention
+	if err := json.Unmarshal(input, &retention); err != nil {
+		debuglogger.Logf("parse object lock retention output: %v", err)
+		return nil, fmt.Errorf("parse object lock retention: %w", err)
+	}
+
+	return &retention, nil
+}
+
+func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResult {
+	if status == nil {
+		return nil
+	}
+
+	if *status {
+		return &s3response.GetObjectLegalHoldResult{
+			Status: types.ObjectLockLegalHoldStatusOn,
+		}
+	}
+
+	return &s3response.GetObjectLegalHoldResult{
+		Status: types.ObjectLockLegalHoldStatusOff,
+	}
+}
+
+func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass, isBucketPublic bool, be backend.Backend) error {
+	data, err := be.GetObjectLockConfiguration(ctx, bucket)
+	if err != nil {
+		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
+			return nil
+		}
+
+		return err
+	}
+
+	var bucketLockConfig BucketLockConfig
+	if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
+		return fmt.Errorf("parse object lock config: %w", err)
+	}
+
+	if !bucketLockConfig.Enabled {
+		return nil
+	}
+
+	checkDefaultRetention := false
+
+	if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
+		expirationDate := *bucketLockConfig.CreatedAt
+		if bucketLockConfig.DefaultRetention.Days != nil {
+			expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
+		}
+		if bucketLockConfig.DefaultRetention.Years != nil {
+			expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
+		}
+
+		if expirationDate.After(time.Now()) {
+			checkDefaultRetention = true
+		}
+	}
+
+	for _, obj := range objects {
+		var key, versionId string
+		if obj.Key != nil {
+			key = *obj.Key
+		}
+		if obj.VersionId != nil {
+			versionId = *obj.VersionId
+		}
+		checkRetention := true
+		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
+		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+			continue
+		}
+		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
+			checkRetention = false
+		}
+		if err != nil && checkRetention {
+			return err
+		}
+
+		if checkRetention {
+			retention, err := ParseObjectLockRetentionOutput(retentionData)
+			if err != nil {
+				return err
+			}
+
+			if retention.Mode != "" && retention.RetainUntilDate != nil {
+				if retention.RetainUntilDate.After(time.Now()) {
+					switch retention.Mode {
+					case types.ObjectLockRetentionModeGovernance:
+						if !bypass {
+							return s3err.GetAPIError(s3err.ErrObjectLocked)
+						} else {
+							policy, err := be.GetBucketPolicy(ctx, bucket)
+							if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+								return s3err.GetAPIError(s3err.ErrObjectLocked)
+							}
+							if err != nil {
+								return err
+							}
+							if isBucketPublic {
+								err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
+							} else {
+								err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+							}
+							if err != nil {
+								return s3err.GetAPIError(s3err.ErrObjectLocked)
+							}
+						}
+					case types.ObjectLockRetentionModeCompliance:
+						return s3err.GetAPIError(s3err.ErrObjectLocked)
+					}
+				}
+			}
+		}
+
+		checkLegalHold := true
+
+		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
+		if err != nil {
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
+				continue
+			}
+			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
+				checkLegalHold = false
+			} else {
+				return err
+			}
+		}
+
+		if checkLegalHold && *status {
+			return s3err.GetAPIError(s3err.ErrObjectLocked)
+		}
+
+		if checkDefaultRetention {
+			switch bucketLockConfig.DefaultRetention.Mode {
+			case types.ObjectLockRetentionModeGovernance:
+				if !bypass {
+					return s3err.GetAPIError(s3err.ErrObjectLocked)
+				} else {
+					policy, err := be.GetBucketPolicy(ctx, bucket)
+					if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
+						return s3err.GetAPIError(s3err.ErrObjectLocked)
+					}
+					if err != nil {
+						return err
+					}
+					if isBucketPublic {
+						err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
+					} else {
+						err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
+					}
+					if err != nil {
+						return s3err.GetAPIError(s3err.ErrObjectLocked)
+					}
+				}
+			case types.ObjectLockRetentionModeCompliance:
+				return s3err.GetAPIError(s3err.ErrObjectLocked)
+			}
+		}
+	}
+
+	return nil
+}

aws/LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

aws/NOTICE.txt

@@ -0,0 +1,4 @@
+AWS SDK for Go
+Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2014-2015 Stripe, Inc.
+Copyright 2024 Versity Software

aws/README.md

@@ -0,0 +1,11 @@
+# AWS SDK Go v2
+
+This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License.
+
+## Description
+
+The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements.
+
+## License
+
+The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information.

aws/internal/awstesting/unit/unit.go

@@ -0,0 +1,61 @@
+// Package unit performs initialization and validation for unit tests
+package unit
+
+import (
+	"context"
+	"crypto/rsa"
+	"math/big"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func init() {
+	config = aws.Config{}
+	config.Region = "mock-region"
+	config.Credentials = StubCredentialsProvider{}
+}
+
+// StubCredentialsProvider provides a stub credential provider that returns
+// static credentials that never expire.
+type StubCredentialsProvider struct{}
+
+// Retrieve satisfies the CredentialsProvider interface. Returns stub
+// credential value, and never error.
+func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) {
+	return aws.Credentials{
+		AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION",
+		Source: "unit test credentials",
+	}, nil
+}
+
+var config aws.Config
+
+// Config returns a copy of the mock configuration for unit tests.
+func Config() aws.Config { return config.Copy() }
+
+// RSAPrivateKey is used for testing functionality that requires some
+// sort of private key. Taken from crypto/rsa/rsa_test.go
+//
+// Credit to golang 1.11
+var RSAPrivateKey = &rsa.PrivateKey{
+	PublicKey: rsa.PublicKey{
+		N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"),
+		E: 3,
+	},
+	D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"),
+	Primes: []*big.Int{
+		fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"),
+		fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"),
+	},
+}
+
+// Taken from crypto/rsa/rsa_test.go
+//
+// Credit to golang 1.11
+func fromBase10(base10 string) *big.Int {
+	i, ok := new(big.Int).SetString(base10, 10)
+	if !ok {
+		panic("bad number: " + base10)
+	}
+	return i
+}

aws/signer/internal/v4/cache.go

@@ -0,0 +1,115 @@
+package v4
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func lookupKey(service, region string) string {
+	var s strings.Builder
+	s.Grow(len(region) + len(service) + 3)
+	s.WriteString(region)
+	s.WriteRune('/')
+	s.WriteString(service)
+	return s.String()
+}
+
+type derivedKey struct {
+	AccessKey  string
+	Date       time.Time
+	Credential []byte
+}
+
+type derivedKeyCache struct {
+	values map[string]derivedKey
+	mutex  sync.RWMutex
+}
+
+func newDerivedKeyCache() derivedKeyCache {
+	return derivedKeyCache{
+		values: make(map[string]derivedKey),
+	}
+}
+
+func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte {
+	key := lookupKey(service, region)
+	s.mutex.RLock()
+	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
+		s.mutex.RUnlock()
+		return cred
+	}
+	s.mutex.RUnlock()
+
+	s.mutex.Lock()
+	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
+		s.mutex.Unlock()
+		return cred
+	}
+	cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime)
+	entry := derivedKey{
+		AccessKey:  credentials.AccessKeyID,
+		Date:       signingTime.Time,
+		Credential: cred,
+	}
+	s.values[key] = entry
+	s.mutex.Unlock()
+
+	return cred
+}
+
+func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) {
+	cacheEntry, ok := s.retrieveFromCache(key)
+	if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) {
+		return cacheEntry.Credential, true
+	}
+	return nil, false
+}
+
+func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) {
+	if v, ok := s.values[key]; ok {
+		return v, true
+	}
+	return derivedKey{}, false
+}
+
+// SigningKeyDeriver derives a signing key from a set of credentials
+type SigningKeyDeriver struct {
+	cache derivedKeyCache
+}
+
+// NewSigningKeyDeriver returns a new SigningKeyDeriver
+func NewSigningKeyDeriver() *SigningKeyDeriver {
+	return &SigningKeyDeriver{
+		cache: newDerivedKeyCache(),
+	}
+}
+
+// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing.
+func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte {
+	return k.cache.Get(credential, service, region, signingTime)
+}
+
+func deriveKey(secret, service, region string, t SigningTime) []byte {
+	hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat()))
+	hmacRegion := HMACSHA256(hmacDate, []byte(region))
+	hmacService := HMACSHA256(hmacRegion, []byte(service))
+	return HMACSHA256(hmacService, []byte("aws4_request"))
+}
+
+func isSameDay(x, y time.Time) bool {
+	xYear, xMonth, xDay := x.Date()
+	yYear, yMonth, yDay := y.Date()
+
+	if xYear != yYear {
+		return false
+	}
+
+	if xMonth != yMonth {
+		return false
+	}
+
+	return xDay == yDay
+}

aws/signer/internal/v4/const.go

@@ -0,0 +1,40 @@
+package v4
+
+// Signature Version 4 (SigV4) Constants
+const (
+	// EmptyStringSHA256 is the hex encoded sha256 value of an empty string
+	EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
+
+	// UnsignedPayload indicates that the request payload body is unsigned
+	UnsignedPayload = "UNSIGNED-PAYLOAD"
+
+	// AmzAlgorithmKey indicates the signing algorithm
+	AmzAlgorithmKey = "X-Amz-Algorithm"
+
+	// AmzSecurityTokenKey indicates the security token to be used with temporary credentials
+	AmzSecurityTokenKey = "X-Amz-Security-Token"
+
+	// AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z'
+	AmzDateKey = "X-Amz-Date"
+
+	// AmzCredentialKey is the access key ID and credential scope
+	AmzCredentialKey = "X-Amz-Credential"
+
+	// AmzSignedHeadersKey is the set of headers signed for the request
+	AmzSignedHeadersKey = "X-Amz-SignedHeaders"
+
+	// AmzSignatureKey is the query parameter to store the SigV4 signature
+	AmzSignatureKey = "X-Amz-Signature"
+
+	// TimeFormat is the time format to be used in the X-Amz-Date header or query parameter
+	TimeFormat = "20060102T150405Z"
+
+	// ShortTimeFormat is the shorten time format used in the credential scope
+	ShortTimeFormat = "20060102"
+
+	// ContentSHAKey is the SHA256 of request body
+	ContentSHAKey = "X-Amz-Content-Sha256"
+
+	// StreamingEventsPayload indicates that the request payload body is a signed event stream.
+	StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS"
+)

aws/signer/internal/v4/header_rules.go

@@ -0,0 +1,88 @@
+package v4
+
+import (
+	"strings"
+)
+
+// Rules houses a set of Rule needed for validation of a
+// string value
+type Rules []Rule
+
+// Rule interface allows for more flexible rules and just simply
+// checks whether or not a value adheres to that Rule
+type Rule interface {
+	IsValid(value string) bool
+}
+
+// IsValid will iterate through all rules and see if any rules
+// apply to the value and supports nested rules
+func (r Rules) IsValid(value string) bool {
+	for _, rule := range r {
+		if rule.IsValid(value) {
+			return true
+		}
+	}
+	return false
+}
+
+// MapRule generic Rule for maps
+type MapRule map[string]struct{}
+
+// IsValid for the map Rule satisfies whether it exists in the map
+func (m MapRule) IsValid(value string) bool {
+	_, ok := m[value]
+	return ok
+}
+
+// AllowList is a generic Rule for include listing
+type AllowList struct {
+	Rule
+}
+
+// IsValid for AllowList checks if the value is within the AllowList
+func (w AllowList) IsValid(value string) bool {
+	return w.Rule.IsValid(value)
+}
+
+// ExcludeList is a generic Rule for exclude listing
+type ExcludeList struct {
+	Rule
+}
+
+// IsValid for AllowList checks if the value is within the AllowList
+func (b ExcludeList) IsValid(value string) bool {
+	return !b.Rule.IsValid(value)
+}
+
+// Patterns is a list of strings to match against
+type Patterns []string
+
+// IsValid for Patterns checks each pattern and returns if a match has
+// been found
+func (p Patterns) IsValid(value string) bool {
+	for _, pattern := range p {
+		if hasPrefixFold(value, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// InclusiveRules rules allow for rules to depend on one another
+type InclusiveRules []Rule
+
+// IsValid will return true if all rules are true
+func (r InclusiveRules) IsValid(value string) bool {
+	for _, rule := range r {
+		if !rule.IsValid(value) {
+			return false
+		}
+	}
+	return true
+}
+
+// hasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
+// under Unicode case-folding.
+func hasPrefixFold(s, prefix string) bool {
+	return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
+}

aws/signer/internal/v4/headers.go

@@ -0,0 +1,72 @@
+package v4
+
+// IgnoredHeaders is a list of headers that are ignored during signing
+var IgnoredHeaders = Rules{
+	ExcludeList{
+		MapRule{
+			"Authorization": struct{}{},
+			// some clients use user-agent in signed headers
+			// "User-Agent":      struct{}{},
+			"X-Amzn-Trace-Id": struct{}{},
+			"Expect":          struct{}{},
+		},
+	},
+}
+
+// RequiredSignedHeaders is a allow list for Build canonical headers.
+var RequiredSignedHeaders = Rules{
+	AllowList{
+		MapRule{
+			"Cache-Control":                         struct{}{},
+			"Content-Disposition":                   struct{}{},
+			"Content-Encoding":                      struct{}{},
+			"Content-Language":                      struct{}{},
+			"Content-Md5":                           struct{}{},
+			"Content-Type":                          struct{}{},
+			"Expires":                               struct{}{},
+			"If-Match":                              struct{}{},
+			"If-Modified-Since":                     struct{}{},
+			"If-None-Match":                         struct{}{},
+			"If-Unmodified-Since":                   struct{}{},
+			"Range":                                 struct{}{},
+			"X-Amz-Acl":                             struct{}{},
+			"X-Amz-Copy-Source":                     struct{}{},
+			"X-Amz-Copy-Source-If-Match":            struct{}{},
+			"X-Amz-Copy-Source-If-Modified-Since":   struct{}{},
+			"X-Amz-Copy-Source-If-None-Match":       struct{}{},
+			"X-Amz-Copy-Source-If-Unmodified-Since": struct{}{},
+			"X-Amz-Copy-Source-Range":               struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   struct{}{},
+			"X-Amz-Expected-Bucket-Owner":                                 struct{}{},
+			"X-Amz-Grant-Full-control":                                    struct{}{},
+			"X-Amz-Grant-Read":                                            struct{}{},
+			"X-Amz-Grant-Read-Acp":                                        struct{}{},
+			"X-Amz-Grant-Write":                                           struct{}{},
+			"X-Amz-Grant-Write-Acp":                                       struct{}{},
+			"X-Amz-Metadata-Directive":                                    struct{}{},
+			"X-Amz-Mfa":                                                   struct{}{},
+			"X-Amz-Request-Payer":                                         struct{}{},
+			"X-Amz-Server-Side-Encryption":                                struct{}{},
+			"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id":                 struct{}{},
+			"X-Amz-Server-Side-Encryption-Context":                        struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm":             struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Key":                   struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":               struct{}{},
+			"X-Amz-Storage-Class":                                         struct{}{},
+			"X-Amz-Website-Redirect-Location":                             struct{}{},
+			"X-Amz-Content-Sha256":                                        struct{}{},
+			"X-Amz-Tagging":                                               struct{}{},
+		},
+	},
+	Patterns{"X-Amz-Object-Lock-"},
+	Patterns{"X-Amz-Meta-"},
+}
+
+// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value
+// represents whether or not it is a pattern.
+var AllowedQueryHoisting = InclusiveRules{
+	ExcludeList{RequiredSignedHeaders},
+	Patterns{"X-Amz-"},
+}

aws/signer/internal/v4/headers_test.go

@@ -0,0 +1,63 @@
+package v4
+
+import "testing"
+
+func TestAllowedQueryHoisting(t *testing.T) {
+	cases := map[string]struct {
+		Header      string
+		ExpectHoist bool
+	}{
+		"object-lock": {
+			Header:      "X-Amz-Object-Lock-Mode",
+			ExpectHoist: false,
+		},
+		"s3 metadata": {
+			Header:      "X-Amz-Meta-SomeName",
+			ExpectHoist: false,
+		},
+		"another header": {
+			Header:      "X-Amz-SomeOtherHeader",
+			ExpectHoist: true,
+		},
+		"non X-AMZ header": {
+			Header:      "X-SomeOtherHeader",
+			ExpectHoist: false,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a {
+				t.Errorf("expect hoist %v, was %v", e, a)
+			}
+		})
+	}
+}
+
+func TestIgnoredHeaders(t *testing.T) {
+	cases := map[string]struct {
+		Header        string
+		ExpectIgnored bool
+	}{
+		"expect": {
+			Header:        "Expect",
+			ExpectIgnored: true,
+		},
+		"authorization": {
+			Header:        "Authorization",
+			ExpectIgnored: true,
+		},
+		"X-AMZ header": {
+			Header:        "X-Amz-Content-Sha256",
+			ExpectIgnored: false,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a {
+				t.Errorf("expect ignored %v, was %v", e, a)
+			}
+		})
+	}
+}

aws/signer/internal/v4/hmac.go

@@ -0,0 +1,13 @@
+package v4
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+)
+
+// HMACSHA256 computes a HMAC-SHA256 of data given the provided key.
+func HMACSHA256(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}

aws/signer/internal/v4/host.go

@@ -0,0 +1,75 @@
+package v4
+
+import (
+	"net/http"
+	"strings"
+)
+
+// SanitizeHostForHeader removes default port from host and updates request.Host
+func SanitizeHostForHeader(r *http.Request) {
+	host := getHost(r)
+	port := portOnly(host)
+	if port != "" && isDefaultPort(r.URL.Scheme, port) {
+		r.Host = stripPort(host)
+	}
+}
+
+// Returns host from request
+func getHost(r *http.Request) string {
+	if r.Host != "" {
+		return r.Host
+	}
+
+	return r.URL.Host
+}
+
+// Hostname returns u.Host, without any port number.
+//
+// If Host is an IPv6 literal with a port number, Hostname returns the
+// IPv6 literal without the square brackets. IPv6 literals may include
+// a zone identifier.
+//
+// Copied from the Go 1.8 standard library (net/url)
+func stripPort(hostport string) string {
+	colon := strings.IndexByte(hostport, ':')
+	if colon == -1 {
+		return hostport
+	}
+	if i := strings.IndexByte(hostport, ']'); i != -1 {
+		return strings.TrimPrefix(hostport[:i], "[")
+	}
+	return hostport[:colon]
+}
+
+// Port returns the port part of u.Host, without the leading colon.
+// If u.Host doesn't contain a port, Port returns an empty string.
+//
+// Copied from the Go 1.8 standard library (net/url)
+func portOnly(hostport string) string {
+	colon := strings.IndexByte(hostport, ':')
+	if colon == -1 {
+		return ""
+	}
+	if i := strings.Index(hostport, "]:"); i != -1 {
+		return hostport[i+len("]:"):]
+	}
+	if strings.Contains(hostport, "]") {
+		return ""
+	}
+	return hostport[colon+len(":"):]
+}
+
+// Returns true if the specified URI is using the standard port
+// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs)
+func isDefaultPort(scheme, port string) bool {
+	if port == "" {
+		return true
+	}
+
+	lowerCaseScheme := strings.ToLower(scheme)
+	if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") {
+		return true
+	}
+
+	return false
+}

aws/signer/internal/v4/scope.go

@@ -0,0 +1,13 @@
+package v4
+
+import "strings"
+
+// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope
+func BuildCredentialScope(signingTime SigningTime, region, service string) string {
+	return strings.Join([]string{
+		signingTime.ShortTimeFormat(),
+		region,
+		service,
+		"aws4_request",
+	}, "/")
+}

aws/signer/internal/v4/time.go

@@ -0,0 +1,36 @@
+package v4
+
+import "time"
+
+// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing.
+type SigningTime struct {
+	time.Time
+	timeFormat      string
+	shortTimeFormat string
+}
+
+// NewSigningTime creates a new SigningTime given a time.Time
+func NewSigningTime(t time.Time) SigningTime {
+	return SigningTime{
+		Time: t,
+	}
+}
+
+// TimeFormat provides a time formatted in the X-Amz-Date format.
+func (m *SigningTime) TimeFormat() string {
+	return m.format(&m.timeFormat, TimeFormat)
+}
+
+// ShortTimeFormat provides a time formatted of 20060102.
+func (m *SigningTime) ShortTimeFormat() string {
+	return m.format(&m.shortTimeFormat, ShortTimeFormat)
+}
+
+func (m *SigningTime) format(target *string, format string) string {
+	if len(*target) > 0 {
+		return *target
+	}
+	v := m.Time.Format(format)
+	*target = v
+	return v
+}

aws/signer/internal/v4/util.go

@@ -0,0 +1,80 @@
+package v4
+
+import (
+	"net/url"
+	"strings"
+)
+
+const doubleSpace = "  "
+
+// StripExcessSpaces will rewrite the passed in slice's string values to not
+// contain multiple side-by-side spaces.
+func StripExcessSpaces(str string) string {
+	var j, k, l, m, spaces int
+	// Trim trailing spaces
+	for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- {
+	}
+
+	// Trim leading spaces
+	for k = 0; k < j && str[k] == ' '; k++ {
+	}
+	str = str[k : j+1]
+
+	// Strip multiple spaces.
+	j = strings.Index(str, doubleSpace)
+	if j < 0 {
+		return str
+	}
+
+	buf := []byte(str)
+	for k, m, l = j, j, len(buf); k < l; k++ {
+		if buf[k] == ' ' {
+			if spaces == 0 {
+				// First space.
+				buf[m] = buf[k]
+				m++
+			}
+			spaces++
+		} else {
+			// End of multiple spaces.
+			spaces = 0
+			buf[m] = buf[k]
+			m++
+		}
+	}
+
+	return string(buf[:m])
+}
+
+// GetURIPath returns the escaped URI component from the provided URL.
+func GetURIPath(u *url.URL) string {
+	var uriPath string
+
+	if len(u.Opaque) > 0 {
+		const schemeSep, pathSep, queryStart = "//", "/", "?"
+
+		opaque := u.Opaque
+		// Cut off the query string if present.
+		if idx := strings.Index(opaque, queryStart); idx >= 0 {
+			opaque = opaque[:idx]
+		}
+
+		// Cutout the scheme separator if present.
+		if strings.Index(opaque, schemeSep) == 0 {
+			opaque = opaque[len(schemeSep):]
+		}
+
+		// capture URI path starting with first path separator.
+		if idx := strings.Index(opaque, pathSep); idx >= 0 {
+			uriPath = opaque[idx:]
+		}
+	} else {
+		uriPath = u.EscapedPath()
+	}
+
+	if len(uriPath) == 0 {
+		uriPath = "/"
+	}
+
+	return uriPath
+}

aws/signer/internal/v4/util_test.go

@@ -0,0 +1,158 @@
+package v4
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+func lazyURLParse(v string) func() (*url.URL, error) {
+	return func() (*url.URL, error) {
+		return url.Parse(v)
+	}
+}
+
+func TestGetURIPath(t *testing.T) {
+	cases := map[string]struct {
+		getURL func() (*url.URL, error)
+		expect string
+	}{
+		// Cases
+		"with scheme": {
+			getURL: lazyURLParse("https://localhost:9000"),
+			expect: "/",
+		},
+		"no port, with scheme": {
+			getURL: lazyURLParse("https://localhost"),
+			expect: "/",
+		},
+		"without scheme": {
+			getURL: lazyURLParse("localhost:9000"),
+			expect: "/",
+		},
+		"without scheme, with path": {
+			getURL: lazyURLParse("localhost:9000/abc123"),
+			expect: "/abc123",
+		},
+		"without scheme, with separator": {
+			getURL: lazyURLParse("//localhost:9000"),
+			expect: "/",
+		},
+		"no port, without scheme, with separator": {
+			getURL: lazyURLParse("//localhost"),
+			expect: "/",
+		},
+		"without scheme, with separator, with path": {
+			getURL: lazyURLParse("//localhost:9000/abc123"),
+			expect: "/abc123",
+		},
+		"no port, without scheme, with separator, with path": {
+			getURL: lazyURLParse("//localhost/abc123"),
+			expect: "/abc123",
+		},
+		"opaque with query string": {
+			getURL: lazyURLParse("localhost:9000/abc123?efg=456"),
+			expect: "/abc123",
+		},
+		"failing test": {
+			getURL: func() (*url.URL, error) {
+				endpoint := "https://service.region.amazonaws.com"
+				req, _ := http.NewRequest("POST", endpoint, nil)
+				u := req.URL
+
+				u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
+
+				query := u.Query()
+				query.Set("some-query-key", "value")
+				u.RawQuery = query.Encode()
+
+				return u, nil
+			},
+			expect: "/bucket/key-._~,!@#$%^&*()",
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			u, err := c.getURL()
+			if err != nil {
+				t.Fatalf("failed to get URL, %v", err)
+			}
+
+			actual := GetURIPath(u)
+			if e, a := c.expect, actual; e != a {
+				t.Errorf("expect %v path, got %v", e, a)
+			}
+		})
+	}
+}
+
+func TestStripExcessHeaders(t *testing.T) {
+	vals := []string{
+		"",
+		"123",
+		"1 2 3",
+		"1 2 3 ",
+		"  1 2 3",
+		"1  2 3",
+		"1  23",
+		"1  2  3",
+		"1  2  ",
+		" 1  2  ",
+		"12   3",
+		"12   3   1",
+		"12           3     1",
+		"12     3       1abc123",
+	}
+
+	expected := []string{
+		"",
+		"123",
+		"1 2 3",
+		"1 2 3",
+		"1 2 3",
+		"1 2 3",
+		"1 23",
+		"1 2 3",
+		"1 2",
+		"1 2",
+		"12 3",
+		"12 3 1",
+		"12 3 1",
+		"12 3 1abc123",
+	}
+
+	for i := 0; i < len(vals); i++ {
+		r := StripExcessSpaces(vals[i])
+		if e, a := expected[i], r; e != a {
+			t.Errorf("%d, expect %v, got %v", i, e, a)
+		}
+	}
+}
+
+var stripExcessSpaceCases = []string{
+	`AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`,
+	`123   321   123   321`,
+	`   123   321   123   321   `,
+	`   123    321    123          321   `,
+	"123",
+	"1 2 3",
+	"  1 2 3",
+	"1  2 3",
+	"1  23",
+	"1  2  3",
+	"1  2  ",
+	" 1  2  ",
+	"12   3",
+	"12   3   1",
+	"12           3     1",
+	"12     3       1abc123",
+}
+
+func BenchmarkStripExcessSpaces(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		for _, v := range stripExcessSpaceCases {
+			StripExcessSpaces(v)
+		}
+	}
+}

aws/signer/v4/functional_test.go

@@ -0,0 +1,139 @@
+package v4_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/versity/versitygw/aws/internal/awstesting/unit"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+var standaloneSignCases = []struct {
+	OrigURI                    string
+	OrigQuery                  string
+	Region, Service, SubDomain string
+	ExpSig                     string
+	EscapedURI                 string
+}{
+	{
+		OrigURI:   `/logs-*/_search`,
+		OrigQuery: `pretty=true`,
+		Region:    "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
+		EscapedURI: `/logs-%2A/_search`,
+		ExpSig:     `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
+	},
+}
+
+func TestStandaloneSign_CustomURIEscape(t *testing.T) {
+	var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967`
+
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner(func(signer *v4.SignerOptions) {
+		signer.DisableURIPathEscaping = true
+	})
+
+	host := "https://subdomain.us-east-1.es.amazonaws.com"
+	req, err := http.NewRequest("GET", host, nil)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	req.URL.Path = `/log-*/_search`
+	req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search"
+
+	err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0))
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	actual := req.Header.Get("Authorization")
+	if e, a := expectSig, actual; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestStandaloneSign(t *testing.T) {
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner()
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		// URL.EscapedPath() will be used by the signer to get the
+		// escaped form of the request's URI path.
+		req.URL.Path = c.OrigURI
+		req.URL.RawQuery = c.OrigQuery
+
+		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		actual := req.Header.Get("Authorization")
+		if e, a := c.ExpSig, actual; e != a {
+			t.Errorf("expected %v, but received %v", e, a)
+		}
+		if e, a := c.OrigURI, req.URL.Path; e != a {
+			t.Errorf("expected %v, but received %v", e, a)
+		}
+		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
+			t.Errorf("expected %v, but received %v", e, a)
+		}
+	}
+}
+
+func TestStandaloneSign_RawPath(t *testing.T) {
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner()
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		// URL.EscapedPath() will be used by the signer to get the
+		// escaped form of the request's URI path.
+		req.URL.Path = c.OrigURI
+		req.URL.RawPath = c.EscapedURI
+		req.URL.RawQuery = c.OrigQuery
+
+		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		actual := req.Header.Get("Authorization")
+		if e, a := c.ExpSig, actual; e != a {
+			t.Errorf("expected %v, but received %v", e, a)
+		}
+		if e, a := c.OrigURI, req.URL.Path; e != a {
+			t.Errorf("expected %v, but received %v", e, a)
+		}
+		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
+			t.Errorf("expected %v, but received %v", e, a)
+		}
+	}
+}

aws/signer/v4/v4.go

@@ -0,0 +1,565 @@
+// Package v4 implements signing for AWS V4 signer
+//
+// Provides request signing for request that need to be signed with
+// AWS V4 Signatures.
+//
+// # Standalone Signer
+//
+// Generally using the signer outside of the SDK should not require any additional
+//
+//	The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires
+//
+// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent
+// to the service as.
+//
+// The signer will first check the URL.Opaque field, and use its value if set.
+// The signer does require the URL.Opaque field to be set in the form of:
+//
+//	"//<hostname>/<path>"
+//
+//	// e.g.
+//	"//example.com/some/path"
+//
+// The leading "//" and hostname are required or the URL.Opaque escaping will
+// not work correctly.
+//
+// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath()
+// method and using the returned value.
+//
+// AWS v4 signature validation requires that the canonical string's URI path
+// element must be the URI escaped form of the HTTP request's path.
+// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+//
+// The Go HTTP client will perform escaping automatically on the request. Some
+// of these escaping may cause signature validation errors because the HTTP
+// request differs from the URI path or query that the signature was generated.
+// https://golang.org/pkg/net/url/#URL.EscapedPath
+//
+// Because of this, it is recommended that when using the signer outside of the
+// SDK that explicitly escaping the request prior to being signed is preferable,
+// and will help prevent signature validation errors. This can be done by setting
+// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then
+// call URL.EscapedPath() if Opaque is not set.
+//
+// Test `TestStandaloneSign` provides a complete example of using the signer
+// outside of the SDK and pre-escaping the URI path.
+package v4
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"net/http"
+	"net/textproto"
+	"net/url"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go/encoding/httpbinding"
+	"github.com/aws/smithy-go/logging"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+const (
+	signingAlgorithm    = "AWS4-HMAC-SHA256"
+	authorizationHeader = "Authorization"
+
+	// Version of signing v4
+	Version = "SigV4"
+)
+
+// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests
+type HTTPSigner interface {
+	SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
+}
+
+type keyDerivator interface {
+	DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte
+}
+
+// SignerOptions is the SigV4 Signer options.
+type SignerOptions struct {
+	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
+	// request header to the request's query string. This is most commonly used
+	// with pre-signed requests preventing headers from being added to the
+	// request's query string.
+	DisableHeaderHoisting bool
+
+	// Disables the automatic escaping of the URI path of the request for the
+	// siganture's canonical string's path. For services that do not need additional
+	// escaping then use this to disable the signer escaping the path.
+	//
+	// S3 is an example of a service that does not need additional escaping.
+	//
+	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+	DisableURIPathEscaping bool
+
+	// The logger to send log messages to.
+	Logger logging.Logger
+
+	// Enable logging of signed requests.
+	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
+	// presigned URL.
+	LogSigning bool
+
+	// Disables setting the session token on the request as part of signing
+	// through X-Amz-Security-Token. This is needed for variations of v4 that
+	// present the token elsewhere.
+	DisableSessionToken bool
+}
+
+// Signer applies AWS v4 signing to given request. Use this to sign requests
+// that need to be signed with AWS V4 Signatures.
+type Signer struct {
+	options      SignerOptions
+	keyDerivator keyDerivator
+}
+
+// NewSigner returns a new SigV4 Signer
+func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
+	options := SignerOptions{}
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()}
+}
+
+type httpSigner struct {
+	Request      *http.Request
+	ServiceName  string
+	Region       string
+	Time         v4Internal.SigningTime
+	Credentials  aws.Credentials
+	KeyDerivator keyDerivator
+	IsPreSign    bool
+	SignedHdrs   []string
+
+	PayloadHash string
+
+	DisableHeaderHoisting  bool
+	DisableURIPathEscaping bool
+	DisableSessionToken    bool
+}
+
+func (s *httpSigner) Build() (signedRequest, error) {
+	req := s.Request
+
+	query := req.URL.Query()
+	headers := req.Header
+
+	s.setRequiredSigningFields(headers, query)
+
+	// Sort Each Query Key's Values
+	for key := range query {
+		sort.Strings(query[key])
+	}
+
+	v4Internal.SanitizeHostForHeader(req)
+
+	credentialScope := s.buildCredentialScope()
+	credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzCredentialKey, credentialStr)
+	}
+
+	unsignedHeaders := headers
+	if s.IsPreSign && !s.DisableHeaderHoisting {
+		var urlValues url.Values
+		urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers)
+		for k := range urlValues {
+			query[k] = urlValues[k]
+		}
+	}
+
+	host := req.URL.Host
+	if len(req.Host) > 0 {
+		host = req.Host
+	}
+
+	signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength)
+
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr)
+	}
+
+	var rawQuery strings.Builder
+	rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1))
+
+	canonicalURI := v4Internal.GetURIPath(req.URL)
+	if !s.DisableURIPathEscaping {
+		canonicalURI = httpbinding.EscapePath(canonicalURI, false)
+	}
+
+	canonicalString := s.buildCanonicalString(
+		req.Method,
+		canonicalURI,
+		rawQuery.String(),
+		signedHeadersStr,
+		canonicalHeaderStr,
+	)
+
+	strToSign := s.buildStringToSign(credentialScope, canonicalString)
+	signingSignature, err := s.buildSignature(strToSign)
+	if err != nil {
+		return signedRequest{}, err
+	}
+
+	if s.IsPreSign {
+		rawQuery.WriteString("&X-Amz-Signature=")
+		rawQuery.WriteString(signingSignature)
+	} else {
+		headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature))
+	}
+
+	req.URL.RawQuery = rawQuery.String()
+
+	return signedRequest{
+		Request:         req,
+		SignedHeaders:   signedHeaders,
+		CanonicalString: canonicalString,
+		StringToSign:    strToSign,
+		PreSigned:       s.IsPreSign,
+	}, nil
+}
+
+func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string {
+	const credential = "Credential="
+	const signedHeaders = "SignedHeaders="
+	const signature = "Signature="
+	const commaSpace = ", "
+
+	var parts strings.Builder
+	parts.Grow(len(signingAlgorithm) + 1 +
+		len(credential) + len(credentialStr) + 2 +
+		len(signedHeaders) + len(signedHeadersStr) + 2 +
+		len(signature) + len(signingSignature),
+	)
+	parts.WriteString(signingAlgorithm)
+	parts.WriteRune(' ')
+	parts.WriteString(credential)
+	parts.WriteString(credentialStr)
+	parts.WriteString(commaSpace)
+	parts.WriteString(signedHeaders)
+	parts.WriteString(signedHeadersStr)
+	parts.WriteString(commaSpace)
+	parts.WriteString(signature)
+	parts.WriteString(signingSignature)
+	return parts.String()
+}
+
+// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the
+// request is made to, and time the request is signed at. The signTime allows
+// you to specify that a request is signed for the future, and cannot be
+// used until then.
+//
+// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
+// must be provided. Even if the request has no payload (aka body). If the
+// request has no payload you should use the hex encoded SHA-256 of an empty
+// string as the payloadHash value.
+//
+//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+//
+// Some services such as Amazon S3 accept alternative values for the payload
+// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
+// included in the request signature.
+//
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+//
+// Sign differs from Presign in that it will sign the request using HTTP
+// header values. This type of signing is intended for http.Request values that
+// will not be shared, or are shared in a way the header values on the request
+// will not be lost.
+//
+// The passed in request will be modified in place.
+func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) error {
+	options := s.options
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	signer := &httpSigner{
+		Request:                r,
+		PayloadHash:            payloadHash,
+		ServiceName:            service,
+		Region:                 region,
+		Credentials:            credentials,
+		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
+		DisableHeaderHoisting:  options.DisableHeaderHoisting,
+		DisableURIPathEscaping: options.DisableURIPathEscaping,
+		DisableSessionToken:    options.DisableSessionToken,
+		KeyDerivator:           s.keyDerivator,
+		SignedHdrs:             signedHdrs,
+	}
+
+	signedRequest, err := signer.Build()
+	if err != nil {
+		return err
+	}
+
+	logSigningInfo(ctx, options, &signedRequest, false)
+
+	return nil
+}
+
+// PresignHTTP signs AWS v4 requests with the payload hash, service name, region
+// the request is made to, and time the request is signed at. The signTime
+// allows you to specify that a request is signed for the future, and cannot
+// be used until then.
+//
+// Returns the signed URL and the map of HTTP headers that were included in the
+// signature or an error if signing the request failed. For presigned requests
+// these headers and their values must be included on the HTTP request when it
+// is made. This is helpful to know what header values need to be shared with
+// the party the presigned request will be distributed to.
+//
+// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
+// must be provided. Even if the request has no payload (aka body). If the
+// request has no payload you should use the hex encoded SHA-256 of an empty
+// string as the payloadHash value.
+//
+//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+//
+// Some services such as Amazon S3 accept alternative values for the payload
+// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
+// included in the request signature.
+//
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+//
+// PresignHTTP differs from SignHTTP in that it will sign the request using
+// query string instead of header values. This allows you to share the
+// Presigned Request's URL with third parties, or distribute it throughout your
+// system with minimal dependencies.
+//
+// PresignHTTP will not set the expires time of the presigned request
+// automatically. To specify the expire duration for a request add the
+// "X-Amz-Expires" query parameter on the request with the value as the
+// duration in seconds the presigned URL should be considered valid for. This
+// parameter is not used by all AWS services, and is most notable used by
+// Amazon S3 APIs.
+//
+//	expires := 20 * time.Minute
+//	query := req.URL.Query()
+//	query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10))
+//	req.URL.RawQuery = query.Encode()
+//
+// This method does not modify the provided request.
+func (s *Signer) PresignHTTP(
+	ctx context.Context, credentials aws.Credentials, r *http.Request,
+	payloadHash string, service string, region string, signingTime time.Time,
+	signedHdrs []string,
+	optFns ...func(*SignerOptions),
+) (signedURI string, signedHeaders http.Header, err error) {
+	options := s.options
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	signer := &httpSigner{
+		Request:                r.Clone(r.Context()),
+		PayloadHash:            payloadHash,
+		ServiceName:            service,
+		Region:                 region,
+		Credentials:            credentials,
+		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
+		IsPreSign:              true,
+		DisableHeaderHoisting:  options.DisableHeaderHoisting,
+		DisableURIPathEscaping: options.DisableURIPathEscaping,
+		DisableSessionToken:    options.DisableSessionToken,
+		KeyDerivator:           s.keyDerivator,
+		SignedHdrs:             signedHdrs,
+	}
+
+	signedRequest, err := signer.Build()
+	if err != nil {
+		return "", nil, err
+	}
+
+	logSigningInfo(ctx, options, &signedRequest, true)
+
+	signedHeaders = make(http.Header)
+
+	// For the signed headers we canonicalize the header keys in the returned map.
+	// This avoids situations where can standard library double headers like host header. For example the standard
+	// library will set the Host header, even if it is present in lower-case form.
+	for k, v := range signedRequest.SignedHeaders {
+		key := textproto.CanonicalMIMEHeaderKey(k)
+		signedHeaders[key] = append(signedHeaders[key], v...)
+	}
+
+	return signedRequest.Request.URL.String(), signedHeaders, nil
+}
+
+func (s *httpSigner) buildCredentialScope() string {
+	return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName)
+}
+
+func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) {
+	query := url.Values{}
+	unsignedHeaders := http.Header{}
+	for k, h := range header {
+		if r.IsValid(k) {
+			query[k] = h
+		} else {
+			unsignedHeaders[k] = h
+		}
+	}
+
+	return query, unsignedHeaders
+}
+
+func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) {
+	signed = make(http.Header)
+
+	var headers []string
+	const hostHeader = "host"
+	headers = append(headers, hostHeader)
+	signed[hostHeader] = append(signed[hostHeader], host)
+
+	const contentLengthHeader = "content-length"
+	if slices.Contains(s.SignedHdrs, contentLengthHeader) {
+		headers = append(headers, contentLengthHeader)
+		signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10))
+	}
+
+	for k, v := range header {
+		if !rule.IsValid(k) {
+			continue // ignored header
+		}
+		if strings.EqualFold(k, contentLengthHeader) {
+			// prevent signing already handled content-length header.
+			continue
+		}
+
+		lowerCaseKey := strings.ToLower(k)
+		if _, ok := signed[lowerCaseKey]; ok {
+			// include additional values
+			signed[lowerCaseKey] = append(signed[lowerCaseKey], v...)
+			continue
+		}
+
+		headers = append(headers, lowerCaseKey)
+		signed[lowerCaseKey] = v
+	}
+	sort.Strings(headers)
+
+	signedHeaders = strings.Join(headers, ";")
+
+	var canonicalHeaders strings.Builder
+	n := len(headers)
+	const colon = ':'
+	for i := 0; i < n; i++ {
+		if headers[i] == hostHeader {
+			canonicalHeaders.WriteString(hostHeader)
+			canonicalHeaders.WriteRune(colon)
+			canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host))
+		} else {
+			canonicalHeaders.WriteString(headers[i])
+			canonicalHeaders.WriteRune(colon)
+			// Trim out leading, trailing, and dedup inner spaces from signed header values.
+			values := signed[headers[i]]
+			for j, v := range values {
+				cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v))
+				canonicalHeaders.WriteString(cleanedValue)
+				if j < len(values)-1 {
+					canonicalHeaders.WriteRune(',')
+				}
+			}
+		}
+		canonicalHeaders.WriteRune('\n')
+	}
+	canonicalHeadersStr = canonicalHeaders.String()
+
+	return signed, signedHeaders, canonicalHeadersStr
+}
+
+func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string {
+	return strings.Join([]string{
+		method,
+		uri,
+		query,
+		canonicalHeaders,
+		signedHeaders,
+		s.PayloadHash,
+	}, "\n")
+}
+
+func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string {
+	return strings.Join([]string{
+		signingAlgorithm,
+		s.Time.TimeFormat(),
+		credentialScope,
+		hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))),
+	}, "\n")
+}
+
+func makeHash(hash hash.Hash, b []byte) []byte {
+	hash.Reset()
+	hash.Write(b)
+	return hash.Sum(nil)
+}
+
+func (s *httpSigner) buildSignature(strToSign string) (string, error) {
+	key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time)
+	return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil
+}
+
+func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) {
+	amzDate := s.Time.TimeFormat()
+
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm)
+		sessionToken := s.Credentials.SessionToken
+		if !s.DisableSessionToken && len(sessionToken) > 0 {
+			query.Set("X-Amz-Security-Token", sessionToken)
+		}
+
+		query.Set(v4Internal.AmzDateKey, amzDate)
+		return
+	}
+
+	headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate)
+
+	if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 {
+		headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken)
+	}
+}
+
+func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) {
+	if !options.LogSigning {
+		return
+	}
+	signedURLMsg := ""
+	if isPresign {
+		signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String())
+	}
+	logger := logging.WithContext(ctx, options.Logger)
+	logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg)
+}
+
+type signedRequest struct {
+	Request         *http.Request
+	SignedHeaders   http.Header
+	CanonicalString string
+	StringToSign    string
+	PreSigned       bool
+}
+
+const logSignInfoMsg = `Request Signature:
+---[ CANONICAL STRING  ]-----------------------------
+%s
+---[ STRING TO SIGN ]--------------------------------
+%s%s
+-----------------------------------------------------`
+const logSignedURLMsg = `
+---[ SIGNED URL ]------------------------------------
+%s`

aws/signer/v4/v4_test.go

@@ -0,0 +1,358 @@
+package v4
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/google/go-cmp/cmp"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"}
+
+func buildRequest(serviceName, region, body string) (*http.Request, string) {
+	reader := strings.NewReader(body)
+	return buildRequestWithBodyReader(serviceName, region, reader)
+}
+
+func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) {
+	var bodyLen int
+
+	type lenner interface {
+		Len() int
+	}
+	if lr, ok := body.(lenner); ok {
+		bodyLen = lr.Len()
+	}
+
+	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
+	req, _ := http.NewRequest("POST", endpoint, body)
+	req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
+	req.Header.Set("X-Amz-Target", "prefix.Operation")
+	req.Header.Set("Content-Type", "application/x-amz-json-1.0")
+
+	if bodyLen > 0 {
+		req.ContentLength = int64(bodyLen)
+	}
+
+	req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)")
+	req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
+	req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
+
+	h := sha256.New()
+	_, _ = io.Copy(h, body)
+	payloadHash := hex.EncodeToString(h.Sum(nil))
+
+	return req, payloadHash
+}
+
+func TestPresignRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "300")
+	req.URL.RawQuery = query.Encode()
+
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	signer := NewSigner()
+	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
+	expectedSig := "122f0b9e091e4ba84286097e2b3404a1f1f4c4aad479adda95b7dff0ccbe5581"
+	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
+	expectedTarget := "prefix.Operation"
+
+	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
+	if err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+
+	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
+		t.Errorf("expect %v to be empty", a)
+	}
+	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+
+	for _, h := range strings.Split(expectedHeaders, ";") {
+		v := headers.Get(h)
+		if len(v) == 0 {
+			t.Errorf("expect %v, to be present in header map", h)
+		}
+	}
+}
+
+func TestPresignBodyWithArrayRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "300")
+	req.URL.RawQuery = query.Encode()
+
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	signer := NewSigner()
+	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
+	if err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
+	expectedSig := "e3ac55addee8711b76c6d608d762cff285fe8b627a057f8b5ec9268cf82c08b1"
+	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
+	expectedTarget := "prefix.Operation"
+
+	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
+		t.Errorf("expect %v to be empty, was not", a)
+	}
+	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+
+	for _, h := range strings.Split(expectedHeaders, ";") {
+		v := headers.Get(h)
+		if len(v) == 0 {
+			t.Errorf("expect %v, to be present in header map", h)
+		}
+	}
+}
+
+func TestSignRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+	signer := NewSigner()
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9"
+
+	q := req.Header
+	if e, a := expectedSig, q.Get("Authorization"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestBuildCanonicalRequest(t *testing.T) {
+	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z"
+	if e, a := expected, build.Request.URL.String(); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+	req.Body = io.NopCloser(bytes.NewReader([]byte{}))
+
+	s := NewSigner()
+
+	origBody := req.Body
+
+	err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	if req.Body != origBody {
+		t.Errorf("expect request body to not be chagned")
+	}
+}
+
+func TestRequestHost(t *testing.T) {
+	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+	req.Host = "myhost"
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if !strings.Contains(build.CanonicalString, "host:"+req.Host) {
+		t.Errorf("canonical host header invalid")
+	}
+}
+
+func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) {
+	body := `{"description": "this is a test"}`
+	req, _ := buildRequest("dynamodb", "us-east-1", body)
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+	req.Host = "myhost"
+
+	contentLength := fmt.Sprintf("%d", len([]byte(body)))
+	req.Header.Add("Content-Length", contentLength)
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	_, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	//if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") {
+	//	t.Errorf("canonical header content-length invalid")
+	//}
+}
+
+func TestSign_buildCanonicalHeaders(t *testing.T) {
+	serviceName := "mockAPI"
+	region := "mock-region"
+	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
+
+	req, err := http.NewRequest("POST", endpoint, nil)
+	if err != nil {
+		t.Fatalf("failed to create request, %v", err)
+	}
+
+	req.Header.Set("FooInnerSpace", "   inner      space    ")
+	req.Header.Set("FooLeadingSpace", "    leading-space")
+	req.Header.Add("FooMultipleSpace", "no-space")
+	req.Header.Add("FooMultipleSpace", "\ttab-space")
+	req.Header.Add("FooMultipleSpace", "trailing-space    ")
+	req.Header.Set("FooNoSpace", "no-space")
+	req.Header.Set("FooTabSpace", "\ttab-space\t")
+	req.Header.Set("FooTrailingSpace", "trailing-space    ")
+	req.Header.Set("FooWrappedSpace", "   wrapped-space    ")
+
+	ctx := &httpSigner{
+		ServiceName:  serviceName,
+		Region:       region,
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectCanonicalString := strings.Join([]string{
+		`POST`,
+		`/`,
+		``,
+		`fooinnerspace:inner space`,
+		`fooleadingspace:leading-space`,
+		`foomultiplespace:no-space,tab-space,trailing-space`,
+		`foonospace:no-space`,
+		`footabspace:tab-space`,
+		`footrailingspace:trailing-space`,
+		`foowrappedspace:wrapped-space`,
+		`host:mockAPI.mock-region.amazonaws.com`,
+		`x-amz-date:20211020T124200Z`,
+		``,
+		`fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`,
+		``,
+	}, "\n")
+	if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" {
+		t.Errorf("expect match, got\n%s", diff)
+	}
+}
+
+func BenchmarkPresignRequest(b *testing.B) {
+	signer := NewSigner()
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	for i := 0; i < b.N; i++ {
+		signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	}
+}
+
+func BenchmarkSignRequest(b *testing.B) {
+	signer := NewSigner()
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+	for i := 0; i < b.N; i++ {
+		signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	}
+}

backend/azure/err.go

@@ -0,0 +1,63 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package azure
+
+import (
+	"errors"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/versity/versitygw/s3err"
+)
+
+// Parses azure ResponseError into AWS APIError
+func azureErrToS3Err(apiErr error) error {
+	var azErr *azcore.ResponseError
+	if !errors.As(apiErr, &azErr) {
+		return apiErr
+	}
+
+	return azErrToS3err(azErr)
+}
+
+func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
+	switch azErr.ErrorCode {
+	case "ContainerAlreadyExists":
+		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
+	case "InvalidResourceName", "ContainerNotFound":
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	case "BlobNotFound":
+		return s3err.GetAPIError(s3err.ErrNoSuchKey)
+	case "TagsTooLarge":
+		return s3err.GetAPIError(s3err.ErrInvalidTagValue)
+	case "Requested Range Not Satisfiable":
+		return s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+	return s3err.APIError{
+		Code:           azErr.ErrorCode,
+		Description:    azErr.RawResponse.Status,
+		HTTPStatusCode: azErr.StatusCode,
+	}
+}
+
+func parseMpError(mpErr error) error {
+	err := azureErrToS3Err(mpErr)
+
+	serr, ok := err.(s3err.APIError)
+	if !ok || serr.Code != "NoSuchKey" {
+		return mpErr
+	}
+
+	return s3err.GetAPIError(s3err.ErrNoSuchUpload)
+}

backend/backend.go

@@ -0,0 +1,288 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+	"github.com/versity/versitygw/s3select"
+)
+
+//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
+type Backend interface {
+	fmt.Stringer
+	Shutdown()
+
+	// bucket operations
+	ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
+	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
+	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
+	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
+	PutBucketAcl(_ context.Context, bucket string, data []byte) error
+	DeleteBucket(_ context.Context, bucket string) error
+	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
+	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
+	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
+	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
+	DeleteBucketPolicy(_ context.Context, bucket string) error
+	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
+	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
+	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
+	PutBucketCors(_ context.Context, bucket string, cors []byte) error
+	GetBucketCors(_ context.Context, bucket string) ([]byte, error)
+	DeleteBucketCors(_ context.Context, bucket string) error
+
+	// multipart operations
+	CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
+	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (_ s3response.CompleteMultipartUploadResult, versionid string, _ error)
+	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
+	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
+	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
+
+	// standard object operations
+	PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
+	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
+	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
+	CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
+	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
+	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
+
+	// special case object operations
+	RestoreObject(context.Context, *s3.RestoreObjectInput) error
+	SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
+
+	// bucket tagging operations
+	GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
+	PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
+	DeleteBucketTagging(_ context.Context, bucket string) error
+
+	// object tagging operations
+	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
+	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
+	DeleteObjectTagging(_ context.Context, bucket, object string) error
+
+	// object lock operations
+	PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
+	GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
+	PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error
+	GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
+	PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
+	GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
+
+	// non AWS actions
+	ChangeBucketOwner(_ context.Context, bucket, owner string) error
+	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
+}
+
+type BackendUnsupported struct{}
+
+var _ Backend = &BackendUnsupported{}
+
+func New() Backend {
+	return &BackendUnsupported{}
+}
+func (BackendUnsupported) Shutdown() {}
+func (BackendUnsupported) String() string {
+	return "Unsupported"
+}
+func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
+	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
+	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketCors(context.Context, string, []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
+	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+	return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
+	return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
+	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
+	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
+	return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+	return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
+	return func(w *bufio.Writer) {
+		var getProgress s3select.GetProgress
+		progress := input.RequestProgress
+		if progress != nil && *progress.Enabled {
+			getProgress = func() (bytesScanned int64, bytesProcessed int64) {
+				return -1, -1
+			}
+		}
+		mh := s3select.NewMessageHandler(ctx, w, getProgress)
+		apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
+		mh.FinishWithError(apiErr.Code, apiErr.Description)
+	}
+}
+
+func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
+	return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, owner string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
+	return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}

backend/common.go

@@ -0,0 +1,572 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"io/fs"
+	"math"
+	"net/url"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+const (
+	// this is the media type for directories in AWS and Nextcloud
+	DirContentType     = "application/x-directory"
+	DefaultContentType = "binary/octet-stream"
+
+	// this is the minimum allowed size for mp parts
+	MinPartSize = 5 * 1024 * 1024
+)
+
+func IsValidBucketName(name string) bool { return true }
+
+type ByBucketName []s3response.ListAllMyBucketsEntry
+
+func (d ByBucketName) Len() int           { return len(d) }
+func (d ByBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d ByBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
+
+type ByObjectName []types.Object
+
+func (d ByObjectName) Len() int           { return len(d) }
+func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
+
+func GetPtrFromString(str string) *string {
+	if str == "" {
+		return nil
+	}
+	return &str
+}
+
+func GetStringFromPtr(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
+}
+
+func GetTimePtr(t time.Time) *time.Time {
+	return &t
+}
+
+func TrimEtag(etag *string) *string {
+	if etag == nil {
+		return nil
+	}
+
+	return GetPtrFromString(strings.Trim(*etag, "\""))
+}
+
+var (
+	errInvalidRange           = s3err.GetAPIError(s3err.ErrInvalidRange)
+	errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
+	errPreconditionFailed     = s3err.GetAPIError(s3err.ErrPreconditionFailed)
+	errNotModified            = s3err.GetAPIError(s3err.ErrNotModified)
+)
+
+// ParseObjectRange parses input range header and returns startoffset, length, isValid
+// and error. If no endoffset specified, then length is set to the object size
+// for invalid inputs, it returns no error, but isValid=false
+// `InvalidRange` error is returnd, only if startoffset is greater than the object size
+func ParseObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
+	// Return full object (invalid range, no error) if header empty
+	if acceptRange == "" {
+		return 0, size, false, nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+	if len(rangeKv) != 2 {
+		return 0, size, false, nil
+	}
+	if rangeKv[0] != "bytes" { // unsupported unit -> ignore
+		return 0, size, false, nil
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) != 2 { // malformed / multi-range
+		return 0, size, false, nil
+	}
+
+	// Parse start; empty start indicates a suffix-byte-range-spec (e.g. bytes=-100)
+	startOffset, err := strconv.ParseInt(bRange[0], 10, strconv.IntSize)
+	if startOffset > int64(math.MaxInt) || startOffset < int64(math.MinInt) {
+		return 0, size, false, errInvalidRange
+	}
+	if err != nil && bRange[0] != "" { // invalid numeric start (non-empty) -> ignore range
+		return 0, size, false, nil
+	}
+
+	// If end part missing (e.g. bytes=100-)
+	if bRange[1] == "" {
+		if bRange[0] == "" { // bytes=- (meaningless) -> ignore
+			return 0, size, false, nil
+		}
+		// start beyond or at size is unsatisfiable -> error (RequestedRangeNotSatisfiable)
+		if startOffset >= size {
+			return 0, 0, false, errInvalidRange
+		}
+		// bytes=100- => from start to end
+		return startOffset, size - startOffset, true, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, strconv.IntSize)
+	if endOffset > int64(math.MaxInt) {
+		return 0, size, false, errInvalidRange
+	}
+	if err != nil { // invalid numeric end -> ignore range
+		return 0, size, false, nil
+	}
+
+	// Suffix range handling (bRange[0] == "")
+	if bRange[0] == "" {
+		// Disallow -0 (always unsatisfiable)
+		if endOffset == 0 {
+			return 0, 0, false, errInvalidRange
+		}
+		// For zero-sized objects any positive suffix is treated as invalid (ignored, no error)
+		if size == 0 {
+			return 0, size, false, nil
+		}
+		// Clamp to object size (request more bytes than exist -> entire object)
+		endOffset = min(endOffset, size)
+		return size - endOffset, endOffset, true, nil
+	}
+
+	// Normal range (start-end)
+	if startOffset > endOffset { // start > end -> ignore
+		return 0, size, false, nil
+	}
+	// Start beyond or at end of object -> error
+	if startOffset >= size {
+		return 0, 0, false, errInvalidRange
+	}
+	// Adjust end beyond object size (trim)
+	if endOffset >= size {
+		endOffset = size - 1
+	}
+	return startOffset, endOffset - startOffset + 1, true, nil
+}
+
+// ParseCopySourceRange parses input range header and returns startoffset, length
+// and error. If no endoffset specified, then length is set to the object size
+func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
+	if acceptRange == "" {
+		return 0, size, nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) != 2 {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if rangeKv[0] != "bytes" {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) != 2 {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if startOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
+	}
+
+	if bRange[1] == "" {
+		return startOffset, size - startOffset + 1, nil
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if endOffset < startOffset {
+		return 0, 0, errInvalidCopySourceRange
+	}
+
+	if endOffset >= size {
+		return 0, 0, s3err.CreateExceedingRangeErr(size)
+	}
+
+	return startOffset, endOffset - startOffset + 1, nil
+}
+
+// ParseCopySource parses x-amz-copy-source header and returns source bucket,
+// source object, versionId, error respectively
+func ParseCopySource(copySourceHeader string) (string, string, string, error) {
+	if copySourceHeader[0] == '/' {
+		copySourceHeader = copySourceHeader[1:]
+	}
+
+	var copySource, versionId string
+	i := strings.LastIndex(copySourceHeader, "?versionId=")
+	if i == -1 {
+		copySource = copySourceHeader
+	} else {
+		copySource = copySourceHeader[:i]
+		versionId = copySourceHeader[i+11:]
+	}
+
+	srcBucket, srcObject, ok := strings.Cut(copySource, "/")
+	if !ok {
+		return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
+	}
+
+	return srcBucket, srcObject, versionId, nil
+}
+
+// ParseObjectTags parses the url encoded input string into
+// map[string]string with unescaped key/value pair
+func ParseObjectTags(tagging string) (map[string]string, error) {
+	if tagging == "" {
+		return nil, nil
+	}
+
+	tagSet := make(map[string]string)
+
+	for tagging != "" {
+		var tag string
+		tag, tagging, _ = strings.Cut(tagging, "&")
+		// if 'tag' before the first appearance of '&' is empty continue
+		if tag == "" {
+			continue
+		}
+
+		key, value, found := strings.Cut(tag, "=")
+		// if key is empty, but "=" is present, return invalid url ecnoding err
+		if found && key == "" {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// return invalid tag key, if the key is longer than 128
+		if len(key) > 128 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// return invalid tag value, if tag value is longer than 256
+		if len(value) > 256 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// query unescape tag key
+		key, err := url.QueryUnescape(key)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// query unescape tag value
+		value, err = url.QueryUnescape(value)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		// check tag key to be valid
+		if !isValidTagComponent(key) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
+		}
+
+		// check tag value to be valid
+		if !isValidTagComponent(value) {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
+		}
+
+		// duplicate keys are not allowed: return invalid url encoding err
+		_, ok := tagSet[key]
+		if ok {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
+		}
+
+		tagSet[key] = value
+	}
+
+	return tagSet, nil
+}
+
+var validTagComponent = regexp.MustCompile(`^[a-zA-Z0-9:/_.\-+ ]+$`)
+
+// isValidTagComponent matches strings which contain letters, decimal digits,
+// and special chars: '/', '_', '-', '+', '.', ' ' (space)
+func isValidTagComponent(str string) bool {
+	if str == "" {
+		return true
+	}
+	return validTagComponent.Match([]byte(str))
+}
+
+func GetMultipartMD5(parts []types.CompletedPart) string {
+	var partsEtagBytes []byte
+	for _, part := range parts {
+		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
+	}
+
+	return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
+}
+
+func getEtagBytes(etag string) []byte {
+	decode, err := hex.DecodeString(strings.ReplaceAll(etag, string('"'), ""))
+	if err != nil {
+		return []byte(etag)
+	}
+	return decode
+}
+
+func md5String(data []byte) string {
+	sum := md5.Sum(data)
+	return hex.EncodeToString(sum[:])
+}
+
+type FileSectionReadCloser struct {
+	R io.Reader
+	F *os.File
+}
+
+func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
+	return f.R.Read(p)
+}
+
+func (f *FileSectionReadCloser) Close() error {
+	return f.F.Close()
+}
+
+// MoveFile moves a file from source to destination.
+func MoveFile(source, destination string, perm os.FileMode) error {
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	err := os.Rename(source, destination)
+	if err == nil || !errors.Is(err, syscall.EXDEV) {
+		return err
+	}
+
+	// Rename can fail if the source and destination are not on the same
+	// filesystem. The fallback is to copy the file and then remove the source.
+	// We need to be careful that the desination does not exist before copying
+	// to prevent any other simultaneous writes to the file.
+	sourceFile, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("open source: %w", err)
+	}
+	defer sourceFile.Close()
+
+	var destFile *os.File
+	for {
+		destFile, err = os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+		if err != nil {
+			if errors.Is(err, fs.ErrExist) {
+				if removeErr := os.Remove(destination); removeErr != nil {
+					return fmt.Errorf("remove existing destination: %w", removeErr)
+				}
+				continue
+			}
+			return fmt.Errorf("create destination: %w", err)
+		}
+		break
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return fmt.Errorf("copy data: %w", err)
+	}
+
+	err = os.Remove(source)
+	if err != nil {
+		return fmt.Errorf("remove source: %w", err)
+	}
+
+	return nil
+}
+
+// GenerateEtag generates a new quoted etag from the provided hash.Hash
+func GenerateEtag(h hash.Hash) string {
+	dataSum := h.Sum(nil)
+	return fmt.Sprintf("\"%s\"", hex.EncodeToString(dataSum[:]))
+}
+
+// AreEtagsSame compares 2 etags by ignoring quotes
+func AreEtagsSame(e1, e2 string) bool {
+	return strings.Trim(e1, `"`) == strings.Trim(e2, `"`)
+}
+
+func getBoolPtr(b bool) *bool {
+	return &b
+}
+
+type PreConditions struct {
+	IfMatch       *string
+	IfNoneMatch   *string
+	IfModSince    *time.Time
+	IfUnmodeSince *time.Time
+}
+
+// EvaluatePreconditions takes the object ETag, the last modified time and
+// evaluates the read preconditions:
+// - if-match,
+// - if-none-match
+// - if-modified-since
+// - if-unmodified-since
+// if-match and if-none-match are ETag comparisions
+// if-modified-since and if-unmodified-since are last modifed time comparisons
+func EvaluatePreconditions(etag string, modTime time.Time, preconditions PreConditions) error {
+	if preconditions.IfMatch == nil && preconditions.IfNoneMatch == nil && preconditions.IfModSince == nil && preconditions.IfUnmodeSince == nil {
+		return nil
+	}
+
+	// convert all conditions to *bool to evaluate the conditions
+	var ifMatch, ifNoneMatch, ifModSince, ifUnmodeSince *bool
+	if preconditions.IfMatch != nil {
+		ifMatch = getBoolPtr(*preconditions.IfMatch == etag)
+	}
+	if preconditions.IfNoneMatch != nil {
+		ifNoneMatch = getBoolPtr(*preconditions.IfNoneMatch != etag)
+	}
+	if preconditions.IfModSince != nil {
+		ifModSince = getBoolPtr(preconditions.IfModSince.UTC().Before(modTime.UTC()))
+	}
+	if preconditions.IfUnmodeSince != nil {
+		ifUnmodeSince = getBoolPtr(preconditions.IfUnmodeSince.UTC().After(modTime.UTC()))
+	}
+
+	if ifMatch != nil {
+		// if `if-match` doesn't matches, return PreconditionFailed
+		if !*ifMatch {
+			return errPreconditionFailed
+		}
+
+		// if-match matches
+		if *ifMatch {
+			if ifNoneMatch != nil {
+				// if `if-none-match` doesn't match return NotModified
+				if !*ifNoneMatch {
+					return errNotModified
+				}
+
+				// if both `if-match` and `if-none-match` match, return no error
+				return nil
+			}
+
+			// if `if-match` matches but `if-modified-since` is false return NotModified
+			if ifModSince != nil && !*ifModSince {
+				return errNotModified
+			}
+
+			// ignore `if-unmodified-since` as `if-match` is true
+			return nil
+		}
+	}
+
+	if ifNoneMatch != nil {
+		if *ifNoneMatch {
+			// if `if-none-match` is true, but `if-unmodified-since` is false
+			// return PreconditionFailed
+			if ifUnmodeSince != nil && !*ifUnmodeSince {
+				return errPreconditionFailed
+			}
+
+			// ignore `if-modified-since` as `if-none-match` is true
+			return nil
+		} else {
+			// if `if-none-match` is false and `if-unmodified-since` is false
+			// return PreconditionFailed
+			if ifUnmodeSince != nil && !*ifUnmodeSince {
+				return errPreconditionFailed
+			}
+
+			// in all other cases when `if-none-match` is false return NotModified
+			return errNotModified
+		}
+	}
+
+	if ifModSince != nil && !*ifModSince {
+		// if both `if-modified-since` and `if-unmodified-since` are false
+		// return PreconditionFailed
+		if ifUnmodeSince != nil && !*ifUnmodeSince {
+			return errPreconditionFailed
+		}
+
+		// if only `if-modified-since` is false, return NotModified
+		return errNotModified
+	}
+
+	// if `if-unmodified-since` is false return PreconditionFailed
+	if ifUnmodeSince != nil && !*ifUnmodeSince {
+		return errPreconditionFailed
+	}
+
+	return nil
+}
+
+// EvaluateMatchPreconditions evaluates if-match and if-none-match preconditions
+func EvaluateMatchPreconditions(etag string, ifMatch, ifNoneMatch *string) error {
+	if ifMatch != nil && *ifMatch != etag {
+		return errPreconditionFailed
+	}
+	if ifNoneMatch != nil && *ifNoneMatch == etag {
+		return errPreconditionFailed
+	}
+
+	return nil
+}
+
+type ObjectDeletePreconditions struct {
+	IfMatch            *string
+	IfMatchLastModTime *time.Time
+	IfMatchSize        *int64
+}
+
+// EvaluateObjectDeletePreconditions evaluates preconditions for DeleteObject
+func EvaluateObjectDeletePreconditions(etag string, modTime time.Time, size int64, preconditions ObjectDeletePreconditions) error {
+	ifMatch := preconditions.IfMatch
+	if ifMatch != nil && *ifMatch != etag {
+		return errPreconditionFailed
+	}
+
+	ifMatchTime := preconditions.IfMatchLastModTime
+	if ifMatchTime != nil && ifMatchTime.Unix() != modTime.Unix() {
+		return errPreconditionFailed
+	}
+
+	ifMatchSize := preconditions.IfMatchSize
+	if ifMatchSize != nil && *ifMatchSize != size {
+		return errPreconditionFailed
+	}
+
+	return nil
+}

backend/meta/meta.go

@@ -0,0 +1,42 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import "os"
+
+// MetadataStorer defines the interface for managing metadata.
+// When object == "", the operation is on the bucket.
+type MetadataStorer interface {
+	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+	// Returns the value of the attribute, or an error if the attribute does not exist.
+	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
+
+	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+	// If attribute already exists, new attribute should replace existing.
+	// Returns an error if the operation fails.
+	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
+
+	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+	// Returns an error if the operation fails.
+	DeleteAttribute(bucket, object, attribute string) error
+
+	// ListAttributes lists all attributes for an object or a bucket.
+	// Returns list of attribute names, or an error if the operation fails.
+	ListAttributes(bucket, object string) ([]string, error)
+
+	// DeleteAttributes removes all attributes for an object or a bucket.
+	// Returns an error if the operation fails.
+	DeleteAttributes(bucket, object string) error
+}

backend/meta/none.go

@@ -0,0 +1,54 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"os"
+)
+
+// NoMeta is a metadata storer that does not store metadata.
+// This can be useful for read only mounts where attempting to store metadata
+// would fail.
+type NoMeta struct{}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+// always returns ErrNoSuchKey
+func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
+	return nil, ErrNoSuchKey
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+// always returns nil without storing the attribute
+func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+// always returns nil without deleting the attribute
+func (NoMeta) DeleteAttribute(_, _, _ string) error {
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+// always returns an empty list of attributes
+func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
+	return []string{}, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+// always returns nil without deleting any attributes
+func (NoMeta) DeleteAttributes(bucket, object string) error {
+	return nil
+}

backend/meta/sidecar.go

@@ -0,0 +1,139 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// SideCar is a metadata storer that uses sidecar files to store metadata.
+type SideCar struct {
+	dir string
+}
+
+const (
+	sidecarmeta = "meta"
+)
+
+// NewSideCar creates a new SideCar metadata storer.
+func NewSideCar(dir string) (SideCar, error) {
+	fi, err := os.Lstat(dir)
+	if err != nil {
+		return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
+	}
+	if !fi.IsDir() {
+		return SideCar{}, fmt.Errorf("not a directory")
+	}
+
+	return SideCar{dir: dir}, nil
+}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	value, err := os.ReadFile(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, ErrNoSuchKey
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attribute: %v", err)
+	}
+
+	return value, nil
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	err := os.MkdirAll(metadir, 0777)
+	if err != nil {
+		return fmt.Errorf("failed to create metadata directory: %v", err)
+	}
+
+	attr := filepath.Join(metadir, attribute)
+	err = os.WriteFile(attr, value, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to write attribute: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	err := os.Remove(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return ErrNoSuchKey
+	}
+	if err != nil {
+		return fmt.Errorf("failed to remove attribute: %v", err)
+	}
+
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	ents, err := os.ReadDir(metadir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []string{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to list attributes: %v", err)
+	}
+
+	var attrs []string
+	for _, ent := range ents {
+		attrs = append(attrs, ent.Name())
+	}
+
+	return attrs, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+func (s SideCar) DeleteAttributes(bucket, object string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	err := os.RemoveAll(metadir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove attributes: %v", err)
+	}
+	return nil
+}

backend/meta/xattr.go

@@ -0,0 +1,126 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	xattrPrefix = "user."
+)
+
+var (
+	// ErrNoSuchKey is returned when the key does not exist.
+	ErrNoSuchKey = errors.New("no such key")
+)
+
+type XattrMeta struct{}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
+	if f != nil {
+		b, err := xattr.FGet(f, xattrPrefix+attribute)
+		if errors.Is(err, xattr.ENOATTR) {
+			return nil, ErrNoSuchKey
+		}
+		return b, err
+	}
+
+	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
+	if errors.Is(err, xattr.ENOATTR) {
+		return nil, ErrNoSuchKey
+	}
+	return b, err
+}
+
+// StoreAttribute stores the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
+	if f != nil {
+		err := xattr.FSet(f, xattrPrefix+attribute, value)
+		if errors.Is(err, syscall.EROFS) {
+			return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return err
+	}
+
+	err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
+	return err
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
+	err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
+	if errors.Is(err, xattr.ENOATTR) {
+		return ErrNoSuchKey
+	}
+	if errors.Is(err, syscall.EROFS) {
+		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+	}
+	return err
+}
+
+// DeleteAttributes is not implemented for xattr since xattrs
+// are automatically removed when the file is deleted.
+func (x XattrMeta) DeleteAttributes(bucket, object string) error {
+	return nil
+}
+
+// ListAttributes lists all attributes for an object in a bucket.
+func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
+	attrs, err := xattr.List(filepath.Join(bucket, object))
+	if err != nil {
+		return nil, err
+	}
+	attributes := make([]string, 0, len(attrs))
+	for _, attr := range attrs {
+		if !isUserAttr(attr) {
+			continue
+		}
+		attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
+	}
+	return attributes, nil
+}
+
+func isUserAttr(attr string) bool {
+	return strings.HasPrefix(attr, xattrPrefix)
+}
+
+// Test is a helper function to test if xattrs are supported.
+func (x XattrMeta) Test(path string) error {
+	// check for platform support
+	if !xattr.XATTR_SUPPORTED {
+		return fmt.Errorf("xattrs are not supported on this platform")
+	}
+
+	// check if the filesystem supports xattrs
+	_, err := xattr.Get(path, "user.test")
+	if errors.Is(err, syscall.ENOTSUP) {
+		return fmt.Errorf("xattrs are not supported on this filesystem")
+	}
+
+	return nil
+}

backend/mkdir.go

@@ -0,0 +1,77 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// Copyright 2024 Versity Software
+
+// MkdirAll borrowed from stdlib to add ability to set ownership
+// as directories are created
+
+package backend
+
+import (
+	"io/fs"
+	"os"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+// MkdirAll is similar to os.MkdirAll but it will return
+// ErrObjectParentIsFile when appropriate
+// MkdirAll creates a directory named path,
+// along with any necessary parents, and returns nil,
+// or else returns an error.
+// The permission bits perm (before umask) are used for all
+// directories that MkdirAll creates.
+// Any newly created directory is set to provided uid/gid ownership.
+// If path is already a directory, MkdirAll does nothing
+// and returns nil.
+// Any directory created will be set to provided uid/gid ownership
+// if doChown is true.
+func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, dirPerm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return err
+	}
+	if doChown {
+		err = os.Chown(path, uid, gid)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

backend/posix/with_otmpfile.go

@@ -0,0 +1,272 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build linux
+// +build linux
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+	"golang.org/x/sys/unix"
+)
+
+const procfddir = "/proc/self/fd"
+
+type tmpfile struct {
+	f          *os.File
+	bucket     string
+	objname    string
+	isOTmp     bool
+	size       int64
+	needsChown bool
+	uid        int
+	gid        int
+	newDirPerm fs.FileMode
+}
+
+var (
+	// TODO: make this configurable
+	defaultFilePerm uint32 = 0644
+)
+
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool, forceNoTmpFile bool) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
+	if forceNoTmpFile {
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
+	}
+
+	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
+	// This can help reduce contention within the namespace (parent directories),
+	// etc. And will auto cleanup the inode on close if we never link this
+	// file descriptor into the namespace.
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+
+		// O_TMPFILE not supported, try fallback
+		return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
+	}
+
+	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
+	// later to link file into namespace
+	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
+
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		isOTmp:     true,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+		newDirPerm: p.newDirPerm,
+	}
+
+	// falloc is best effort, its fine if this fails
+	if size > 0 && dofalloc {
+		tmp.falloc()
+	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
+	return tmp, nil
+}
+
+func (p *Posix) openMkTemp(dir, bucket, obj string, size int64, dofalloc bool, uid, gid int, doChown bool) (*tmpfile, error) {
+	err := backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, err
+	}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+	// falloc is best effort, its fine if this fails
+	if size > 0 && dofalloc {
+		tmp.falloc()
+	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
+	return tmp, nil
+}
+
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
+func (tmp *tmpfile) link() error {
+	// make sure this is cleaned up in all error cases
+	defer tmp.f.Close()
+
+	// We use Linkat/Rename as the atomic operation for object puts. The
+	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
+	// with any other simultaneous uploads. The final operation is to move the
+	// temp file into place for the object. This ensures the object semantics
+	// of last upload completed wins and is not some combination of writes
+	// from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+
+	dir := filepath.Dir(objPath)
+
+	err := backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
+	if err != nil {
+		return fmt.Errorf("make parent dir: %w", err)
+	}
+
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
+	}
+
+	procdir, err := os.Open(procfddir)
+	if err != nil {
+		return fmt.Errorf("open proc dir: %w", err)
+	}
+	defer procdir.Close()
+
+	dirf, err := os.Open(dir)
+	if err != nil {
+		return fmt.Errorf("open parent dir: %w", err)
+	}
+	defer dirf.Close()
+
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if errors.Is(err, syscall.EEXIST) {
+		// Linkat cannot overwrite files; we will allocate a temporary file, Linkat to it and then Renameat it
+		// to avoid potential race condition
+		retries := 1
+		for {
+			tmpName := fmt.Sprintf(".%s.sgwtmp.%d", filepath.Base(objPath), time.Now().UnixNano())
+			err := unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+				int(dirf.Fd()), tmpName, unix.AT_SYMLINK_FOLLOW)
+			if errors.Is(err, syscall.EEXIST) && retries < 3 {
+				retries += 1
+				continue
+			}
+			if err != nil {
+				return fmt.Errorf("cannot find free temporary file: %w", err)
+			}
+
+			err = unix.Renameat(int(dirf.Fd()), tmpName, int(dirf.Fd()), filepath.Base(objPath))
+			if err != nil {
+				return fmt.Errorf("overwriting renameat failed: %w", err)
+			}
+			break
+		}
+	} else if err != nil {
+		return fmt.Errorf("link tmpfile (fd %q as %q): %w",
+			filepath.Base(tmp.f.Name()), objPath, err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(fs.FileMode(defaultFilePerm))
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		// rename only works for files within the same filesystem
+		// if this fails fallback to copy
+		return backend.MoveFile(tempname, objPath, fs.FileMode(defaultFilePerm))
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}
+
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}

backend/posix/without_otmpfile.go

@@ -0,0 +1,112 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !linux
+// +build !linux
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool, _ bool) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
+	// Create a temp file for upload while in progress (see link comments below).
+	var err error
+	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		if errors.Is(err, syscall.EROFS) {
+			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
+		}
+		return nil, fmt.Errorf("create temp file: %w", err)
+	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+var (
+	// TODO: make this configurable
+	defaultFilePerm fs.FileMode = 0644
+)
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(defaultFilePerm)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	return backend.MoveFile(tempname, objPath, defaultFilePerm)
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}
+
+func (tmp *tmpfile) File() *os.File {
+	return tmp.f
+}

backend/s3proxy/client.go

@@ -0,0 +1,75 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+)
+
+func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
+	cfg, err := s.getConfig(ctx, s.access, s.secret)
+	if err != nil {
+		return nil, err
+	}
+
+	if s.endpoint != "" {
+		return s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &s.endpoint
+			o.UsePathStyle = s.usePathStyle
+			// The http body stream is not seekable, so most operations cannot
+			// be retried. The error returned to the original client may be
+			// retried by the client.
+			o.Retryer = aws.NopRetryer{}
+		}), nil
+	}
+
+	return s3.NewFromConfig(cfg), nil
+}
+
+func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.disableChecksum {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(ctx, opts...)
+}

backend/scoutfs/scoutfs.go

@@ -0,0 +1,372 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package scoutfs
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+// ScoutfsOpts are the options for the ScoutFS backend
+type ScoutfsOpts struct {
+	// ChownUID sets the UID of the object to the UID of the user on PUT
+	ChownUID bool
+	// ChownGID sets the GID of the object to the GID of the user on PUT
+	ChownGID bool
+	// BucketLinks enables symlinks to directories to be treated as buckets
+	BucketLinks bool
+	//VersioningDir sets the version directory to enable object versioning
+	VersioningDir string
+	// NewDirPerm specifies the permission to set on newly created directories
+	NewDirPerm fs.FileMode
+	// GlacierMode enables glacier emulation for offline files
+	GlacierMode bool
+	// DisableNoArchive prevents setting noarchive on temporary files
+	DisableNoArchive bool
+}
+
+type ScoutFS struct {
+	*posix.Posix
+	rootfd  *os.File
+	rootdir string
+
+	// glaciermode enables the following behavior:
+	// GET object:  if file offline, return invalid object state
+	// HEAD object: if file offline, set obj storage class to GLACIER
+	//              if file offline and staging, x-amz-restore: ongoing-request="true"
+	//              if file offline and not staging, x-amz-restore: ongoing-request="false"
+	//              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
+	//              note: this expiry-date is not used but provided for client glacier compatibility
+	// ListObjects: if file offline, set obj storage class to GLACIER
+	// RestoreObject: add batch stage request to file
+	glaciermode bool
+
+	// disableNoArchive is used to disable setting scoutam noarchive flag
+	// on mutlipart parts. This is enabled by default to prevent archive
+	// copies of temporary multipart parts.
+	disableNoArchive bool
+}
+
+var _ backend.Backend = &ScoutFS{}
+
+const (
+	stageComplete      = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\""
+	stageInProgress    = "true"
+	stageNotInProgress = "false"
+)
+
+const (
+	// ScoutFS special xattr types
+	systemPrefix = "scoutfs.hide."
+	onameAttr    = systemPrefix + "objname"
+	flagskey     = systemPrefix + "sam_flags"
+	stagecopykey = systemPrefix + "sam_stagereq"
+)
+
+const (
+	// ScoutAM Flags
+
+	// Staging - file requested stage
+	Staging uint64 = 1 << iota
+	// StageFail - all copies failed to stage
+	StageFail
+	// NoArchive - no archive copies of file should be made
+	NoArchive
+	// ExtCacheRequested means file policy requests Ext Cache
+	ExtCacheRequested
+	// ExtCacheDone means this file ext cache copy has been
+	// created already (and possibly pruned, so may not exist)
+	ExtCacheDone
+)
+
+func (s *ScoutFS) Shutdown() {
+	s.Posix.Shutdown()
+	s.rootfd.Close()
+	_ = s.rootdir
+}
+
+func (*ScoutFS) String() string {
+	return "ScoutFS Gateway"
+}
+
+func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	out, err := s.Posix.UploadPart(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	if !s.disableNoArchive {
+		sum := sha256.Sum256([]byte(*input.Key))
+		partPath := filepath.Join(
+			*input.Bucket,                        // bucket
+			posix.MetaTmpMultipartDir,            // temp multipart dir
+			fmt.Sprintf("%x", sum),               // hashed objname
+			*input.UploadId,                      // upload id
+			fmt.Sprintf("%v", *input.PartNumber), // part number
+		)
+
+		err = setNoArchive(partPath)
+		if err != nil {
+			return nil, fmt.Errorf("set noarchive: %w", err)
+		}
+	}
+
+	return out, err
+}
+
+// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
+// ioctl to not have to read and copy the part data to the final object. This
+// saves a read and write cycle for all mutlipart uploads.
+func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
+	return s.Posix.CompleteMultipartUploadWithCopy(ctx, input, moveData)
+}
+
+func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	res, err := s.Posix.HeadObject(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	if s.glaciermode {
+		objPath := filepath.Join(*input.Bucket, *input.Key)
+
+		stclass := types.StorageClassStandard
+		requestOngoing := ""
+
+		requestOngoing = stageComplete
+
+		// Check if there are any offline exents associated with this file.
+		// If so, we will set storage class to glacier.
+		st, err := statMore(objPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat more: %w", err)
+		}
+		if st.Offline_blocks != 0 {
+			stclass = types.StorageClassGlacier
+			requestOngoing = stageNotInProgress
+
+			ok, err := isStaging(objPath)
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+			}
+			if err != nil {
+				return nil, fmt.Errorf("check stage status: %w", err)
+			}
+			if ok {
+				requestOngoing = stageInProgress
+			}
+		}
+
+		res.Restore = &requestOngoing
+		res.StorageClass = stclass
+	}
+
+	return res, nil
+}
+
+func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	objPath := filepath.Join(bucket, object)
+
+	fi, err := os.Stat(objPath)
+	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if errors.Is(err, syscall.ENAMETOOLONG) {
+		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat object: %w", err)
+	}
+
+	if strings.HasSuffix(object, "/") && !fi.IsDir() {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+
+	if s.glaciermode {
+		// Check if there are any offline exents associated with this file.
+		// If so, we will return the InvalidObjectState error.
+		st, err := statMore(objPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat more: %w", err)
+		}
+		if st.Offline_blocks != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidObjectState)
+		}
+	}
+
+	return s.Posix.GetObject(ctx, input)
+}
+
+func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
+	if s.glaciermode {
+		return s.Posix.ListObjectsParametrized(ctx, input, s.glacierFileToObj)
+	} else {
+		return s.Posix.ListObjects(ctx, input)
+	}
+}
+
+func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
+	if s.glaciermode {
+		return s.Posix.ListObjectsV2Parametrized(ctx, input, s.glacierFileToObj)
+	} else {
+		return s.Posix.ListObjectsV2(ctx, input)
+	}
+}
+
+// FileToObj function for ListObject calls that adds a Glacier storage class if the file is offline
+func (s *ScoutFS) glacierFileToObj(bucket string, fetchOwner bool) backend.GetObjFunc {
+	posixFileToObj := s.Posix.FileToObj(bucket, fetchOwner)
+
+	return func(path string, d fs.DirEntry) (s3response.Object, error) {
+		res, err := posixFileToObj(path, d)
+		if err != nil || d.IsDir() {
+			return res, err
+		}
+		objPath := filepath.Join(bucket, path)
+		// Check if there are any offline exents associated with this file.
+		// If so, we will return the Glacier storage class
+		st, err := statMore(objPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return s3response.Object{}, backend.ErrSkipObj
+		}
+		if err != nil {
+			return s3response.Object{}, fmt.Errorf("stat more: %w", err)
+		}
+		if st.Offline_blocks != 0 {
+			res.StorageClass = types.ObjectStorageClassGlacier
+		}
+		return res, nil
+	}
+}
+
+// RestoreObject will set stage request on file if offline and do nothing if
+// file is online
+func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
+	bucket := *input.Bucket
+	object := *input.Key
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	err = setStaging(filepath.Join(bucket, object))
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return fmt.Errorf("stage object: %w", err)
+	}
+
+	return nil
+}
+
+func isStaging(objname string) (bool, error) {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return false, err
+	}
+
+	var flags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &flags)
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return flags&Staging == Staging, nil
+}
+
+func setFlag(objname string, flag uint64) error {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return err
+	}
+
+	var oldflags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &oldflags)
+		if err != nil {
+			return err
+		}
+	}
+
+	newflags := oldflags | flag
+
+	if newflags == oldflags {
+		// no flags change, just return
+		return nil
+	}
+
+	b, err = json.Marshal(&newflags)
+	if err != nil {
+		return err
+	}
+
+	return xattr.Set(objname, flagskey, b)
+}
+
+func setStaging(objname string) error {
+	return setFlag(objname, Staging)
+}
+
+func setNoArchive(objname string) error {
+	return setFlag(objname, NoArchive)
+}
+
+func isNoAttr(err error) bool {
+	xerr, ok := err.(*xattr.Error)
+	if ok && xerr.Err == xattr.ENOATTR {
+		return true
+	}
+	return false
+}

backend/scoutfs/scoutfs_compat.go

@@ -0,0 +1,94 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build linux && amd64
+
+package scoutfs
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/versity/scoutfs-go"
+	"github.com/versity/versitygw/backend/meta"
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/debuglogger"
+)
+
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
+	metastore := meta.XattrMeta{}
+
+	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
+		ChownUID:      opts.ChownUID,
+		ChownGID:      opts.ChownGID,
+		BucketLinks:   opts.BucketLinks,
+		NewDirPerm:    opts.NewDirPerm,
+		VersioningDir: opts.VersioningDir,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := os.Open(rootdir)
+	if err != nil {
+		return nil, fmt.Errorf("open %v: %w", rootdir, err)
+	}
+
+	return &ScoutFS{
+		Posix:            p,
+		rootfd:           f,
+		rootdir:          rootdir,
+		glaciermode:      opts.GlacierMode,
+		disableNoArchive: opts.DisableNoArchive,
+	}, nil
+}
+
+func moveData(from *os.File, to *os.File) error {
+	// May fail if the files are not 4K aligned; check for alignment
+	ffi, err := from.Stat()
+	if err != nil {
+		return fmt.Errorf("stat from: %v", err)
+	}
+	tfi, err := to.Stat()
+	if err != nil {
+		return fmt.Errorf("stat to: %v", err)
+	}
+	if ffi.Size()%4096 != 0 || tfi.Size()%4096 != 0 {
+		return os.ErrInvalid
+	}
+
+	err = scoutfs.MoveData(from, to)
+	if err != nil {
+		debuglogger.Logf("ScoutFs MoveData failed: %v", err)
+	}
+	return err
+}
+
+func statMore(path string) (stat, error) {
+	st, err := scoutfs.StatMore(path)
+	if err != nil {
+		return stat{}, err
+	}
+	var s stat
+
+	s.Meta_seq = st.Meta_seq
+	s.Data_seq = st.Data_seq
+	s.Data_version = st.Data_version
+	s.Online_blocks = st.Online_blocks
+	s.Offline_blocks = st.Offline_blocks
+	s.Crtime_sec = st.Crtime_sec
+	s.Crtime_nsec = st.Crtime_nsec
+
+	return s, nil
+}

backend/scoutfs/scoutfs_incompat.go

@@ -0,0 +1,39 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !(linux && amd64)
+
+package scoutfs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+)
+
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
+	return nil, fmt.Errorf("scoutfs only available on linux")
+}
+
+var (
+	errNotSupported = errors.New("not supported")
+)
+
+func moveData(_, _ *os.File) error {
+	return errNotSupported
+}
+
+func statMore(_ string) (stat, error) {
+	return stat{}, errNotSupported
+}

backend/scoutfs/stat.go

@@ -0,0 +1,25 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package scoutfs
+
+type stat struct {
+	Meta_seq       uint64
+	Data_seq       uint64
+	Data_version   uint64
+	Online_blocks  uint64
+	Offline_blocks uint64
+	Crtime_sec     uint64
+	Crtime_nsec    uint32
+}

cmd/versitygw/admin.go

@@ -0,0 +1,550 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/tls"
+	"encoding/hex"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go"
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	adminAccess   string
+	adminSecret   string
+	adminRegion   string
+	adminEndpoint string
+	allowInsecure bool
+)
+
+func adminCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "admin",
+		Usage:       "admin CLI tool",
+		Description: `Admin CLI tool for interacting with admin APIs.`,
+		Subcommands: []*cli.Command{
+			{
+				Name:   "create-user",
+				Usage:  "Create a new user",
+				Action: createUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "access key id for the new user",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:     "secret",
+						Usage:    "secret access key for the new user",
+						Required: true,
+						Aliases:  []string{"s"},
+					},
+					&cli.StringFlag{
+						Name:     "role",
+						Usage:    "role for the new user",
+						Required: true,
+						Aliases:  []string{"r"},
+					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+				},
+			},
+			{
+				Name:   "update-user",
+				Usage:  "Updates a user account",
+				Action: updateUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "user access key id to be updated",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:    "secret",
+						Usage:   "secret access key for the new user",
+						Aliases: []string{"s"},
+					},
+					&cli.StringFlag{
+						Name:    "role",
+						Usage:   "the new user role",
+						Aliases: []string{"r"},
+					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+				},
+			},
+			{
+				Name:   "delete-user",
+				Usage:  "Delete a user",
+				Action: deleteUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "access key id of the user to be deleted",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+				},
+			},
+			{
+				Name:   "list-users",
+				Usage:  "List all the gateway users",
+				Action: listUsers,
+			},
+			{
+				Name:  "change-bucket-owner",
+				Usage: "Changes the bucket owner",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "bucket",
+						Usage:    "the bucket name to change the owner",
+						Required: true,
+						Aliases:  []string{"b"},
+					},
+					&cli.StringFlag{
+						Name:     "owner",
+						Usage:    "the user access key id, who should be the bucket owner",
+						Required: true,
+						Aliases:  []string{"o"},
+					},
+				},
+				Action: changeBucketOwner,
+			},
+			{
+				Name:   "list-buckets",
+				Usage:  "Lists all the gateway buckets and owners.",
+				Action: listBuckets,
+			},
+		},
+		Flags: []cli.Flag{
+			// TODO: create a configuration file for this
+			&cli.StringFlag{
+				Name:        "access",
+				Usage:       "admin access key id",
+				EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
+				Aliases:     []string{"a"},
+				Required:    true,
+				Destination: &adminAccess,
+			},
+			&cli.StringFlag{
+				Name:        "secret",
+				Usage:       "admin secret access key",
+				EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
+				Aliases:     []string{"s"},
+				Required:    true,
+				Destination: &adminSecret,
+			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "admin s3 region string",
+				EnvVars:     []string{"ADMIN_REGION"},
+				Value:       "us-east-1",
+				Destination: &adminRegion,
+				Aliases:     []string{"r"},
+			},
+			&cli.StringFlag{
+				Name:        "endpoint-url",
+				Usage:       "admin apis endpoint url",
+				EnvVars:     []string{"ADMIN_ENDPOINT_URL"},
+				Aliases:     []string{"er"},
+				Required:    true,
+				Destination: &adminEndpoint,
+			},
+			&cli.BoolFlag{
+				Name:        "allow-insecure",
+				Usage:       "disable tls certificate verification for the admin endpoint",
+				EnvVars:     []string{"ADMIN_ALLOW_INSECURE"},
+				Aliases:     []string{"ai"},
+				Destination: &allowInsecure,
+			},
+		},
+	}
+}
+
+func initHTTPClient() *http.Client {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
+	}
+	return &http.Client{Transport: tr}
+}
+
+func createUser(ctx *cli.Context) error {
+	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
+	userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
+	if access == "" || secret == "" {
+		return fmt.Errorf("invalid input parameters for the new user access/secret keys")
+	}
+	if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
+		return fmt.Errorf("invalid input parameter for role: %v", role)
+	}
+
+	acc := auth.Account{
+		Access:  access,
+		Secret:  secret,
+		Role:    auth.Role(role),
+		UserID:  userID,
+		GroupID: groupID,
+	}
+
+	accxml, err := xml.Marshal(acc)
+	if err != nil {
+		return fmt.Errorf("failed to parse user data: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(accxml)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
+
+	return nil
+}
+
+func deleteUser(ctx *cli.Context) error {
+	access := ctx.String("access")
+	if access == "" {
+		return fmt.Errorf("invalid input parameter for the user access key")
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
+
+	return nil
+}
+
+func updateUser(ctx *cli.Context) error {
+	access, secret, userId, groupId, role := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id"), auth.Role(ctx.String("role"))
+	props := auth.MutableProps{}
+	if ctx.IsSet("role") {
+		if !role.IsValid() {
+			return fmt.Errorf("invalid user role: %v", role)
+		}
+		props.Role = role
+	}
+	if ctx.IsSet("secret") {
+		props.Secret = &secret
+	}
+	if ctx.IsSet("user-id") {
+		props.UserID = &userId
+	}
+	if ctx.IsSet("group-id") {
+		props.GroupID = &groupId
+	}
+
+	propsxml, err := xml.Marshal(props)
+	if err != nil {
+		return fmt.Errorf("failed to parse user attributes: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(propsxml)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
+
+	return nil
+}
+
+func listUsers(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
+
+	var accs auth.ListUserAccountsResult
+	if err := xml.Unmarshal(body, &accs); err != nil {
+		return err
+	}
+
+	printAcctTable(accs.Accounts)
+
+	return nil
+}
+
+const (
+	// account table formatting
+	minwidth int  = 2   // minimal cell width including any padding
+	tabwidth int  = 0   // width of tab characters (equivalent number of spaces)
+	padding  int  = 2   // padding added to a cell before computing its width
+	padchar  byte = ' ' // ASCII char used for padding
+	flags    uint = 0   // formatting control flags
+)
+
+func printAcctTable(accs []auth.Account) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------")
+	for _, acc := range accs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func changeBucketOwner(ctx *cli.Context) error {
+	bucket, owner := ctx.String("bucket"), ctx.String("owner")
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
+
+	return nil
+}
+
+func printBuckets(buckets []s3response.Bucket) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Bucket\tOwner")
+	fmt.Fprintln(w, "-------\t----")
+	for _, acc := range buckets {
+		fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func listBuckets(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return parseApiError(body)
+	}
+
+	var result s3response.ListBucketsResult
+	if err := xml.Unmarshal(body, &result); err != nil {
+		return err
+	}
+
+	printBuckets(result.Buckets)
+
+	return nil
+}
+
+func parseApiError(body []byte) error {
+	var apiErr smithy.GenericAPIError
+	err := xml.Unmarshal(body, &apiErr)
+	if err != nil {
+		apiErr.Code = "InternalServerError"
+		apiErr.Message = err.Error()
+	}
+
+	return &apiErr
+}

cmd/versitygw/azure.go

@@ -0,0 +1,74 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/azure"
+)
+
+var (
+	azAccount, azKey, azServiceURL, azSASToken string
+)
+
+func azureCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "azure",
+		Usage:       "azure blob storage backend",
+		Description: `direct translation from s3 objects to azure blobs`,
+		Action:      runAzure,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "account",
+				Usage:       "azure account name",
+				EnvVars:     []string{"AZ_ACCOUNT_NAME"},
+				Aliases:     []string{"a"},
+				Destination: &azAccount,
+			},
+			&cli.StringFlag{
+				Name:        "access-key",
+				Usage:       "azure account key",
+				EnvVars:     []string{"AZ_ACCESS_KEY"},
+				Aliases:     []string{"k"},
+				Destination: &azKey,
+			},
+			&cli.StringFlag{
+				Name:        "sas-token",
+				Usage:       "azure blob storage SAS token",
+				EnvVars:     []string{"AZ_SAS_TOKEN"},
+				Aliases:     []string{"st"},
+				Destination: &azSASToken,
+			},
+			&cli.StringFlag{
+				Name:        "url",
+				Usage:       "azure service URL",
+				EnvVars:     []string{"AZ_ENDPOINT"},
+				Aliases:     []string{"u"},
+				Destination: &azServiceURL,
+			},
+		},
+	}
+}
+
+func runAzure(ctx *cli.Context) error {
+	be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
+	if err != nil {
+		return fmt.Errorf("init azure: %w", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/gateway_test.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/versity/versitygw/backend/meta"
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/tests/integration"
+)
+
+const (
+	tdir = "tempdir"
+)
+
+var (
+	wg sync.WaitGroup
+)
+
+func initEnv(dir string) {
+	// both
+	debug = true
+	region = "us-east-1"
+
+	// server
+	rootUserAccess = "user"
+	rootUserSecret = "pass"
+	iamDir = dir
+	port = "127.0.0.1:7070"
+
+	// client
+	awsID = "user"
+	awsSecret = "pass"
+	endpoint = "http://127.0.0.1:7070"
+}
+
+func initPosix(ctx context.Context) {
+	path, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("get current directory: %v", err)
+	}
+
+	tempdir := filepath.Join(path, tdir)
+	initEnv(tempdir)
+
+	err = os.RemoveAll(tempdir)
+	if err != nil {
+		log.Fatalf("remove temp directory: %v", err)
+	}
+
+	err = os.Mkdir(tempdir, 0755)
+	if err != nil {
+		log.Fatalf("make temp directory: %v", err)
+	}
+
+	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
+		NewDirPerm: 0755,
+	})
+	if err != nil {
+		log.Fatalf("init posix: %v", err)
+	}
+
+	wg.Add(1)
+	go func() {
+		err = runGateway(ctx, be)
+		if err != nil && err != context.Canceled {
+			log.Fatalf("run gateway: %v", err)
+		}
+
+		err := os.RemoveAll(tempdir)
+		if err != nil {
+			log.Fatalf("remove temp directory: %v", err)
+		}
+		wg.Done()
+	}()
+
+	// wait for server to start
+	time.Sleep(1 * time.Second)
+}
+
+func TestIntegration(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	initPosix(ctx)
+
+	opts := []integration.Option{
+		integration.WithAccess(awsID),
+		integration.WithSecret(awsSecret),
+		integration.WithRegion(region),
+		integration.WithEndpoint(endpoint),
+	}
+	if debug {
+		opts = append(opts, integration.WithDebug())
+	}
+
+	s := integration.NewS3Conf(opts...)
+
+	// replace below with desired test
+	err := integration.HeadBucket_non_existing_bucket(s)
+	if err != nil {
+		t.Error(err)
+	}
+
+	cancel()
+	wg.Wait()
+}

cmd/versitygw/plugin.go

@@ -0,0 +1,74 @@
+// Copyright 2025 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"plugin"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/plugins"
+)
+
+func pluginCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "plugin",
+		Usage:       "load a backend from a plugin",
+		Description: "Runs a s3 gateway and redirects the requests to the backend defined in the plugin",
+		Action:      runPluginBackend,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "config",
+				Usage:   "location of the config file",
+				Aliases: []string{"c"},
+			},
+		},
+	}
+}
+
+func runPluginBackend(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no plugin file provided to be loaded")
+	}
+
+	pluginPath := ctx.Args().Get(0)
+	config := ctx.String("config")
+
+	p, err := plugin.Open(pluginPath)
+	if err != nil {
+		return err
+	}
+
+	backendSymbol, err := p.Lookup("Backend")
+	if err != nil {
+		return err
+	}
+	backendPluginPtr, ok := backendSymbol.(*plugins.BackendPlugin)
+	if !ok {
+		return errors.New("plugin is not of type *plugins.BackendPlugin")
+	}
+
+	if backendPluginPtr == nil {
+		return errors.New("variable Backend is nil")
+	}
+
+	be, err := (*backendPluginPtr).New(config)
+	if err != nil {
+		return err
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/posix.go

@@ -0,0 +1,156 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"math"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/meta"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+var (
+	chownuid, chowngid bool
+	bucketlinks        bool
+	versioningDir      string
+	dirPerms           uint
+	sidecar            string
+	nometa             bool
+	forceNoTmpFile     bool
+)
+
+func posixCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "posix",
+		Usage: "posix filesystem storage backend",
+		Description: `Any posix filesystem that supports extended attributes. The top level
+directory for the gateway must be provided. All sub directories of the
+top level directory are treated as buckets, and all files/directories
+below the "bucket directory" are treated as the objects. The object
+name is split on "/" separator to translate to posix storage.
+For example:
+top level: /mnt/fs/gwroot
+bucket: mybucket
+object: a/b/c/myobject
+will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
+		Action: runPosix,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
+			&cli.StringFlag{
+				Name:        "versioning-dir",
+				Usage:       "the directory path to enable bucket versioning",
+				EnvVars:     []string{"VGW_VERSIONING_DIR"},
+				Destination: &versioningDir,
+			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
+			&cli.StringFlag{
+				Name:        "sidecar",
+				Usage:       "use provided sidecar directory to store metadata",
+				EnvVars:     []string{"VGW_META_SIDECAR"},
+				Destination: &sidecar,
+			},
+			&cli.BoolFlag{
+				Name:        "nometa",
+				Usage:       "disable metadata storage",
+				EnvVars:     []string{"VGW_META_NONE"},
+				Destination: &nometa,
+			},
+			&cli.BoolFlag{
+				Name:        "disableotmp",
+				Usage:       "disable O_TMPFILE support for new objects",
+				EnvVars:     []string{"VGW_DISABLE_OTMP"},
+				Destination: &forceNoTmpFile,
+			},
+		},
+	}
+}
+
+func runPosix(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no directory provided for operation")
+	}
+
+	gwroot := (ctx.Args().Get(0))
+
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
+	if nometa && sidecar != "" {
+		return fmt.Errorf("cannot use both nometa and sidecar metadata")
+	}
+
+	opts := posix.PosixOpts{
+		ChownUID:       chownuid,
+		ChownGID:       chowngid,
+		BucketLinks:    bucketlinks,
+		VersioningDir:  versioningDir,
+		NewDirPerm:     fs.FileMode(dirPerms),
+		ForceNoTmpFile: forceNoTmpFile,
+	}
+
+	var ms meta.MetadataStorer
+	switch {
+	case sidecar != "":
+		sc, err := meta.NewSideCar(sidecar)
+		if err != nil {
+			return fmt.Errorf("failed to init sidecar metadata: %w", err)
+		}
+		ms = sc
+		opts.SideCarDir = sidecar
+	case nometa:
+		ms = meta.NoMeta{}
+	default:
+		ms = meta.XattrMeta{}
+		err := meta.XattrMeta{}.Test(gwroot)
+		if err != nil {
+			return fmt.Errorf("xattr check failed: %w", err)
+		}
+	}
+
+	be, err := posix.New(gwroot, ms, opts)
+	if err != nil {
+		return fmt.Errorf("failed to init posix backend: %w", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/s3.go

@@ -0,0 +1,121 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/s3proxy"
+)
+
+var (
+	s3proxyAccess          string
+	s3proxySecret          string
+	s3proxyEndpoint        string
+	s3proxyRegion          string
+	s3proxyMetaBucket      string
+	s3proxyDisableChecksum bool
+	s3proxySslSkipVerify   bool
+	s3proxyUsePathStyle    bool
+	s3proxyDebug           bool
+)
+
+func s3Command() *cli.Command {
+	return &cli.Command{
+		Name:  "s3",
+		Usage: "s3 storage backend",
+		Description: `This runs the gateway like an s3 proxy redirecting requests
+to an s3 storage backend service.`,
+		Action: runS3,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "access",
+				Usage:       "s3 proxy server access key id",
+				Value:       "",
+				Required:    true,
+				EnvVars:     []string{"VGW_S3_ACCESS_KEY"},
+				Destination: &s3proxyAccess,
+				Aliases:     []string{"a"},
+			},
+			&cli.StringFlag{
+				Name:        "secret",
+				Usage:       "s3 proxy server secret access key",
+				Value:       "",
+				Required:    true,
+				EnvVars:     []string{"VGW_S3_SECRET_KEY"},
+				Destination: &s3proxySecret,
+				Aliases:     []string{"s"},
+			},
+			&cli.StringFlag{
+				Name:        "endpoint",
+				Usage:       "s3 service endpoint, default AWS if not specified",
+				Value:       "",
+				EnvVars:     []string{"VGW_S3_ENDPOINT"},
+				Destination: &s3proxyEndpoint,
+			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "s3 service region, default 'us-east-1' if not specified",
+				Value:       "us-east-1",
+				EnvVars:     []string{"VGW_S3_REGION"},
+				Destination: &s3proxyRegion,
+			},
+			&cli.StringFlag{
+				Name:        "meta-bucket",
+				Usage:       "s3 service meta bucket to store buckets acl/policy",
+				EnvVars:     []string{"VGW_S3_META_BUCKET"},
+				Destination: &s3proxyMetaBucket,
+			},
+			&cli.BoolFlag{
+				Name:        "disable-checksum",
+				Usage:       "disable gateway to server object checksums",
+				Value:       false,
+				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
+				Destination: &s3proxyDisableChecksum,
+			},
+			&cli.BoolFlag{
+				Name:        "ssl-skip-verify",
+				Usage:       "skip ssl cert verification for s3 service",
+				EnvVars:     []string{"VGW_S3_SSL_SKIP_VERIFY"},
+				Value:       false,
+				Destination: &s3proxySslSkipVerify,
+			},
+			&cli.BoolFlag{
+				Name:        "use-path-style",
+				Usage:       "use path style addressing for s3 proxy",
+				EnvVars:     []string{"VGW_S3_USE_PATH_STYLE"},
+				Value:       false,
+				Destination: &s3proxyUsePathStyle,
+			},
+			&cli.BoolFlag{
+				Name:        "debug",
+				Usage:       "output extra debug tracing",
+				Value:       false,
+				EnvVars:     []string{"VGW_S3_DEBUG"},
+				Destination: &s3proxyDebug,
+			},
+		},
+	}
+}
+
+func runS3(ctx *cli.Context) error {
+	be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
+	if err != nil {
+		return fmt.Errorf("init s3 backend: %w", err)
+	}
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/scoutfs.go

@@ -0,0 +1,123 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"math"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/scoutfs"
+)
+
+var (
+	glacier          bool
+	disableNoArchive bool
+)
+
+func scoutfsCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "scoutfs",
+		Usage: "scoutfs filesystem storage backend",
+		Description: `Support for ScoutFS.
+The top level directory for the gateway must be provided. All sub directories
+of the top level directory are treated as buckets, and all files/directories
+below the "bucket directory" are treated as the objects. The object name is
+split on "/" separator to translate to posix storage.
+For example:
+top level: /mnt/fs/gwroot
+bucket: mybucket
+object: a/b/c/myobject
+will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+
+ScoutFS contains optimizations for multipart uploads using extent
+move interfaces as well as support for tiered filesystems.`,
+		Action: runScoutfs,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "glacier",
+				Usage:       "enable glacier emulation mode",
+				Aliases:     []string{"g"},
+				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
+				Destination: &glacier,
+			},
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
+			&cli.BoolFlag{
+				Name:        "bucketlinks",
+				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
+				EnvVars:     []string{"VGW_BUCKET_LINKS"},
+				Destination: &bucketlinks,
+			},
+			&cli.StringFlag{
+				Name:        "versioning-dir",
+				Usage:       "the directory path to enable bucket versioning",
+				EnvVars:     []string{"VGW_VERSIONING_DIR"},
+				Destination: &versioningDir,
+			},
+			&cli.UintFlag{
+				Name:        "dir-perms",
+				Usage:       "default directory permissions for new directories",
+				EnvVars:     []string{"VGW_DIR_PERMS"},
+				Destination: &dirPerms,
+				DefaultText: "0755",
+				Value:       0755,
+			},
+			&cli.BoolFlag{
+				Name:        "disable-noarchive",
+				Usage:       "disable setting noarchive for multipart part uploads",
+				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
+				Destination: &disableNoArchive,
+			},
+		},
+	}
+}
+
+func runScoutfs(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no directory provided for operation")
+	}
+
+	if dirPerms > math.MaxUint32 {
+		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	}
+
+	var opts scoutfs.ScoutfsOpts
+	opts.GlacierMode = glacier
+	opts.ChownUID = chownuid
+	opts.ChownGID = chowngid
+	opts.BucketLinks = bucketlinks
+	opts.NewDirPerm = fs.FileMode(dirPerms)
+	opts.DisableNoArchive = disableNoArchive
+	opts.VersioningDir = versioningDir
+
+	be, err := scoutfs.New(ctx.Args().Get(0), opts)
+	if err != nil {
+		return fmt.Errorf("init scoutfs: %v", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/signal.go

@@ -0,0 +1,44 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+var (
+	sigDone = make(chan bool, 1)
+	sigHup  = make(chan bool, 1)
+)
+
+func setupSignalHandler() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
+
+	go func() {
+		for sig := range sigs {
+			fmt.Fprintf(os.Stderr, "caught signal %v\n", sig)
+			switch sig {
+			case syscall.SIGINT, syscall.SIGTERM:
+				sigDone <- true
+			case syscall.SIGHUP:
+				sigHup <- true
+			}
+		}
+	}()
+}

cmd/versitygw/test.go

@@ -0,0 +1,384 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/tests/integration"
+)
+
+var (
+	awsID             string
+	awsSecret         string
+	endpoint          string
+	prefix            string
+	dstBucket         string
+	partSize          int64
+	objSize           int64
+	concurrency       int
+	files             int
+	totalReqs         int
+	upload            bool
+	download          bool
+	hostStyle         bool
+	checksumDisable   bool
+	versioningEnabled bool
+	azureTests        bool
+	tlsStatus         bool
+)
+
+func testCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "test",
+		Usage: "Client side testing command for the gateway",
+		Description: `The testing CLI is used to test group of versitygw actions.
+		It also includes some performance and stress testing`,
+		Subcommands: initTestCommands(),
+		Flags:       initTestFlags(),
+	}
+}
+
+func initTestFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "access",
+			Usage:       "aws user access key",
+			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"},
+			Aliases:     []string{"a"},
+			Destination: &awsID,
+		},
+		&cli.StringFlag{
+			Name:        "secret",
+			Usage:       "aws user secret access key",
+			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"},
+			Aliases:     []string{"s"},
+			Destination: &awsSecret,
+		},
+		&cli.StringFlag{
+			Name:        "endpoint",
+			Usage:       "s3 server endpoint",
+			Destination: &endpoint,
+			Aliases:     []string{"e"},
+		},
+		&cli.BoolFlag{
+			Name:        "host-style",
+			Usage:       "Use host-style bucket addressing",
+			Value:       false,
+			Destination: &hostStyle,
+		},
+		&cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "enable debug mode",
+			Aliases:     []string{"d"},
+			Destination: &debug,
+		},
+		&cli.BoolFlag{
+			Name:        "allow-insecure",
+			Usage:       "skip tls verification",
+			Aliases:     []string{"ai"},
+			Destination: &tlsStatus,
+		},
+	}
+}
+
+func initTestCommands() []*cli.Command {
+	return append([]*cli.Command{
+		{
+			Name:        "full-flow",
+			Usage:       "Tests the full flow of gateway.",
+			Description: `Runs all the available tests to test the full flow of the gateway.`,
+			Action:      getAction(integration.TestFullFlow),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+				&cli.BoolFlag{
+					Name:        "azure-test-mode",
+					Usage:       "Skips tests that are not supported by Azure",
+					Destination: &azureTests,
+					Aliases:     []string{"azure"},
+				},
+			},
+		},
+		{
+			Name:   "posix",
+			Usage:  "Tests posix specific features",
+			Action: getAction(integration.TestPosix),
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test posix when versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
+		},
+		{
+			Name:   "scoutfs",
+			Usage:  "Tests scoutfs full flow",
+			Action: getAction(integration.TestScoutfs),
+		},
+		{
+			Name:   "iam",
+			Usage:  "Tests iam service",
+			Action: getAction(integration.TestIAM),
+		},
+		{
+			Name:   "access-control",
+			Usage:  "Tests gateway access control with bucket ACLs and Policies",
+			Action: getAction(integration.TestAccessControl),
+		},
+		{
+			Name:  "bench",
+			Usage: "Runs download/upload performance test on the gateway",
+			Description: `Uploads/downloads some number(specified by flags) of files with some capacity(bytes).
+			Logs the results to the console`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "files",
+					Usage:       "Number of objects to read/write",
+					Value:       1,
+					Destination: &files,
+				},
+				&cli.Int64Flag{
+					Name:        "objsize",
+					Usage:       "Uploading object size",
+					Value:       0,
+					Destination: &objSize,
+				},
+				&cli.StringFlag{
+					Name:        "prefix",
+					Usage:       "Object name prefix",
+					Destination: &prefix,
+				},
+				&cli.BoolFlag{
+					Name:        "upload",
+					Usage:       "Upload data to the gateway",
+					Value:       false,
+					Destination: &upload,
+				},
+				&cli.BoolFlag{
+					Name:        "download",
+					Usage:       "Download data to the gateway",
+					Value:       false,
+					Destination: &download,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to read/write data",
+					Destination: &dstBucket,
+				},
+				&cli.Int64Flag{
+					Name:        "partSize",
+					Usage:       "Upload/download size per thread",
+					Value:       64 * 1024 * 1024,
+					Destination: &partSize,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "Upload/download threads per object",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				if upload && download {
+					return fmt.Errorf("must only specify one of upload or download")
+				}
+				if !upload && !download {
+					return fmt.Errorf("must specify one of upload or download")
+				}
+
+				if dstBucket == "" {
+					return fmt.Errorf("must specify bucket")
+				}
+
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+					integration.WithPartSize(partSize),
+					integration.WithTLSStatus(tlsStatus),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				if upload {
+					return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+				} else {
+					return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+				}
+			},
+		},
+		{
+			Name:        "throughput",
+			Usage:       "Runs throughput performance test on the gateway",
+			Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "reqs",
+					Usage:       "Total number of requests to send.",
+					Value:       1000,
+					Destination: &totalReqs,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to make the requests",
+					Destination: &dstBucket,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "threads per request",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				if dstBucket == "" {
+					return fmt.Errorf("must specify the destination bucket")
+				}
+
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+					integration.WithTLSStatus(tlsStatus),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+			},
+		},
+	}, extractIntTests()...)
+}
+
+type testFunc func(*integration.S3Conf)
+
+func getAction(tf testFunc) func(*cli.Context) error {
+	return func(ctx *cli.Context) error {
+		opts := []integration.Option{
+			integration.WithAccess(awsID),
+			integration.WithSecret(awsSecret),
+			integration.WithRegion(region),
+			integration.WithEndpoint(endpoint),
+			integration.WithTLSStatus(tlsStatus),
+		}
+		if debug {
+			opts = append(opts, integration.WithDebug())
+		}
+		if versioningEnabled {
+			opts = append(opts, integration.WithVersioningEnabled())
+		}
+		if azureTests {
+			opts = append(opts, integration.WithAzureMode())
+		}
+		if hostStyle {
+			opts = append(opts, integration.WithHostStyle())
+		}
+
+		s := integration.NewS3Conf(opts...)
+		tf(s)
+
+		fmt.Println()
+		fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount)
+		if integration.FailCount > 0 {
+			return fmt.Errorf("test failed with %v errors", integration.FailCount)
+		}
+		return nil
+	}
+}
+
+func extractIntTests() (commands []*cli.Command) {
+	tests := integration.GetIntTests()
+	for key, val := range tests {
+		k := key
+		testFunc := val
+		commands = append(commands, &cli.Command{
+			Name:  k,
+			Usage: fmt.Sprintf("Runs %v integration test", key),
+			Action: func(ctx *cli.Context) error {
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithTLSStatus(tlsStatus),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if versioningEnabled {
+					opts = append(opts, integration.WithVersioningEnabled())
+				}
+				if hostStyle {
+					opts = append(opts, integration.WithHostStyle())
+				}
+
+				s := integration.NewS3Conf(opts...)
+				err := testFunc(s)
+				return err
+			},
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:        "versioning-enabled",
+					Usage:       "Test the bucket object versioning, if the versioning is enabled",
+					Destination: &versioningEnabled,
+					Aliases:     []string{"vs"},
+				},
+			},
+		})
+	}
+	return
+}

cmd/versitygw/utils.go

@@ -0,0 +1,91 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/s3event"
+)
+
+func utilsCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "utils",
+		Usage: "utility helper CLI tool",
+		Subcommands: []*cli.Command{
+			{
+				Name:    "gen-event-filter-config",
+				Aliases: []string{"gefc"},
+				Usage:   "Create a new configuration file for bucket event notifications filter.",
+				Action:  generateEventFiltersConfig,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:    "path",
+						Usage:   "the path where the config file has to be created",
+						Aliases: []string{"p"},
+					},
+				},
+			},
+		},
+	}
+}
+
+func generateEventFiltersConfig(ctx *cli.Context) error {
+	pathFlag := ctx.String("path")
+	path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
+	if err != nil {
+		return err
+	}
+
+	config := s3event.EventFilter{
+		s3event.EventObjectCreated:              true,
+		s3event.EventObjectCreatedPut:           true,
+		s3event.EventObjectCreatedPost:          true,
+		s3event.EventObjectCreatedCopy:          true,
+		s3event.EventCompleteMultipartUpload:    true,
+		s3event.EventObjectRemoved:              true,
+		s3event.EventObjectRemovedDelete:        true,
+		s3event.EventObjectRemovedDeleteObjects: true,
+		s3event.EventObjectTagging:              true,
+		s3event.EventObjectTaggingPut:           true,
+		s3event.EventObjectTaggingDelete:        true,
+		s3event.EventObjectAclPut:               true,
+		s3event.EventObjectRestore:              true,
+		s3event.EventObjectRestorePost:          true,
+		s3event.EventObjectRestoreCompleted:     true,
+	}
+
+	configBytes, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		return fmt.Errorf("parse event config: %w", err)
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create config file: %w", err)
+	}
+	defer file.Close()
+
+	_, err = file.Write(configBytes)
+	if err != nil {
+		return fmt.Errorf("write config file: %w", err)
+	}
+
+	return nil
+}

debuglogger/logger.go

@@ -0,0 +1,275 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package debuglogger
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"sync/atomic"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type Color string
+type prefix string
+
+const (
+	green  Color = "\033[32m"
+	yellow Color = "\033[33m"
+	blue   Color = "\033[34m"
+	red    Color = "\033[31m"
+	Purple Color = "\033[0;35m"
+
+	prefixPanic        prefix = "[PANIC]: "
+	prefixInernalError prefix = "[INTERNAL ERROR]: "
+	prefixInfo         prefix = "[INFO]: "
+	prefixDebug        prefix = "[DEBUG]: "
+
+	reset      = "\033[0m"
+	borderChar = "─"
+	boxWidth   = 120
+)
+
+// Panic prints the panics out in the console
+func Panic(er error) {
+	printError(prefixPanic, er)
+}
+
+// InernalError prints the internal error out in the console
+func InernalError(er error) {
+	printError(prefixInernalError, er)
+}
+
+func printError(prefix prefix, er error) {
+	fmt.Fprintf(os.Stderr, string(red)+string(prefix)+"%v"+reset+"\n", er)
+}
+
+// Logs http request details: headers, body, params, query args
+func LogFiberRequestDetails(ctx *fiber.Ctx) {
+	// Log the full request url
+	fullURL := ctx.Protocol() + "://" + ctx.Hostname() + ctx.OriginalURL()
+	fmt.Printf("%s[URL]: %s%s\n", green, fullURL, reset)
+
+	// log request headers
+	wrapInBox(green, "REQUEST HEADERS", boxWidth, func() {
+		for key, value := range ctx.Request().Header.All() {
+			printWrappedLine(yellow, string(key), string(value))
+		}
+	})
+	// skip request body log for PutObject and UploadPart
+	skipBodyLog := isLargeDataAction(ctx)
+	if !skipBodyLog {
+		body := ctx.Request().Body()
+		if len(body) != 0 {
+			printBoxTitleLine(blue, "REQUEST BODY", boxWidth, false)
+			fmt.Printf("%s%s%s\n", blue, body, reset)
+			printHorizontalBorder(blue, boxWidth, false)
+		}
+	}
+
+	if ctx.Request().URI().QueryArgs().Len() != 0 {
+		for key, value := range ctx.Request().URI().QueryArgs().All() {
+			log.Printf("%s: %s", key, value)
+		}
+	}
+}
+
+// Logs http response details: body, headers
+func LogFiberResponseDetails(ctx *fiber.Ctx) {
+	wrapInBox(green, "RESPONSE HEADERS", boxWidth, func() {
+		for key, value := range ctx.Response().Header.All() {
+			printWrappedLine(yellow, string(key), string(value))
+		}
+	})
+
+	_, ok := ctx.Locals("skip-res-body-log").(bool)
+	if !ok {
+		body := ctx.Response().Body()
+		if len(body) != 0 {
+			PrintInsideHorizontalBorders(blue, "RESPONSE BODY", string(body), boxWidth)
+		}
+	}
+}
+
+var debugEnabled atomic.Bool
+
+// SetDebugEnabled sets the debug mode
+func SetDebugEnabled() {
+	debugEnabled.Store(true)
+}
+
+// IsDebugEnabled returns true if debugging is enabled
+func IsDebugEnabled() bool {
+	return debugEnabled.Load()
+}
+
+// Logf is the same as 'fmt.Printf' with debug prefix,
+// a color added and '\n' at the end
+func Logf(format string, v ...any) {
+	if !debugEnabled.Load() {
+		return
+	}
+
+	fmt.Printf(string(yellow)+string(prefixDebug)+format+reset+"\n", v...)
+}
+
+// Infof prints out green info block with [INFO]: prefix
+func Infof(format string, v ...any) {
+	if !debugEnabled.Load() {
+		return
+	}
+
+	fmt.Printf(string(green)+string(prefixInfo)+format+reset+"\n", v...)
+}
+
+var debugIAMEnabled atomic.Bool
+
+// SetIAMDebugEnabled sets the IAM debug mode
+func SetIAMDebugEnabled() {
+	debugIAMEnabled.Store(true)
+}
+
+// IsDebugEnabled returns true if debugging enabled
+func IsIAMDebugEnabled() bool {
+	return debugEnabled.Load()
+}
+
+// IAMLogf is the same as 'fmt.Printf' with debug prefix,
+// a color added and '\n' at the end
+func IAMLogf(format string, v ...any) {
+	if !debugIAMEnabled.Load() {
+		return
+	}
+
+	fmt.Printf(string(yellow)+string(prefixDebug)+format+reset+"\n", v...)
+}
+
+// PrintInsideHorizontalBorders prints the text inside horizontal
+// border and title in the center of upper border
+func PrintInsideHorizontalBorders(color Color, title, text string, width int) {
+	if !debugEnabled.Load() {
+		return
+	}
+	printBoxTitleLine(color, title, width, false)
+	fmt.Printf("%s%s%s\n", color, text, reset)
+	printHorizontalBorder(color, width, false)
+}
+
+// Prints out box title either with closing characters or not:  "┌", "┐"
+// e.g ┌────────────────[ RESPONSE HEADERS ]────────────────┐
+func printBoxTitleLine(color Color, title string, length int, closing bool) {
+	leftCorner, rightCorner := "┌", "┐"
+
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	// Calculate how many border characters are needed
+	titleFormatted := fmt.Sprintf("[ %s ]", title)
+	borderSpace := length - len(titleFormatted) - 2 // 2 for corners
+	leftLen := borderSpace / 2
+	rightLen := borderSpace - leftLen
+
+	// Build the line
+	line := leftCorner +
+		strings.Repeat(borderChar, leftLen) +
+		titleFormatted +
+		strings.Repeat(borderChar, rightLen) +
+		rightCorner
+
+	fmt.Println(string(color) + line + reset)
+}
+
+// Prints out a horizontal line either with closing characters or not: "└", "┘"
+func printHorizontalBorder(color Color, length int, closing bool) {
+	leftCorner, rightCorner := "└", "┘"
+	if !closing {
+		leftCorner, rightCorner = borderChar, borderChar
+	}
+
+	line := leftCorner + strings.Repeat(borderChar, length-2) + rightCorner + reset
+	fmt.Println(string(color) + line)
+}
+
+// wrapInBox wraps the output of a function call (fn) inside a styled box with a title.
+func wrapInBox(color Color, title string, length int, fn func()) {
+	printBoxTitleLine(color, title, length, true)
+	fn()
+	printHorizontalBorder(color, length, true)
+}
+
+// returns the provided string length
+// defaulting to 13 for exceeding lengths
+func getLen(str string) int {
+	if len(str) < 13 {
+		return 13
+	}
+
+	return len(str)
+}
+
+// prints a formatted key-value pair within a box layout,
+// wrapping the value text if it exceeds the allowed width.
+func printWrappedLine(keyColor Color, key, value string) {
+	prefix := fmt.Sprintf("%s│%s %s%-13s%s : ", green, reset, keyColor, key, reset)
+	prefixLen := len(prefix) - len(green) - len(reset) - len(keyColor) - len(reset)
+	// the actual prefix size without colors
+	actualPrefixLen := getLen(key) + 5
+
+	lineWidth := boxWidth - prefixLen
+	valueLines := wrapText(value, lineWidth)
+
+	for i, line := range valueLines {
+		if i == 0 {
+			if len(line) < lineWidth {
+				line += strings.Repeat(" ", lineWidth-len(line))
+			}
+			fmt.Printf("%s%s%s %s│%s\n", prefix, reset, line, green, reset)
+		} else {
+			line = strings.Repeat(" ", actualPrefixLen-2) + line
+			if len(line) < boxWidth-4 {
+				line += strings.Repeat(" ", boxWidth-len(line)-4)
+			}
+			fmt.Printf("%s│ %s%s %s│%s\n", green, reset, line, green, reset)
+		}
+	}
+}
+
+// wrapText splits the input text into lines of at most `width` characters each.
+func wrapText(text string, width int) []string {
+	var lines []string
+	for len(text) > width {
+		lines = append(lines, text[:width])
+		text = text[width:]
+	}
+	if text != "" {
+		lines = append(lines, text)
+	}
+	return lines
+}
+
+// TODO: remove this and use utils.IsBidDataAction after refactoring
+// and creating 'internal' package
+func isLargeDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}

extra/dashboard/README.md

@@ -0,0 +1,19 @@
+# Versity Gateway Dashboard
+
+This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana. 
+
+The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
+
+This implementation uses the influxql query language. 
+
+## Usage
+
+From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
+
+To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`. 
+
+The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
+
+The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with  username: `admin` and password: `admin`.
+
+To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

extra/dashboard/aws_env_setup.sh

@@ -0,0 +1,4 @@
+export AWS_SECRET_ACCESS_KEY=password
+export AWS_ACCESS_KEY_ID=user
+export AWS_ENDPOINT_URL=http://127.0.0.1:7070
+export AWS_REGION=us-east-1

extra/dashboard/docker-compose-metrics.yml

@@ -0,0 +1,64 @@
+services:
+  telegraf:
+    image: telegraf
+    container_name: telegraf
+    restart: always
+    volumes:
+    - ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      - influxdb
+    links:
+      - influxdb
+    ports:
+    - '8125:8125/udp'
+
+  influxdb:
+    image: influxdb
+    container_name: influxdb
+    restart: always
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
+      - DOCKER_INFLUXDB_INIT_ORG=myorg
+      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
+    ports:
+      - '8086:8086'
+    volumes:
+      - influxdb_data:/var/lib/influxdb
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana-server
+    restart: always
+    depends_on:
+      - influxdb
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_INSTALL_PLUGINS=
+    links:
+      - influxdb
+    ports:
+      - '3000:3000'
+    volumes:
+      - ./metrics-exploration/grafana_data:/var/lib/grafana
+      - ./metrics-exploration/provisioning:/etc/grafana/provisioning
+
+  versitygw:
+    image: versity/versitygw:latest
+    container_name: versitygw
+    ports:
+      - "7070:7070"
+    environment:
+      - ROOT_ACCESS_KEY=user
+      - ROOT_SECRET_KEY=password
+      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
+    depends_on:
+      - telegraf
+    command: >
+      posix /tmp/vgw
+
+volumes:
+  influxdb_data: {}