Merge pull request #432 from versity/dependabot/go_modules/dev-dependencies-56c9ed1da1

Bumps the dev-dependencies group with 8 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/Azure/azure-sdk-for-go/sdk/azcore](https://github.com/Azure/azure-sdk-for-go) | `1.9.2` | `1.10.0` |
| [github.com/Azure/azure-sdk-for-go/sdk/storage/azblob](https://github.com/Azure/azure-sdk-for-go) | `1.3.0` | `1.3.1` |
| [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.51.1` | `1.51.2` |
| [github.com/gofiber/fiber/v2](https://github.com/gofiber/fiber) | `2.52.1` | `2.52.2` |
| [golang.org/x/sys](https://github.com/golang/sys) | `0.17.0` | `0.18.0` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.27.4` | `1.27.5` |
| [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.4` | `1.17.5` |
| [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.16.6` | `1.16.7` |


Updates `github.com/Azure/azure-sdk-for-go/sdk/azcore` from 1.9.2 to 1.10.0
- [Release notes](https://github.com/Azure/azure-sdk-for-go/releases)
- [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md)
- [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.9.2...sdk/azcore/v1.10.0)

Updates `github.com/Azure/azure-sdk-for-go/sdk/storage/azblob` from 1.3.0 to 1.3.1
- [Release notes](https://github.com/Azure/azure-sdk-for-go/releases)
- [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md)
- [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.3.0...sdk/azcore/v1.3.1)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.51.1 to 1.51.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.51.1...service/s3/v1.51.2)

Updates `github.com/gofiber/fiber/v2` from 2.52.1 to 2.52.2
- [Release notes](https://github.com/gofiber/fiber/releases)
- [Commits](https://github.com/gofiber/fiber/compare/v2.52.1...v2.52.2)

Updates `golang.org/x/sys` from 0.17.0 to 0.18.0
- [Commits](https://github.com/golang/sys/compare/v0.17.0...v0.18.0)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.27.4 to 1.27.5
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.27.4...config/v1.27.5)

Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.4 to 1.17.5
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/v1.17.5/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.17.4...v1.17.5)

Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.16.6 to 1.16.7
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.16.6...v1.16.7)

---
updated-dependencies:
- dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azcore
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: dev-dependencies
- dependency-name: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: dev-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: dev-dependencies
- dependency-name: github.com/gofiber/fiber/v2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: dev-dependencies
- dependency-name: golang.org/x/sys
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: dev-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: dev-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/credentials
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: dev-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: dev-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>

.dockerignore

@@ -0,0 +1,46 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+cmd/versitygw/versitygw
+/versitygw
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# ignore IntelliJ directories
+.idea
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+/versitygw.spec.in
+*.tar
+*.tar.gz
+**/rand.data
+/profile.txt
+
+dist/
+
+# Release config files
+/.github
+
+# Docker configuration files
+*Dockerfile
+/docker-compose.yml
+
+# read files
+/LICENSE
+/NOTICE
+/CODE_OF_CONDUCT.md
+/README.md

.env.dev

@@ -0,0 +1,8 @@
+POSIX_PORT=7071
+PROXY_PORT=7070
+ACCESS_KEY_ID=user
+SECRET_ACCESS_KEY=pass
+IAM_DIR=.
+SETUP_DIR=.
+AZ_ACCOUNT_NAME=devstoreaccount1
+AZ_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Server Version**
+ output of
+```
+./versitygw -version
+uname -a
+```
+
+**Additional context**
+Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      dev-dependencies:
+        patterns:
+          - "*"

.github/workflows/docker.yaml

@@ -0,0 +1,45 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registries:
+    name: Push Docker image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            versity/versitygw
+            ghcr.io/${{ github.repository }}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/functional.yml

@@ -0,0 +1,30 @@
+name: functional tests
+on: pull_request
+jobs:
+
+  build:
+    name: RunTests
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+
+    - name: Get Dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Build and Run
+      run: |
+        make testbin
+        ./runtests.sh
+
+    - name: Coverage Report
+      run: |
+        go tool covdata percent -i=/tmp/covdata

.github/workflows/go.yml

@@ -7,14 +7,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: "1.20"
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
 
     - name: Verify all files pass gofmt formatting
       run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
@@ -23,5 +23,16 @@ jobs:
       run: |
         go get -v -t -d ./...
 
+    - name: Build
+      run: make
+
     - name: Test
-      run: go test -v -timeout 30s -tags=github ./...
+      run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
+
+    - name: Install govulncheck
+      run: go install golang.org/x/vuln/cmd/govulncheck@latest
+      shell: bash
+
+    - name: Run govulncheck
+      run: govulncheck ./...
+      shell: bash

.github/workflows/goreleaser.yml

@@ -0,0 +1,31 @@
+name: goreleaser
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+  # packages: write
+  # issues: write
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/static.yml

@@ -1,23 +1,22 @@
 name: staticcheck
 on: pull_request
 jobs: 
+
   build: 
     name: Check
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: "1.20"
-        id: go
-      - 
-        name: "Set up repo"
-        uses: actions/checkout@v1
-        with: 
-          fetch-depth: 1
-      - 
-        name: "staticcheck"
-        uses: dominikh/staticcheck-action@v1.3.0
-        with: 
-          install-go: false
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+ 
+    - name: "staticcheck"
+      uses: dominikh/staticcheck-action@v1.3.0
+      with: 
+        install-go: false

.github/workflows/system.yml

@@ -0,0 +1,49 @@
+name: system tests
+on: pull_request
+jobs:
+  build:
+    name: RunTests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Install ShellCheck
+        run: sudo apt-get install shellcheck
+
+      - name: Run ShellCheck
+        run: shellcheck -S warning ./tests/*.sh
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Get Dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Install BATS
+        run: |
+          git clone https://github.com/bats-core/bats-core.git
+          cd bats-core && ./install.sh $HOME
+
+      - name: Install s3cmd
+        run: |
+          sudo apt-get install s3cmd
+
+      - name: Build and run
+        run: |
+          make testbin
+          export AWS_ACCESS_KEY_ID=user
+          export AWS_SECRET_ACCESS_KEY=pass
+          export AWS_REGION=us-east-1
+          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
+          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
+          aws configure set aws_region $AWS_REGION --profile versity
+          mkdir /tmp/gw
+          export WORKSPACE=$GITHUB_WORKSPACE
+          openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh

.gitignore

@@ -7,6 +7,8 @@
 *.dll
 *.so
 *.dylib
+cmd/versitygw/versitygw
+/versitygw
 
 # Test binary, built with `go test -c`
 *.test
@@ -22,3 +24,28 @@ go.work
 
 # ignore IntelliJ directories
 .idea
+
+# ignore VS code directories
+.vscode
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+*.tar
+*.tar.gz
+**/rand.data
+/profile.txt
+
+dist/
+
+# secrets file for local github-actions testing
+tests/.secrets
+
+# IAM users files often created in testing
+users.json
+
+# env files for testing
+.env*
+!.env.default

.goreleaser.yaml

@@ -0,0 +1,87 @@
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - goos:
+      - linux
+      - darwin
+      # windows is untested, we can start doing windows releases
+      # if someone is interested in taking on testing
+      # - windows
+    env:
+      # disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
+      # once we need to enable this, we will need to do per distro releases
+      - CGO_ENABLED=0
+    main: ./cmd/versitygw
+    binary: versitygw
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_v{{ .Version }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge '
+
+nfpms:
+  - id: packages
+    package_name: versitygw
+    vendor: Versity Software
+    homepage: https://github.com/versity/versitygw
+    maintainer: Ben McClelland <ben.mcclelland@versity.com>
+
+    description: |-
+      The Versity S3 Gateway.
+      A high-performance tool facilitating translation between AWS S3 API
+      requests and various backend storage systems, including POSIX file
+      backend storage. Its stateless architecture enables deployment in
+      clusters for increased throughput, distributing requests across gateways
+      for optimal performance. With a focus on modularity, it supports future
+      extensions for additional backend systems.
+
+    license: Apache 2.0
+
+    builds:
+      - versitygw
+
+    formats:
+      - deb
+      - rpm
+
+    umask: 0o002
+    bindir: /usr/bin
+    epoch: "1"
+    release: "1"
+
+    rpm:
+      group: "System Environment/Daemons"
+
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+versitygw@versity.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

Dockerfile

@@ -0,0 +1,25 @@
+FROM golang:latest
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY ./ ./
+
+WORKDIR /app/cmd/versitygw
+ENV CGO_ENABLED=0
+RUN go build -o versitygw
+
+FROM alpine:latest
+
+# These arguments can be overriden when building the image
+ARG IAM_DIR=/tmp/vgw
+ARG SETUP_DIR=/tmp/vgw
+
+RUN mkdir -p $IAM_DIR
+RUN mkdir -p $SETUP_DIR
+
+COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
+
+ENTRYPOINT [ "/app/versitygw" ]

Dockerfile.dev

@@ -0,0 +1,18 @@
+FROM golang:latest
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY ./ ./
+COPY certs/* /etc/pki/tls/certs/
+
+ARG IAM_DIR=/tmp/vgw
+ARG SETUP_DIR=/tmp/vgw
+
+RUN mkdir -p $IAM_DIR
+RUN mkdir -p $SETUP_DIR
+
+RUN go get github.com/githubnemo/CompileDaemon
+RUN go install github.com/githubnemo/CompileDaemon

Dockerfile_test_bats

@@ -0,0 +1,70 @@
+FROM --platform=linux/arm64 ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    git \
+    make \
+    wget \
+    curl \
+    unzip \
+    tzdata \
+    s3cmd \
+    jq \
+    ca-certificates && \
+    update-ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /tmp
+
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+
+# Download Go 1.21 (adjust the version and platform as needed)
+RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
+
+# Extract the downloaded archive
+RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
+
+# Set Go environment variables
+ENV PATH="/usr/local/go/bin:${PATH}"
+ENV GOPATH="/go"
+ENV GOBIN="$GOPATH/bin"
+
+# Make the directory for Go packages
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+# Create tester user
+RUN groupadd -r tester && useradd -r -g tester tester
+RUN mkdir /home/tester && chown tester:tester /home/tester
+ENV HOME=/home/tester
+
+RUN git clone https://github.com/bats-core/bats-core.git && \
+    cd bats-core && \
+    ./install.sh /home/tester
+
+USER tester
+COPY --chown=tester:tester . /home/tester
+
+WORKDIR /home/tester
+RUN cp tests/.env.docker.default tests/.env.docker
+RUN cp tests/s3cfg.local.default tests/s3cfg.local
+RUN make
+
+RUN . tests/.secrets && \
+    export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
+    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
+    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
+    aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
+
+RUN mkdir /tmp/gw
+
+RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen_bits:2048 && \
+    openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
+        -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+ENV WORKSPACE=.
+ENV VERSITYGW_TEST_ENV=tests/.env.docker
+
+CMD ["tests/run_all.sh"]

Makefile

@@ -0,0 +1,96 @@
+# Copyright 2023 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOCLEAN=$(GOCMD) clean
+GOTEST=$(GOCMD) test
+
+BIN=versitygw
+
+VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
+BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
+TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
+
+LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
+
+all: build
+
+build: $(BIN)
+
+.PHONY: $(BIN)
+$(BIN):
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
+
+testbin:
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
+
+.PHONY: test
+test: 
+	$(GOTEST) ./...
+
+.PHONY: check
+check:
+# note this requires staticcheck be in your PATH:
+# export PATH=$PATH:~/go/bin
+# go install honnef.co/go/tools/cmd/staticcheck@latest
+	staticcheck ./...
+	golint ./...
+	gofmt -s -l .
+
+.PHONY: clean
+clean: 
+	$(GOCLEAN)
+
+.PHONY: cleanall
+cleanall: clean
+	rm -f $(BIN)
+	rm -f versitygw-*.tar
+	rm -f versitygw-*.tar.gz
+	rm -f versitygw.spec
+
+%.spec: %.spec.in
+	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
+	mv $@+ $@
+
+TARFILE = $(BIN)-$(VERSION).tar
+
+dist: $(BIN).spec
+	echo $(VERSION) >VERSION
+	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
+	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
+	rm -f VERSION
+	rm -f $(BIN).spec
+	gzip -f $(TARFILE)
+
+# Creates and runs S3 gateway instance in a docker container
+.PHONY: up-posix
+up-posix:
+	docker compose --env-file .env.dev up posix
+
+# Creates and runs S3 gateway proxy instance in a docker container
+.PHONY: up-proxy
+up-proxy:
+	docker compose --env-file .env.dev up proxy
+
+# Creates and runs S3 gateway to azurite instance in a docker container
+.PHONY: up-azurite
+up-azurite:
+	docker compose --env-file .env.dev up azurite azuritegw
+
+# Creates and runs both S3 gateway and proxy server instances in docker containers
+.PHONY: up-app
+up-app:
+	docker compose --env-file .env.dev up

NOTICE

@@ -0,0 +1,2 @@
+versitygw - Versity S3 Gateway
+Copyright 2023 Versity Software

README.md

@@ -0,0 +1,72 @@
+# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
+  <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
+</picture>
+
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
+
+**Current status:** Ready for general testing, Issue reports welcome.
+
+**News:**<br>
+* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
+
+
+See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
+
+* Share filesystem directory via S3 protocol
+* Proxy S3 requests to S3 storage
+* Simple to deploy S3 server with a single command
+* Protocol compatibility in `posix` allows common access to files via posix or S3 
+
+Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
+
+The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
+
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
+
+The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
+
+The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework.  This framework is actively developed with a focus on performance.  S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3. 
+
+## Getting Started
+See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
+
+### Run the gateway with posix backend:
+
+```
+mkdir /tmp/vgw
+ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
+```
+This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
+
+To get the usage output, run the following:
+
+```
+./versitygw --help
+```
+
+The command format is
+
+```
+versitygw [global options] command [command options] [arguments...]
+```
+The global options are specified before the backend type and the backend options are specified after.
+
+***
+
+#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
+
+### Contact
+![versity logo](https://www.versity.com/wp-content/uploads/2022/12/cropped-android-chrome-512x512-1-32x32.png)
+info@versity.com <br />
++1 844 726 8826
+
+### @versitysoftware 
+[![linkedin](https://github.com/versity/versitygw/blob/assets/assets/linkedin.jpg)](https://www.linkedin.com/company/versity/) &nbsp; 
+[![twitter](https://github.com/versity/versitygw/blob/assets/assets/twitter.jpg)](https://twitter.com/VersitySoftware) &nbsp;
+[![facebook](https://github.com/versity/versitygw/blob/assets/assets/facebook.jpg)](https://www.facebook.com/versitysoftware) &nbsp;
+[![instagram](https://github.com/versity/versitygw/blob/assets/assets/instagram.jpg)](https://www.instagram.com/versitysoftware/) &nbsp;
+

auth/acl.go

@@ -0,0 +1,275 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+)
+
+type ACL struct {
+	ACL      types.BucketCannedACL
+	Owner    string
+	Grantees []Grantee
+}
+
+type Grantee struct {
+	Permission types.Permission
+	Access     string
+}
+
+type GetBucketAclOutput struct {
+	Owner             *types.Owner
+	AccessControlList AccessControlList
+}
+
+type AccessControlList struct {
+	Grants []types.Grant `xml:"Grant"`
+}
+type AccessControlPolicy struct {
+	AccessControlList AccessControlList `xml:"AccessControlList"`
+	Owner             types.Owner
+}
+
+func ParseACL(data []byte) (ACL, error) {
+	if len(data) == 0 {
+		return ACL{}, nil
+	}
+
+	var acl ACL
+	if err := json.Unmarshal(data, &acl); err != nil {
+		return acl, fmt.Errorf("parse acl: %w", err)
+	}
+	return acl, nil
+}
+
+func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
+	var acl ACL
+	if err := json.Unmarshal(data, &acl); err != nil {
+		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
+	}
+
+	grants := []types.Grant{}
+
+	for _, elem := range acl.Grantees {
+		acs := elem.Access
+		grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
+	}
+
+	return GetBucketAclOutput{
+		Owner: &types.Owner{
+			ID: &acl.Owner,
+		},
+		AccessControlList: AccessControlList{
+			Grants: grants,
+		},
+	}, nil
+}
+
+func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
+	if input == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	// if the ACL is specified, set the ACL, else replace the grantees
+	if input.ACL != "" {
+		acl.ACL = input.ACL
+		acl.Grantees = []Grantee{}
+	} else {
+		grantees := []Grantee{}
+		accs := []string{}
+
+		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
+			fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
+
+			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
+				fullControlList = splitUnique(*input.GrantFullControl, ",")
+				for _, str := range fullControlList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+				}
+			}
+			if input.GrantRead != nil && *input.GrantRead != "" {
+				readList = splitUnique(*input.GrantRead, ",")
+				for _, str := range readList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+				}
+			}
+			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
+				readACPList = splitUnique(*input.GrantReadACP, ",")
+				for _, str := range readACPList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+				}
+			}
+			if input.GrantWrite != nil && *input.GrantWrite != "" {
+				writeList = splitUnique(*input.GrantWrite, ",")
+				for _, str := range writeList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+				}
+			}
+			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
+				writeACPList = splitUnique(*input.GrantWriteACP, ",")
+				for _, str := range writeACPList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+				}
+			}
+
+			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
+		} else {
+			cache := make(map[string]bool)
+			for _, grt := range input.AccessControlPolicy.Grants {
+				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
+					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+				}
+				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
+				if _, ok := cache[*grt.Grantee.ID]; !ok {
+					cache[*grt.Grantee.ID] = true
+					accs = append(accs, *grt.Grantee.ID)
+				}
+			}
+		}
+
+		// Check if the specified accounts exist
+		accList, err := CheckIfAccountsExist(accs, iam)
+		if err != nil {
+			return nil, err
+		}
+		if len(accList) > 0 {
+			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
+		}
+
+		acl.Grantees = grantees
+		acl.ACL = ""
+	}
+
+	result, err := json.Marshal(acl)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
+	result := []string{}
+
+	for _, acc := range accs {
+		_, err := iam.GetUserAccount(acc)
+		if err != nil {
+			if err == ErrNoSuchUser {
+				result = append(result, acc)
+				continue
+			}
+			if err == ErrNotSupported {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			}
+			return nil, fmt.Errorf("check user account: %w", err)
+		}
+	}
+	return result, nil
+}
+
+func splitUnique(s, divider string) []string {
+	elements := strings.Split(s, divider)
+	uniqueElements := make(map[string]bool)
+	result := make([]string, 0, len(elements))
+
+	for _, element := range elements {
+		if _, ok := uniqueElements[element]; !ok {
+			result = append(result, element)
+			uniqueElements[element] = true
+		}
+	}
+
+	return result
+}
+
+func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
+	if isRoot {
+		return nil
+	}
+
+	if acl.Owner == access {
+		return nil
+	}
+
+	if acl.ACL != "" {
+		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+
+		return nil
+	} else {
+		grantee := Grantee{Access: access, Permission: permission}
+		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
+
+		isFound := false
+
+		for _, grt := range acl.Grantees {
+			if grt == grantee || grt == granteeFullCtrl {
+				isFound = true
+				break
+			}
+		}
+
+		if isFound {
+			return nil
+		}
+	}
+
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}
+
+func MayCreateBucket(acct Account, isRoot bool) error {
+	if isRoot {
+		return nil
+	}
+
+	if acct.Role == RoleUser {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
+	// Owner check
+	if acct.Access == acl.Owner {
+		return nil
+	}
+
+	// Root user has access over almost everything
+	if isRoot {
+		return nil
+	}
+
+	// Admin user case
+	if acct.Role == RoleAdmin {
+		return nil
+	}
+
+	// Return access denied in all other cases
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}

auth/iam.go

@@ -0,0 +1,111 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+	"fmt"
+	"time"
+)
+
+type Role string
+
+const (
+	RoleUser     Role = "user"
+	RoleAdmin    Role = "admin"
+	RoleUserPlus Role = "userplus"
+)
+
+// Account is a gateway IAM account
+type Account struct {
+	Access    string `json:"access"`
+	Secret    string `json:"secret"`
+	Role      Role   `json:"role"`
+	UserID    int    `json:"userID"`
+	GroupID   int    `json:"groupID"`
+	ProjectID int    `json:"projectID"`
+}
+
+// IAMService is the interface for all IAM service implementations
+//
+//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
+type IAMService interface {
+	CreateAccount(account Account) error
+	GetUserAccount(access string) (Account, error)
+	DeleteUserAccount(access string) error
+	ListUserAccounts() ([]Account, error)
+	Shutdown() error
+}
+
+var ErrNoSuchUser = errors.New("user not found")
+
+type Opts struct {
+	Dir                string
+	LDAPServerURL      string
+	LDAPBindDN         string
+	LDAPPassword       string
+	LDAPQueryBase      string
+	LDAPObjClasses     string
+	LDAPAccessAtr      string
+	LDAPSecretAtr      string
+	LDAPRoleAtr        string
+	S3Access           string
+	S3Secret           string
+	S3Region           string
+	S3Bucket           string
+	S3Endpoint         string
+	S3DisableSSlVerfiy bool
+	S3Debug            bool
+	CacheDisable       bool
+	CacheTTL           int
+	CachePrune         int
+}
+
+func New(o *Opts) (IAMService, error) {
+	var svc IAMService
+	var err error
+
+	switch {
+	case o.Dir != "":
+		svc, err = NewInternal(o.Dir)
+		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
+	case o.LDAPServerURL != "":
+		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
+			o.LDAPObjClasses)
+		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
+	case o.S3Endpoint != "":
+		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
+		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
+			o.S3Endpoint, o.S3Bucket)
+	default:
+		// if no iam options selected, default to the single user mode
+		fmt.Println("No IAM service configured, enabling single account mode")
+		return IAMServiceSingle{}, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if o.CacheDisable {
+		return svc, nil
+	}
+
+	return NewCache(svc,
+		time.Duration(o.CacheTTL)*time.Second,
+		time.Duration(o.CachePrune)*time.Second), nil
+}

auth/iam_cache.go

@@ -0,0 +1,179 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+)
+
+// IAMCache is an in memory cache of the IAM accounts
+// with expiration. This helps to alleviate the load on
+// the real IAM service if the gateway is handling
+// many requests. This forwards account updates to the
+// underlying service, and returns cached results while
+// the in memory account is not expired.
+type IAMCache struct {
+	service  IAMService
+	iamcache *icache
+	cancel   context.CancelFunc
+}
+
+var _ IAMService = &IAMCache{}
+
+type item struct {
+	value Account
+	exp   time.Time
+}
+
+type icache struct {
+	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
+}
+
+func (i *icache) set(k string, v Account) {
+	cpy := v
+	i.Lock()
+	i.items[k] = item{
+		exp:   time.Now().Add(i.expire),
+		value: cpy,
+	}
+	i.Unlock()
+}
+
+func (i *icache) get(k string) (Account, bool) {
+	i.RLock()
+	v, ok := i.items[k]
+	i.RUnlock()
+	if !ok || !v.exp.After(time.Now()) {
+		return Account{}, false
+	}
+	return v.value, true
+}
+
+func (i *icache) Delete(k string) {
+	i.Lock()
+	delete(i.items, k)
+	i.Unlock()
+}
+
+func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+
+		now := time.Now()
+
+		i.Lock()
+		// prune expired entries
+		for k, v := range i.items {
+			if now.After(v.exp) {
+				delete(i.items, k)
+			}
+		}
+		i.Unlock()
+
+		// sleep for the clean interval or context cancelation,
+		// whichever comes first
+		select {
+		case <-ctx.Done():
+		case <-time.After(interval):
+		}
+	}
+}
+
+// NewCache initializes an IAM cache for the provided service. The expireTime
+// is the duration a cache entry can be valid, and the cleanupInterval is
+// how often to scan cache and cleanup expired entries.
+func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
+	i := &IAMCache{
+		service: service,
+		iamcache: &icache{
+			items:  make(map[string]item),
+			expire: expireTime,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go i.iamcache.gcCache(ctx, cleanupInterval)
+	i.cancel = cancel
+
+	return i
+}
+
+// CreateAccount send create to IAM service and creates an account cache entry
+func (c *IAMCache) CreateAccount(account Account) error {
+	err := c.service.CreateAccount(account)
+	if err != nil {
+		return err
+	}
+
+	// we need a copy of account to be able to store beyond the
+	// lifetime of the request, otherwise Fiber will reuse and corrupt
+	// these entries
+	acct := Account{
+		Access: strings.Clone(account.Access),
+		Secret: strings.Clone(account.Secret),
+		Role:   Role(strings.Clone(string(account.Role))),
+	}
+
+	c.iamcache.set(acct.Access, acct)
+	return nil
+}
+
+// GetUserAccount retrieves the cache account if it is in the cache and not
+// expired. Otherwise retrieves from underlying IAM service and caches
+// result for the expire duration.
+func (c *IAMCache) GetUserAccount(access string) (Account, error) {
+	acct, found := c.iamcache.get(access)
+	if found {
+		return acct, nil
+	}
+
+	a, err := c.service.GetUserAccount(access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	c.iamcache.set(access, a)
+	return a, nil
+}
+
+// DeleteUserAccount deletes account from IAM service and cache
+func (c *IAMCache) DeleteUserAccount(access string) error {
+	err := c.service.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.Delete(access)
+	return nil
+}
+
+// ListUserAccounts is a passthrough to the underlying service and
+// does not make use of the cache
+func (c *IAMCache) ListUserAccounts() ([]Account, error) {
+	return c.service.ListUserAccounts()
+}
+
+// Shutdown graceful termination of service
+func (c *IAMCache) Shutdown() error {
+	c.cancel()
+	return nil
+}

auth/iam_internal.go

@@ -0,0 +1,335 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+const (
+	iamFile       = "users.json"
+	iamBackupFile = "users.json.backup"
+)
+
+// IAMServiceInternal manages the internal IAM service
+type IAMServiceInternal struct {
+	dir string
+}
+
+// UpdateAcctFunc accepts the current data and returns the new data to be stored
+type UpdateAcctFunc func([]byte) ([]byte, error)
+
+// iAMConfig stores all internal IAM accounts
+type iAMConfig struct {
+	AccessAccounts map[string]Account `json:"accessAccounts"`
+}
+
+var _ IAMService = &IAMServiceInternal{}
+
+// NewInternal creates a new instance for the Internal IAM service
+func NewInternal(dir string) (*IAMServiceInternal, error) {
+	i := &IAMServiceInternal{
+		dir: dir,
+	}
+
+	err := i.initIAM()
+	if err != nil {
+		return nil, fmt.Errorf("init iam: %w", err)
+	}
+
+	return i, nil
+}
+
+// CreateAccount creates a new IAM account. Returns an error if the account
+// already exists.
+func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		_, ok := conf.AccessAccounts[account.Access]
+		if ok {
+			return nil, fmt.Errorf("account already exists")
+		}
+		conf.AccessAccounts[account.Access] = account
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
+// GetUserAccount retrieves account info for the requested user. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	conf, err := s.getIAM()
+	if err != nil {
+		return Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+// DeleteUserAccount deletes the specified user account. Does not check if
+// account exists.
+func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		delete(conf.AccessAccounts, access)
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
+// ListUserAccounts lists all the user accounts stored.
+func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getIAM()
+	if err != nil {
+		return []Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// Shutdown graceful termination of service
+func (s *IAMServiceInternal) Shutdown() error {
+	return nil
+}
+
+const (
+	iamMode = 0600
+)
+
+func (s *IAMServiceInternal) initIAM() error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	_, err := os.ReadFile(fname)
+	if errors.Is(err, fs.ErrNotExist) {
+		b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
+		if err != nil {
+			return fmt.Errorf("marshal default iam: %w", err)
+		}
+		err = os.WriteFile(fname, b, iamMode)
+		if err != nil {
+			return fmt.Errorf("write default iam: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
+	b, err := s.readIAMData()
+	if err != nil {
+		return iAMConfig{}, err
+	}
+
+	return parseIAM(b)
+}
+
+func parseIAM(b []byte) (iAMConfig, error) {
+	var conf iAMConfig
+	if err := json.Unmarshal(b, &conf); err != nil {
+		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
+	}
+
+	return conf, nil
+}
+
+const (
+	backoff  = 100 * time.Millisecond
+	maxretry = 300
+)
+
+func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
+	// We are going to be racing with other running gateways without any
+	// coordination. So we might find the file does not exist at times.
+	// For this case we need to retry for a while assuming the other gateway
+	// will eventually write the file. If it doesn't after the max retries,
+	// then we will return the error.
+
+	retries := 0
+
+	for {
+		b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+			return nil, fmt.Errorf("read iam file: %w", err)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		return b, nil
+	}
+}
+
+func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
+	// We are going to be racing with other running gateways without any
+	// coordination. So the strategy here is to read the current file data.
+	// If the file doesn't exist, then we assume someone else is currently
+	// updating the file. So we just need to keep retrying. We also need
+	// to make sure the data is consistent within a single update. So racing
+	// writes to a file would possibly leave this in some invalid state.
+	// We can get atomic updates with rename. If we read the data, update
+	// the data, write to a temp file, then rename the tempfile back to the
+	// data file. This should always result in a complete data image.
+
+	// There is at least one unsolved failure mode here.
+	// If a gateway removes the data file and then crashes, all other
+	// gateways will retry forever thinking that the original will eventually
+	// write the file.
+
+	retries := 0
+	fname := filepath.Join(s.dir, iamFile)
+
+	for {
+		b, err := os.ReadFile(fname)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+
+			// we have been unsuccessful trying to read the iam file
+			// so this must be the case where something happened and
+			// the file did not get updated successfully, and probably
+			// isn't going to be. The recovery procedure would be to
+			// copy the backup file into place of the original.
+			return fmt.Errorf("no iam file, needs backup recovery")
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("read iam file: %w", err)
+		}
+
+		// reset retries on successful read
+		retries = 0
+
+		err = os.Remove(fname)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			time.Sleep(backoff)
+			continue
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("remove old iam file: %w", err)
+		}
+
+		// save copy of data
+		datacopy := make([]byte, len(b))
+		copy(datacopy, b)
+
+		// make a backup copy in case we crash before update
+		// this is after remove, so there is a small window something
+		// can go wrong, but the remove should barrier other gateways
+		// from trying to write backup at the same time. Only one
+		// gateway will successfully remove the file.
+		os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
+
+		b, err = update(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return fmt.Errorf("update iam data: %w", err)
+		}
+
+		err = s.writeTempFile(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return err
+		}
+
+		break
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) writeTempFile(b []byte) error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	f, err := os.CreateTemp(s.dir, iamFile)
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	_, err = f.Write(b)
+	if err != nil {
+		return fmt.Errorf("write temp file: %w", err)
+	}
+
+	err = os.Rename(f.Name(), fname)
+	if err != nil {
+		return fmt.Errorf("rename temp file: %w", err)
+	}
+
+	return nil
+}

auth/iam_ldap.go

@@ -0,0 +1,133 @@
+package auth
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-ldap/ldap/v3"
+)
+
+type LdapIAMService struct {
+	conn       *ldap.Conn
+	queryBase  string
+	objClasses []string
+	accessAtr  string
+	secretAtr  string
+	roleAtr    string
+}
+
+var _ IAMService = &LdapIAMService{}
+
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+		return nil, fmt.Errorf("required parameters list not fully provided")
+	}
+	conn, err := ldap.Dial("tcp", url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
+	}
+
+	err = conn.Bind(bindDN, pass)
+	if err != nil {
+		return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
+	}
+	return &LdapIAMService{
+		conn:       conn,
+		queryBase:  queryBase,
+		objClasses: strings.Split(objClasses, ","),
+		accessAtr:  accAtr,
+		secretAtr:  secAtr,
+		roleAtr:    roleAtr,
+	}, nil
+}
+
+func (ld *LdapIAMService) CreateAccount(account Account) error {
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry.Attribute("objectClass", ld.objClasses)
+	userEntry.Attribute(ld.accessAtr, []string{account.Access})
+	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
+	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
+
+	err := ld.conn.Add(userEntry)
+	if err != nil {
+		return fmt.Errorf("error adding an entry: %w", err)
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	result, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return Account{}, err
+	}
+
+	entry := result.Entries[0]
+	return Account{
+		Access: entry.GetAttributeValue(ld.accessAtr),
+		Secret: entry.GetAttributeValue(ld.secretAtr),
+		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
+	}, nil
+}
+
+func (ld *LdapIAMService) DeleteUserAccount(access string) error {
+	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+
+	err := ld.conn.Del(delReq)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
+	searchFilter := ""
+	for _, el := range ld.objClasses {
+		searchFilter += fmt.Sprintf("(objectClass=%v)", el)
+	}
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(&%v)", searchFilter),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	resp, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	result := []Account{}
+	for _, el := range resp.Entries {
+		result = append(result, Account{
+			Access: el.GetAttributeValue(ld.accessAtr),
+			Secret: el.GetAttributeValue(ld.secretAtr),
+			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
+		})
+	}
+
+	return result, nil
+}
+
+// Shutdown graceful termination of service
+func (ld *LdapIAMService) Shutdown() error {
+	return ld.conn.Close()
+}

auth/iam_s3_object.go

@@ -0,0 +1,263 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+)
+
+// IAMServiceS3 stores user accounts in an S3 object
+// The endpoint, credentials, bucket, and region are provided
+// from cli configuration.
+// The object format and name is the same as the internal IAM service:
+// coming from iAMConfig and iamFile in iam_internal.
+
+type IAMServiceS3 struct {
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
+	sslSkipVerify bool
+	debug         bool
+	client        *s3.Client
+}
+
+var _ IAMService = &IAMServiceS3{}
+
+func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+	if access == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service access key")
+	}
+	if secret == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service secret key")
+	}
+	if region == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service region")
+	}
+	if bucket == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service bucket")
+	}
+	if endpoint == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service endpoint")
+	}
+
+	i := &IAMServiceS3{
+		access:        access,
+		secret:        secret,
+		region:        region,
+		bucket:        bucket,
+		endpoint:      endpoint,
+		sslSkipVerify: sslSkipVerify,
+		debug:         debug,
+	}
+
+	cfg, err := i.getConfig()
+	if err != nil {
+		return nil, fmt.Errorf("init s3 IAM: %v", err)
+	}
+
+	i.client = s3.NewFromConfig(cfg)
+	return i, nil
+}
+
+func (s *IAMServiceS3) CreateAccount(account Account) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[account.Access]
+	if ok {
+		return fmt.Errorf("account already exists")
+	}
+	conf.AccessAccounts[account.Access] = account
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[access]
+	if !ok {
+		return fmt.Errorf("account does not exist")
+	}
+	delete(conf.AccessAccounts, access)
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.region,
+		HostnameImmutable: true,
+	}, nil
+}
+
+func (s *IAMServiceS3) Shutdown() error {
+	return nil
+}
+
+func (s *IAMServiceS3) getConfig() (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.region),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(context.Background(), opts...)
+}
+
+func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
+	obj := iamFile
+
+	out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
+		Bucket: &s.bucket,
+		Key:    &obj,
+	})
+	if err != nil {
+		// if the error is object not exists,
+		// init empty accounts stuct and return that
+		var nsk *types.NoSuchKey
+		if errors.As(err, &nsk) {
+			return iAMConfig{}, nil
+		}
+		var apiErr smithy.APIError
+		if errors.As(err, &apiErr) {
+			if apiErr.ErrorCode() == "NotFound" {
+				return iAMConfig{}, nil
+			}
+		}
+
+		// all other errors, return the error
+		return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
+	}
+
+	defer out.Body.Close()
+
+	b, err := io.ReadAll(out.Body)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
+	}
+
+	conf, err := parseIAM(b)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
+	}
+
+	return conf, nil
+}
+
+func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
+	b, err := json.Marshal(conf)
+	if err != nil {
+		return fmt.Errorf("failed to serialize iam: %w", err)
+	}
+
+	obj := iamFile
+	uploader := manager.NewUploader(s.client)
+	upinfo := &s3.PutObjectInput{
+		Body:   bytes.NewReader(b),
+		Bucket: &s.bucket,
+		Key:    &obj,
+	}
+	_, err = uploader.Upload(context.Background(), upinfo)
+	if err != nil {
+		return fmt.Errorf("store accounts in %v: %w", iamFile, err)
+	}
+
+	return nil
+}

auth/iam_single.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+)
+
+// IAMServiceSingle manages the single tenant (root-only) IAM service
+type IAMServiceSingle struct{}
+
+var _ IAMService = &IAMServiceSingle{}
+
+var ErrNotSupported = errors.New("method is not supported")
+
+// CreateAccount not valid in single tenant mode
+func (IAMServiceSingle) CreateAccount(account Account) error {
+	return ErrNotSupported
+}
+
+// GetUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	return Account{}, ErrNotSupported
+}
+
+// DeleteUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) DeleteUserAccount(access string) error {
+	return ErrNotSupported
+}
+
+// ListUserAccounts no accounts in single tenant mode
+func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
+	return []Account{}, nil
+}
+
+// Shutdown graceful termination of service
+func (IAMServiceSingle) Shutdown() error {
+	return nil
+}

aws/LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

aws/NOTICE.txt

@@ -0,0 +1,3 @@
+AWS SDK for Go
+Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2014-2015 Stripe, Inc.

aws/README.md

@@ -0,0 +1,11 @@
+# AWS SDK Go v2
+
+This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License.
+
+## Description
+
+The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements.
+
+## License
+
+The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information.

aws/internal/auth/auth.go

@@ -0,0 +1,45 @@
+package auth
+
+import (
+	"github.com/aws/smithy-go/auth"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+)
+
+// HTTPAuthScheme is the SDK's internal implementation of smithyhttp.AuthScheme
+// for pre-existing implementations where the signer was added to client
+// config. SDK clients will key off of this type and ensure per-operation
+// updates to those signers persist on the scheme itself.
+type HTTPAuthScheme struct {
+	schemeID string
+	signer   smithyhttp.Signer
+}
+
+var _ smithyhttp.AuthScheme = (*HTTPAuthScheme)(nil)
+
+// NewHTTPAuthScheme returns an auth scheme instance with the given config.
+func NewHTTPAuthScheme(schemeID string, signer smithyhttp.Signer) *HTTPAuthScheme {
+	return &HTTPAuthScheme{
+		schemeID: schemeID,
+		signer:   signer,
+	}
+}
+
+// SchemeID identifies the auth scheme.
+func (s *HTTPAuthScheme) SchemeID() string {
+	return s.schemeID
+}
+
+// IdentityResolver gets the identity resolver for the auth scheme.
+func (s *HTTPAuthScheme) IdentityResolver(o auth.IdentityResolverOptions) auth.IdentityResolver {
+	return o.GetIdentityResolver(s.schemeID)
+}
+
+// Signer gets the signer for the auth scheme.
+func (s *HTTPAuthScheme) Signer() smithyhttp.Signer {
+	return s.signer
+}
+
+// WithSigner returns a new instance of the auth scheme with the updated signer.
+func (s *HTTPAuthScheme) WithSigner(signer smithyhttp.Signer) *HTTPAuthScheme {
+	return NewHTTPAuthScheme(s.schemeID, signer)
+}

aws/internal/auth/scheme.go

@@ -0,0 +1,191 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+
+	smithy "github.com/aws/smithy-go"
+	"github.com/aws/smithy-go/middleware"
+)
+
+// SigV4 is a constant representing
+// Authentication Scheme Signature Version 4
+const SigV4 = "sigv4"
+
+// SigV4A is a constant representing
+// Authentication Scheme Signature Version 4A
+const SigV4A = "sigv4a"
+
+// SigV4S3Express identifies the S3 S3Express auth scheme.
+const SigV4S3Express = "sigv4-s3express"
+
+// None is a constant representing the
+// None Authentication Scheme
+const None = "none"
+
+// SupportedSchemes is a data structure
+// that indicates the list of supported AWS
+// authentication schemes
+var SupportedSchemes = map[string]bool{
+	SigV4:          true,
+	SigV4A:         true,
+	SigV4S3Express: true,
+	None:           true,
+}
+
+// AuthenticationScheme is a representation of
+// AWS authentication schemes
+type AuthenticationScheme interface {
+	isAuthenticationScheme()
+}
+
+// AuthenticationSchemeV4 is a AWS SigV4 representation
+type AuthenticationSchemeV4 struct {
+	Name                  string
+	SigningName           *string
+	SigningRegion         *string
+	DisableDoubleEncoding *bool
+}
+
+func (a *AuthenticationSchemeV4) isAuthenticationScheme() {}
+
+// AuthenticationSchemeV4A is a AWS SigV4A representation
+type AuthenticationSchemeV4A struct {
+	Name                  string
+	SigningName           *string
+	SigningRegionSet      []string
+	DisableDoubleEncoding *bool
+}
+
+func (a *AuthenticationSchemeV4A) isAuthenticationScheme() {}
+
+// AuthenticationSchemeNone is a representation for the none auth scheme
+type AuthenticationSchemeNone struct{}
+
+func (a *AuthenticationSchemeNone) isAuthenticationScheme() {}
+
+// NoAuthenticationSchemesFoundError is used in signaling
+// that no authentication schemes have been specified.
+type NoAuthenticationSchemesFoundError struct{}
+
+func (e *NoAuthenticationSchemesFoundError) Error() string {
+	return fmt.Sprint("No authentication schemes specified.")
+}
+
+// UnSupportedAuthenticationSchemeSpecifiedError is used in
+// signaling that only unsupported authentication schemes
+// were specified.
+type UnSupportedAuthenticationSchemeSpecifiedError struct {
+	UnsupportedSchemes []string
+}
+
+func (e *UnSupportedAuthenticationSchemeSpecifiedError) Error() string {
+	return fmt.Sprint("Unsupported authentication scheme specified.")
+}
+
+// GetAuthenticationSchemes extracts the relevant authentication scheme data
+// into a custom strongly typed Go data structure.
+func GetAuthenticationSchemes(p *smithy.Properties) ([]AuthenticationScheme, error) {
+	var result []AuthenticationScheme
+	if !p.Has("authSchemes") {
+		return nil, &NoAuthenticationSchemesFoundError{}
+	}
+
+	authSchemes, _ := p.Get("authSchemes").([]interface{})
+
+	var unsupportedSchemes []string
+	for _, scheme := range authSchemes {
+		authScheme, _ := scheme.(map[string]interface{})
+
+		version := authScheme["name"].(string)
+		switch version {
+		case SigV4, SigV4S3Express:
+			v4Scheme := AuthenticationSchemeV4{
+				Name:                  version,
+				SigningName:           getSigningName(authScheme),
+				SigningRegion:         getSigningRegion(authScheme),
+				DisableDoubleEncoding: getDisableDoubleEncoding(authScheme),
+			}
+			result = append(result, AuthenticationScheme(&v4Scheme))
+		case SigV4A:
+			v4aScheme := AuthenticationSchemeV4A{
+				Name:                  SigV4A,
+				SigningName:           getSigningName(authScheme),
+				SigningRegionSet:      getSigningRegionSet(authScheme),
+				DisableDoubleEncoding: getDisableDoubleEncoding(authScheme),
+			}
+			result = append(result, AuthenticationScheme(&v4aScheme))
+		case None:
+			noneScheme := AuthenticationSchemeNone{}
+			result = append(result, AuthenticationScheme(&noneScheme))
+		default:
+			unsupportedSchemes = append(unsupportedSchemes, authScheme["name"].(string))
+			continue
+		}
+	}
+
+	if len(result) == 0 {
+		return nil, &UnSupportedAuthenticationSchemeSpecifiedError{
+			UnsupportedSchemes: unsupportedSchemes,
+		}
+	}
+
+	return result, nil
+}
+
+type disableDoubleEncoding struct{}
+
+// SetDisableDoubleEncoding sets or modifies the disable double encoding option
+// on the context.
+//
+// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
+// to clear all stack values.
+func SetDisableDoubleEncoding(ctx context.Context, value bool) context.Context {
+	return middleware.WithStackValue(ctx, disableDoubleEncoding{}, value)
+}
+
+// GetDisableDoubleEncoding retrieves the disable double encoding option
+// from the context.
+//
+// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
+// to clear all stack values.
+func GetDisableDoubleEncoding(ctx context.Context) (value bool, ok bool) {
+	value, ok = middleware.GetStackValue(ctx, disableDoubleEncoding{}).(bool)
+	return value, ok
+}
+
+func getSigningName(authScheme map[string]interface{}) *string {
+	signingName, ok := authScheme["signingName"].(string)
+	if !ok || signingName == "" {
+		return nil
+	}
+	return &signingName
+}
+
+func getSigningRegionSet(authScheme map[string]interface{}) []string {
+	untypedSigningRegionSet, ok := authScheme["signingRegionSet"].([]interface{})
+	if !ok {
+		return nil
+	}
+	signingRegionSet := []string{}
+	for _, item := range untypedSigningRegionSet {
+		signingRegionSet = append(signingRegionSet, item.(string))
+	}
+	return signingRegionSet
+}
+
+func getSigningRegion(authScheme map[string]interface{}) *string {
+	signingRegion, ok := authScheme["signingRegion"].(string)
+	if !ok || signingRegion == "" {
+		return nil
+	}
+	return &signingRegion
+}
+
+func getDisableDoubleEncoding(authScheme map[string]interface{}) *bool {
+	disableDoubleEncoding, ok := authScheme["disableDoubleEncoding"].(bool)
+	if !ok {
+		return nil
+	}
+	return &disableDoubleEncoding
+}

aws/internal/auth/scheme_test.go

@@ -0,0 +1,101 @@
+package auth
+
+import (
+	"testing"
+
+	smithy "github.com/aws/smithy-go"
+)
+
+func TestV4(t *testing.T) {
+
+	propsV4 := smithy.Properties{}
+
+	propsV4.Set("authSchemes", interface{}([]interface{}{
+		map[string]interface{}{
+			"disableDoubleEncoding": true,
+			"name":                  "sigv4",
+			"signingName":           "s3",
+			"signingRegion":         "us-west-2",
+		},
+	}))
+
+	result, err := GetAuthenticationSchemes(&propsV4)
+	if err != nil {
+		t.Fatalf("Did not expect error, got %v", err)
+	}
+
+	_, ok := result[0].(AuthenticationScheme)
+	if !ok {
+		t.Fatalf("Did not get expected AuthenticationScheme. %v", result[0])
+	}
+
+	v4Scheme, ok := result[0].(*AuthenticationSchemeV4)
+	if !ok {
+		t.Fatalf("Did not get expected AuthenticationSchemeV4. %v", result[0])
+	}
+
+	if v4Scheme.Name != "sigv4" {
+		t.Fatalf("Did not get expected AuthenticationSchemeV4 signer version name")
+	}
+
+}
+
+func TestV4A(t *testing.T) {
+
+	propsV4A := smithy.Properties{}
+
+	propsV4A.Set("authSchemes", []interface{}{
+		map[string]interface{}{
+			"disableDoubleEncoding": true,
+			"name":                  "sigv4a",
+			"signingName":           "s3",
+			"signingRegionSet":      []string{"*"},
+		},
+	})
+
+	result, err := GetAuthenticationSchemes(&propsV4A)
+	if err != nil {
+		t.Fatalf("Did not expect error, got %v", err)
+	}
+
+	_, ok := result[0].(AuthenticationScheme)
+	if !ok {
+		t.Fatalf("Did not get expected AuthenticationScheme. %v", result[0])
+	}
+
+	v4AScheme, ok := result[0].(*AuthenticationSchemeV4A)
+	if !ok {
+		t.Fatalf("Did not get expected AuthenticationSchemeV4A. %v", result[0])
+	}
+
+	if v4AScheme.Name != "sigv4a" {
+		t.Fatalf("Did not get expected AuthenticationSchemeV4A signer version name")
+	}
+
+}
+
+func TestV4S3Express(t *testing.T) {
+	props := smithy.Properties{}
+	props.Set("authSchemes", []interface{}{
+		map[string]interface{}{
+			"name":                  SigV4S3Express,
+			"signingName":           "s3",
+			"signingRegion":         "us-east-1",
+			"disableDoubleEncoding": true,
+		},
+	})
+
+	result, err := GetAuthenticationSchemes(&props)
+	if err != nil {
+		t.Fatalf("Did not expect error, got %v", err)
+	}
+
+	scheme, ok := result[0].(*AuthenticationSchemeV4)
+	if !ok {
+		t.Fatalf("Did not get expected AuthenticationSchemeV4. %v", result[0])
+	}
+
+	if scheme.Name != SigV4S3Express {
+		t.Fatalf("expected %s, got %s", SigV4S3Express, scheme.Name)
+	}
+}

aws/internal/auth/smithy/bearer_token_adapter.go

@@ -0,0 +1,43 @@
+package smithy
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/aws/smithy-go"
+	"github.com/aws/smithy-go/auth"
+	"github.com/aws/smithy-go/auth/bearer"
+)
+
+// BearerTokenAdapter adapts smithy bearer.Token to smithy auth.Identity.
+type BearerTokenAdapter struct {
+	Token bearer.Token
+}
+
+var _ auth.Identity = (*BearerTokenAdapter)(nil)
+
+// Expiration returns the time of expiration for the token.
+func (v *BearerTokenAdapter) Expiration() time.Time {
+	return v.Token.Expires
+}
+
+// BearerTokenProviderAdapter adapts smithy bearer.TokenProvider to smithy
+// auth.IdentityResolver.
+type BearerTokenProviderAdapter struct {
+	Provider bearer.TokenProvider
+}
+
+var _ (auth.IdentityResolver) = (*BearerTokenProviderAdapter)(nil)
+
+// GetIdentity retrieves a bearer token using the underlying provider.
+func (v *BearerTokenProviderAdapter) GetIdentity(ctx context.Context, _ smithy.Properties) (
+	auth.Identity, error,
+) {
+	token, err := v.Provider.RetrieveBearerToken(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("get token: %w", err)
+	}
+
+	return &BearerTokenAdapter{Token: token}, nil
+}

aws/internal/auth/smithy/bearer_token_signer_adapter.go

@@ -0,0 +1,35 @@
+package smithy
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/smithy-go"
+	"github.com/aws/smithy-go/auth"
+	"github.com/aws/smithy-go/auth/bearer"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+)
+
+// BearerTokenSignerAdapter adapts smithy bearer.Signer to smithy http
+// auth.Signer.
+type BearerTokenSignerAdapter struct {
+	Signer bearer.Signer
+}
+
+var _ (smithyhttp.Signer) = (*BearerTokenSignerAdapter)(nil)
+
+// SignRequest signs the request with the provided bearer token.
+func (v *BearerTokenSignerAdapter) SignRequest(ctx context.Context, r *smithyhttp.Request, identity auth.Identity, _ smithy.Properties) error {
+	ca, ok := identity.(*BearerTokenAdapter)
+	if !ok {
+		return fmt.Errorf("unexpected identity type: %T", identity)
+	}
+
+	signed, err := v.Signer.SignWithBearerToken(ctx, ca.Token, r)
+	if err != nil {
+		return fmt.Errorf("sign request: %w", err)
+	}
+
+	*r = *signed.(*smithyhttp.Request)
+	return nil
+}

aws/internal/auth/smithy/credentials_adapter.go

@@ -0,0 +1,46 @@
+package smithy
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go"
+	"github.com/aws/smithy-go/auth"
+)
+
+// CredentialsAdapter adapts aws.Credentials to auth.Identity.
+type CredentialsAdapter struct {
+	Credentials aws.Credentials
+}
+
+var _ auth.Identity = (*CredentialsAdapter)(nil)
+
+// Expiration returns the time of expiration for the credentials.
+func (v *CredentialsAdapter) Expiration() time.Time {
+	return v.Credentials.Expires
+}
+
+// CredentialsProviderAdapter adapts aws.CredentialsProvider to auth.IdentityResolver.
+type CredentialsProviderAdapter struct {
+	Provider aws.CredentialsProvider
+}
+
+var _ (auth.IdentityResolver) = (*CredentialsProviderAdapter)(nil)
+
+// GetIdentity retrieves AWS credentials using the underlying provider.
+func (v *CredentialsProviderAdapter) GetIdentity(ctx context.Context, _ smithy.Properties) (
+	auth.Identity, error,
+) {
+	if v.Provider == nil {
+		return &CredentialsAdapter{Credentials: aws.Credentials{}}, nil
+	}
+
+	creds, err := v.Provider.Retrieve(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("get credentials: %w", err)
+	}
+
+	return &CredentialsAdapter{Credentials: creds}, nil
+}

aws/internal/auth/smithy/smithy.go

@@ -0,0 +1,2 @@
+// Package smithy adapts concrete AWS auth and signing types to the generic smithy versions.
+package smithy

aws/internal/auth/smithy/v4signer_adapter.go

@@ -0,0 +1,53 @@
+package smithy
+
+import (
+	"context"
+	"fmt"
+
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go"
+	"github.com/aws/smithy-go/auth"
+	"github.com/aws/smithy-go/logging"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/versity/versitygw/aws/internal/sdk"
+)
+
+// V4SignerAdapter adapts v4.HTTPSigner to smithy http.Signer.
+type V4SignerAdapter struct {
+	Signer     v4.HTTPSigner
+	Logger     logging.Logger
+	LogSigning bool
+}
+
+var _ (smithyhttp.Signer) = (*V4SignerAdapter)(nil)
+
+// SignRequest signs the request with the provided identity.
+func (v *V4SignerAdapter) SignRequest(ctx context.Context, r *smithyhttp.Request, identity auth.Identity, props smithy.Properties) error {
+	ca, ok := identity.(*CredentialsAdapter)
+	if !ok {
+		return fmt.Errorf("unexpected identity type: %T", identity)
+	}
+
+	name, ok := smithyhttp.GetSigV4SigningName(&props)
+	if !ok {
+		return fmt.Errorf("sigv4 signing name is required")
+	}
+
+	region, ok := smithyhttp.GetSigV4SigningRegion(&props)
+	if !ok {
+		return fmt.Errorf("sigv4 signing region is required")
+	}
+
+	hash := v4.GetPayloadHash(ctx)
+	err := v.Signer.SignHTTP(ctx, ca.Credentials, r.Request, hash, name, region, sdk.NowTime(), func(o *v4.SignerOptions) {
+		o.DisableURIPathEscaping, _ = smithyhttp.GetDisableDoubleEncoding(&props)
+
+		o.Logger = v.Logger
+		o.LogSigning = v.LogSigning
+	})
+	if err != nil {
+		return fmt.Errorf("sign http: %w", err)
+	}
+
+	return nil
+}

aws/internal/awstesting/assert.go

@@ -0,0 +1,222 @@
+package awstesting
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"net/url"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+)
+
+// Match is a testing helper to test for testing error by comparing expected
+// with a regular expression.
+func Match(t *testing.T, regex, expected string) {
+	t.Helper()
+
+	if !regexp.MustCompile(regex).Match([]byte(expected)) {
+		t.Errorf("%q\n\tdoes not match /%s/", expected, regex)
+	}
+}
+
+// AssertURL verifies the expected URL is matches the actual.
+func AssertURL(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
+	t.Helper()
+
+	expectURL, err := url.Parse(expect)
+	if err != nil {
+		t.Errorf(errMsg("unable to parse expected URL", err, msgAndArgs))
+		return false
+	}
+	actualURL, err := url.Parse(actual)
+	if err != nil {
+		t.Errorf(errMsg("unable to parse actual URL", err, msgAndArgs))
+		return false
+	}
+
+	equal(t, expectURL.Host, actualURL.Host, msgAndArgs...)
+	equal(t, expectURL.Scheme, actualURL.Scheme, msgAndArgs...)
+	equal(t, expectURL.Path, actualURL.Path, msgAndArgs...)
+
+	return AssertQuery(t, expectURL.Query().Encode(), actualURL.Query().Encode(), msgAndArgs...)
+}
+
+var queryMapKey = regexp.MustCompile(`(.*?)\.[0-9]+\.key`)
+
+// AssertQuery verifies the expect HTTP query string matches the actual.
+func AssertQuery(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
+	t.Helper()
+
+	expectQ, err := url.ParseQuery(expect)
+	if err != nil {
+		t.Errorf(errMsg("unable to parse expected Query", err, msgAndArgs))
+		return false
+	}
+	actualQ, err := url.ParseQuery(actual)
+	if err != nil {
+		t.Errorf(errMsg("unable to parse actual Query", err, msgAndArgs))
+		return false
+	}
+
+	// Make sure the keys are the same
+	if !equal(t, queryValueKeys(expectQ), queryValueKeys(actualQ), msgAndArgs...) {
+		return false
+	}
+
+	keys := map[string][]string{}
+	for key, v := range expectQ {
+		if queryMapKey.Match([]byte(key)) {
+			submatch := queryMapKey.FindStringSubmatch(key)
+			keys[submatch[1]] = append(keys[submatch[1]], v...)
+		}
+	}
+
+	for k, v := range keys {
+		// clear all keys that have prefix
+		for key := range expectQ {
+			if strings.HasPrefix(key, k) {
+				delete(expectQ, key)
+			}
+		}
+
+		sort.Strings(v)
+		for i, value := range v {
+			expectQ[fmt.Sprintf("%s.%d.key", k, i+1)] = []string{value}
+		}
+	}
+
+	for k, expectQVals := range expectQ {
+		sort.Strings(expectQVals)
+		actualQVals := actualQ[k]
+		sort.Strings(actualQVals)
+		if !equal(t, expectQVals, actualQVals, msgAndArgs...) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// AssertJSON verifies that the expect json string matches the actual.
+func AssertJSON(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
+	t.Helper()
+
+	expectVal := map[string]interface{}{}
+	if err := json.Unmarshal([]byte(expect), &expectVal); err != nil {
+		t.Errorf(errMsg("unable to parse expected JSON", err, msgAndArgs...))
+		return false
+	}
+
+	actualVal := map[string]interface{}{}
+	if err := json.Unmarshal([]byte(actual), &actualVal); err != nil {
+		t.Errorf(errMsg("unable to parse actual JSON", err, msgAndArgs...))
+		return false
+	}
+
+	return equal(t, expectVal, actualVal, msgAndArgs...)
+}
+
+// AssertXML verifies that the expect xml string matches the actual.
+func AssertXML(t *testing.T, expect, actual string, container interface{}, msgAndArgs ...interface{}) bool {
+	expectVal := container
+	if err := xml.Unmarshal([]byte(expect), &expectVal); err != nil {
+		t.Errorf(errMsg("unable to parse expected XML", err, msgAndArgs...))
+	}
+
+	actualVal := container
+	if err := xml.Unmarshal([]byte(actual), &actualVal); err != nil {
+		t.Errorf(errMsg("unable to parse actual XML", err, msgAndArgs...))
+	}
+	return equal(t, expectVal, actualVal, msgAndArgs...)
+}
+
+// DidPanic returns if the function paniced and returns true if the function paniced.
+func DidPanic(fn func()) (bool, interface{}) {
+	var paniced bool
+	var msg interface{}
+	func() {
+		defer func() {
+			if msg = recover(); msg != nil {
+				paniced = true
+			}
+		}()
+		fn()
+	}()
+
+	return paniced, msg
+}
+
+// objectsAreEqual determines if two objects are considered equal.
+//
+// This function does no assertion of any kind.
+//
+// Based on github.com/stretchr/testify/assert.ObjectsAreEqual
+// Copied locally to prevent non-test build dependencies on testify
+func objectsAreEqual(expected, actual interface{}) bool {
+	if expected == nil || actual == nil {
+		return expected == actual
+	}
+
+	return reflect.DeepEqual(expected, actual)
+}
+
+// Equal asserts that two objects are equal.
+//
+//	assert.Equal(t, 123, 123, "123 and 123 should be equal")
+//
+// Returns whether the assertion was successful (true) or not (false).
+//
+// Based on github.com/stretchr/testify/assert.Equal
+// Copied locally to prevent non-test build dependencies on testify
+func equal(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) bool {
+	t.Helper()
+
+	if !objectsAreEqual(expected, actual) {
+		t.Errorf("%s\n%s", messageFromMsgAndArgs(msgAndArgs),
+			SprintExpectActual(expected, actual))
+		return false
+	}
+
+	return true
+}
+
+func errMsg(baseMsg string, err error, msgAndArgs ...interface{}) string {
+	message := messageFromMsgAndArgs(msgAndArgs)
+	if message != "" {
+		message += ", "
+	}
+	return fmt.Sprintf("%s%s, %v", message, baseMsg, err)
+}
+
+// Based on github.com/stretchr/testify/assert.messageFromMsgAndArgs
+// Copied locally to prevent non-test build dependencies on testify
+func messageFromMsgAndArgs(msgAndArgs []interface{}) string {
+	if len(msgAndArgs) == 0 || msgAndArgs == nil {
+		return ""
+	}
+	if len(msgAndArgs) == 1 {
+		return msgAndArgs[0].(string)
+	}
+	if len(msgAndArgs) > 1 {
+		return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...)
+	}
+	return ""
+}
+
+func queryValueKeys(v url.Values) []string {
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// SprintExpectActual returns a string for test failure cases when the actual
+// value is not the same as the expected.
+func SprintExpectActual(expect, actual interface{}) string {
+	return fmt.Sprintf("expect: %+v\nactual: %+v\n", expect, actual)
+}

aws/internal/awstesting/assert_test.go

@@ -0,0 +1,89 @@
+package awstesting_test
+
+import (
+	"encoding/xml"
+	"testing"
+
+	"github.com/versity/versitygw/aws/internal/awstesting"
+)
+
+func TestAssertJSON(t *testing.T) {
+	cases := []struct {
+		e, a    string
+		asserts bool
+	}{
+		{
+			e:       `{"RecursiveStruct":{"RecursiveMap":{"foo":{"NoRecurse":"foo"},"bar":{"NoRecurse":"bar"}}}}`,
+			a:       `{"RecursiveStruct":{"RecursiveMap":{"bar":{"NoRecurse":"bar"},"foo":{"NoRecurse":"foo"}}}}`,
+			asserts: true,
+		},
+	}
+
+	for i, c := range cases {
+		mockT := &testing.T{}
+		if awstesting.AssertJSON(mockT, c.e, c.a) != c.asserts {
+			t.Error("Assert JSON result was not expected.", i)
+		}
+	}
+}
+
+func TestAssertXML(t *testing.T) {
+	cases := []struct {
+		e, a      string
+		asserts   bool
+		container struct {
+			XMLName         xml.Name `xml:"OperationRequest"`
+			NS              string   `xml:"xmlns,attr"`
+			RecursiveStruct struct {
+				RecursiveMap struct {
+					Entries []struct {
+						XMLName xml.Name `xml:"entries"`
+						Key     string   `xml:"key"`
+						Value   struct {
+							XMLName   xml.Name `xml:"value"`
+							NoRecurse string
+						}
+					}
+				}
+			}
+		}
+	}{
+		{
+			e:       `<OperationRequest xmlns="https://foo/"><RecursiveStruct xmlns="https://foo/"><RecursiveMap xmlns="https://foo/"><entry xmlns="https://foo/"><key xmlns="https://foo/">foo</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">foo</NoRecurse></value></entry><entry xmlns="https://foo/"><key xmlns="https://foo/">bar</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">bar</NoRecurse></value></entry></RecursiveMap></RecursiveStruct></OperationRequest>`,
+			a:       `<OperationRequest xmlns="https://foo/"><RecursiveStruct xmlns="https://foo/"><RecursiveMap xmlns="https://foo/"><entry xmlns="https://foo/"><key xmlns="https://foo/">bar</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">bar</NoRecurse></value></entry><entry xmlns="https://foo/"><key xmlns="https://foo/">foo</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">foo</NoRecurse></value></entry></RecursiveMap></RecursiveStruct></OperationRequest>`,
+			asserts: true,
+		},
+	}
+
+	for i, c := range cases {
+		//		mockT := &testing.T{}
+		if awstesting.AssertXML(t, c.e, c.a, c.container) != c.asserts {
+			t.Error("Assert XML result was not expected.", i)
+		}
+	}
+}
+
+func TestAssertQuery(t *testing.T) {
+	cases := []struct {
+		e, a    string
+		asserts bool
+	}{
+		{
+			e:       `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
+			a:       `Action=OperationName&Version=2014-01-01&Foo=val2&Bar=val3`,
+			asserts: false,
+		},
+		{
+			e:       `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
+			a:       `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
+			asserts: true,
+		},
+	}
+
+	for i, c := range cases {
+		mockT := &testing.T{}
+		if awstesting.AssertQuery(mockT, c.e, c.a) != c.asserts {
+			t.Error("Assert Query result was not expected.", i)
+		}
+	}
+}

aws/internal/awstesting/certificate_utils.go

@@ -0,0 +1,291 @@
+package awstesting
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"time"
+)
+
+var (
+	// TLSBundleCA is the CA PEM
+	TLSBundleCA []byte
+
+	// TLSBundleCert is the Server PEM
+	TLSBundleCert []byte
+
+	// TLSBundleKey is the Server private key PEM
+	TLSBundleKey []byte
+
+	// ClientTLSCert is the Client PEM
+	ClientTLSCert []byte
+
+	// ClientTLSKey is the Client private key PEM
+	ClientTLSKey []byte
+)
+
+func init() {
+	caPEM, _, caCert, caPrivKey, err := generateRootCA()
+	if err != nil {
+		panic("failed to generate testing root CA, " + err.Error())
+	}
+	TLSBundleCA = caPEM
+
+	serverCertPEM, serverCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey)
+	if err != nil {
+		panic("failed to generate testing server cert, " + err.Error())
+	}
+	TLSBundleCert = serverCertPEM
+	TLSBundleKey = serverCertPrivKeyPEM
+
+	clientCertPEM, clientCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey)
+	if err != nil {
+		panic("failed to generate testing client cert, " + err.Error())
+	}
+	ClientTLSCert = clientCertPEM
+	ClientTLSKey = clientCertPrivKeyPEM
+}
+
+func generateRootCA() (
+	caPEM, caPrivKeyPEM []byte, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, err error,
+) {
+	caCert = &x509.Certificate{
+		SerialNumber: big.NewInt(42),
+		Subject: pkix.Name{
+			Country:      []string{"US"},
+			Organization: []string{"AWS SDK for Go Test Certificate"},
+			CommonName:   "Test Root CA",
+		},
+		NotBefore: time.Now().Add(-time.Minute),
+		NotAfter:  time.Now().AddDate(1, 0, 0),
+		KeyUsage:  x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+		},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+
+	// Create CA private and public key
+	caPrivKey, err = rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("failed generate CA RSA key, %w", err)
+	}
+
+	// Create CA certificate
+	caBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, &caPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("failed generate CA certificate, %w", err)
+	}
+
+	// PEM encode CA certificate and private key
+	var caPEMBuf bytes.Buffer
+	pem.Encode(&caPEMBuf, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caBytes,
+	})
+
+	var caPrivKeyPEMBuf bytes.Buffer
+	pem.Encode(&caPrivKeyPEMBuf, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey),
+	})
+
+	return caPEMBuf.Bytes(), caPrivKeyPEMBuf.Bytes(), caCert, caPrivKey, nil
+}
+
+func generateLocalCert(parentCert *x509.Certificate, parentPrivKey *rsa.PrivateKey) (
+	certPEM, certPrivKeyPEM []byte, err error,
+) {
+	cert := &x509.Certificate{
+		SerialNumber: big.NewInt(42),
+		Subject: pkix.Name{
+			Country:      []string{"US"},
+			Organization: []string{"AWS SDK for Go Test Certificate"},
+			CommonName:   "Test Root CA",
+		},
+		IPAddresses: []net.IP{
+			net.IPv4(127, 0, 0, 1),
+			net.IPv6loopback,
+		},
+		NotBefore: time.Now().Add(-time.Minute),
+		NotAfter:  time.Now().AddDate(1, 0, 0),
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+		},
+		KeyUsage: x509.KeyUsageDigitalSignature,
+	}
+
+	// Create server private and public key
+	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to generate server RSA private key, %w", err)
+	}
+
+	// Create server certificate
+	certBytes, err := x509.CreateCertificate(rand.Reader, cert, parentCert, &certPrivKey.PublicKey, parentPrivKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to generate server certificate, %w", err)
+	}
+
+	// PEM encode certificate and private key
+	var certPEMBuf bytes.Buffer
+	pem.Encode(&certPEMBuf, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+
+	var certPrivKeyPEMBuf bytes.Buffer
+	pem.Encode(&certPrivKeyPEMBuf, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
+	})
+
+	return certPEMBuf.Bytes(), certPrivKeyPEMBuf.Bytes(), nil
+}
+
+// NewTLSClientCertServer creates a new HTTP test server initialize to require
+// HTTP clients authenticate with TLS client certificates.
+func NewTLSClientCertServer(handler http.Handler) (*httptest.Server, error) {
+	server := httptest.NewUnstartedServer(handler)
+
+	if server.TLS == nil {
+		server.TLS = &tls.Config{}
+	}
+	server.TLS.ClientAuth = tls.RequireAndVerifyClientCert
+
+	if server.TLS.ClientCAs == nil {
+		server.TLS.ClientCAs = x509.NewCertPool()
+	}
+	certPem := append(ClientTLSCert, ClientTLSKey...)
+	if ok := server.TLS.ClientCAs.AppendCertsFromPEM(certPem); !ok {
+		return nil, fmt.Errorf("failed to append client certs")
+	}
+
+	return server, nil
+}
+
+// CreateClientTLSCertFiles returns a set of temporary files for the client
+// certificate and key files.
+func CreateClientTLSCertFiles() (cert, key string, err error) {
+	cert, err = createTmpFile(ClientTLSCert)
+	if err != nil {
+		return "", "", err
+	}
+
+	key, err = createTmpFile(ClientTLSKey)
+	if err != nil {
+		return "", "", err
+	}
+
+	return cert, key, nil
+}
+
+func availableLocalAddr(ip string) (v string, err error) {
+	l, err := net.Listen("tcp", ip+":0")
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		closeErr := l.Close()
+		if err == nil {
+			err = closeErr
+		} else if closeErr != nil {
+			err = fmt.Errorf("ip listener close error: %v, original error: %w", closeErr, err)
+		}
+	}()
+
+	return l.Addr().String(), nil
+}
+
+// CreateTLSServer will create the TLS server on an open port using the
+// certificate and key. The address will be returned that the server is running on.
+func CreateTLSServer(cert, key string, mux *http.ServeMux) (string, error) {
+	addr, err := availableLocalAddr("127.0.0.1")
+	if err != nil {
+		return "", err
+	}
+
+	if mux == nil {
+		mux = http.NewServeMux()
+		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {})
+	}
+
+	go func() {
+		if err := http.ListenAndServeTLS(addr, cert, key, mux); err != nil {
+			panic(err)
+		}
+	}()
+
+	for i := 0; i < 60; i++ {
+		if _, err := http.Get("https://" + addr); err != nil && !strings.Contains(err.Error(), "connection refused") {
+			break
+		}
+
+		time.Sleep(1 * time.Second)
+	}
+
+	return "https://" + addr, nil
+}
+
+// CreateTLSBundleFiles returns the temporary filenames for the certificate
+// key, and CA PEM content. These files should be deleted when no longer
+// needed. CleanupTLSBundleFiles can be used for this cleanup.
+func CreateTLSBundleFiles() (cert, key, ca string, err error) {
+	cert, err = createTmpFile(TLSBundleCert)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	key, err = createTmpFile(TLSBundleKey)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	ca, err = createTmpFile(TLSBundleCA)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	return cert, key, ca, nil
+}
+
+// CleanupTLSBundleFiles takes variadic list of files to be deleted.
+func CleanupTLSBundleFiles(files ...string) error {
+	for _, file := range files {
+		if err := os.Remove(file); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func createTmpFile(b []byte) (string, error) {
+	bundleFile, err := ioutil.TempFile(os.TempDir(), "aws-sdk-go-session-test")
+	if err != nil {
+		return "", err
+	}
+
+	_, err = bundleFile.Write(b)
+	if err != nil {
+		return "", err
+	}
+
+	defer bundleFile.Close()
+	return bundleFile.Name(), nil
+}

aws/internal/awstesting/discard.go

@@ -0,0 +1,11 @@
+package awstesting
+
+// DiscardAt is an io.WriteAt that discards
+// the requested bytes to be written
+type DiscardAt struct{}
+
+// WriteAt discards the given []byte slice and returns len(p) bytes
+// as having been written at the given offset. It will never return an error.
+func (d DiscardAt) WriteAt(p []byte, off int64) (n int, err error) {
+	return len(p), nil
+}

aws/internal/awstesting/endless_reader.go

@@ -0,0 +1,12 @@
+package awstesting
+
+// EndlessReader is an io.Reader that will always return
+// that bytes have been read.
+type EndlessReader struct{}
+
+// Read will report that it has read len(p) bytes in p.
+// The content in the []byte will be unmodified.
+// This will never return an error.
+func (e EndlessReader) Read(p []byte) (int, error) {
+	return len(p), nil
+}

aws/internal/awstesting/sandbox/Dockerfile.golang-tip

@@ -0,0 +1,43 @@
+# Based on docker-library's golang 1.6 alpine and wheezy docker files.
+# https://github.com/docker-library/golang/blob/master/1.6/alpine/Dockerfile
+# https://github.com/docker-library/golang/blob/master/1.6/wheezy/Dockerfile
+FROM buildpack-deps:buster-scm
+
+ENV GOLANG_SRC_REPO_URL https://github.com/golang/go
+
+# as of 1.20 Go 1.17 is required to bootstrap
+# see https://github.com/golang/go/issues/44505
+ENV GOLANG_BOOTSTRAP_URL https://go.dev/dl/go1.17.13.linux-amd64.tar.gz
+ENV GOLANG_BOOTSTRAP_SHA256 4cdd2bc664724dc7db94ad51b503512c5ae7220951cac568120f64f8e94399fc
+ENV GOLANG_BOOTSTRAP_PATH /usr/local/bootstrap
+
+# gcc for cgo
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		g++ \
+		gcc \
+		libc6-dev \
+		make \
+		git \
+	&& rm -rf /var/lib/apt/lists/*
+
+# Setup the Bootstrap
+RUN mkdir -p "$GOLANG_BOOTSTRAP_PATH" \
+	&& curl -fsSL "$GOLANG_BOOTSTRAP_URL" -o golang.tar.gz \
+	&& echo "$GOLANG_BOOTSTRAP_SHA256  golang.tar.gz" | sha256sum -c - \
+	&& tar -C "$GOLANG_BOOTSTRAP_PATH" -xzf golang.tar.gz \
+	&& rm golang.tar.gz
+
+# Get and build Go tip
+RUN export GOROOT_BOOTSTRAP=$GOLANG_BOOTSTRAP_PATH/go \
+	&& git clone "$GOLANG_SRC_REPO_URL" /usr/local/go \
+	&& cd /usr/local/go/src \
+	&& ./make.bash \
+	&& rm -rf "$GOLANG_BOOTSTRAP_PATH" /usr/local/go/pkg/bootstrap
+
+# Build Go workspace and environment
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" \
+	&& chmod -R 777 "$GOPATH"
+
+WORKDIR $GOPATH

aws/internal/awstesting/sandbox/Dockerfile.test.gotip

@@ -0,0 +1,16 @@
+FROM aws-golang:tip
+
+ENV GOPROXY=direct
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        software-properties-common \
+    && wget -O- https://apt.corretto.aws/corretto.key | apt-key add - \
+    && add-apt-repository 'deb https://apt.corretto.aws stable main' \
+    && apt-get update && apt-get install -y --no-install-recommends \
+        vim \
+        java-17-amazon-corretto-jdk \
+    && rm -rf /var/list/apt/lists/*
+
+ADD . /go/src/github.com/aws/aws-sdk-go-v2
+WORKDIR /go/src/github.com/aws/aws-sdk-go-v2
+CMD ["make", "unit"]

aws/internal/awstesting/sandbox/Dockerfile.test.goversion

@@ -0,0 +1,18 @@
+ARG GO_VERSION
+FROM golang:${GO_VERSION}
+
+ENV GOPROXY=direct
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        software-properties-common \
+    && wget -O- https://apt.corretto.aws/corretto.key | apt-key add - \
+    && add-apt-repository 'deb https://apt.corretto.aws stable main' \
+    && apt-get update && apt-get install -y --no-install-recommends \
+        vim \
+        java-17-amazon-corretto-jdk \
+    && rm -rf /var/list/apt/lists/*
+
+ADD . /go/src/github.com/aws/aws-sdk-go-v2
+
+WORKDIR /go/src/github.com/aws/aws-sdk-go-v2
+CMD ["make", "unit"]

aws/internal/awstesting/unit/unit.go

@@ -0,0 +1,61 @@
+// Package unit performs initialization and validation for unit tests
+package unit
+
+import (
+	"context"
+	"crypto/rsa"
+	"math/big"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func init() {
+	config = aws.Config{}
+	config.Region = "mock-region"
+	config.Credentials = StubCredentialsProvider{}
+}
+
+// StubCredentialsProvider provides a stub credential provider that returns
+// static credentials that never expire.
+type StubCredentialsProvider struct{}
+
+// Retrieve satisfies the CredentialsProvider interface. Returns stub
+// credential value, and never error.
+func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) {
+	return aws.Credentials{
+		AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION",
+		Source: "unit test credentials",
+	}, nil
+}
+
+var config aws.Config
+
+// Config returns a copy of the mock configuration for unit tests.
+func Config() aws.Config { return config.Copy() }
+
+// RSAPrivateKey is used for testing functionality that requires some
+// sort of private key. Taken from crypto/rsa/rsa_test.go
+//
+// Credit to golang 1.11
+var RSAPrivateKey = &rsa.PrivateKey{
+	PublicKey: rsa.PublicKey{
+		N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"),
+		E: 3,
+	},
+	D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"),
+	Primes: []*big.Int{
+		fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"),
+		fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"),
+	},
+}
+
+// Taken from crypto/rsa/rsa_test.go
+//
+// Credit to golang 1.11
+func fromBase10(base10 string) *big.Int {
+	i, ok := new(big.Int).SetString(base10, 10)
+	if !ok {
+		panic("bad number: " + base10)
+	}
+	return i
+}

aws/internal/awstesting/util.go

@@ -0,0 +1,201 @@
+package awstesting
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+// ZeroReader is a io.Reader which will always write zeros to the byte slice provided.
+type ZeroReader struct{}
+
+// Read fills the provided byte slice with zeros returning the number of bytes written.
+func (r *ZeroReader) Read(b []byte) (int, error) {
+	for i := 0; i < len(b); i++ {
+		b[i] = 0
+	}
+	return len(b), nil
+}
+
+// ReadCloser is a io.ReadCloser for unit testing.
+// Designed to test for leaks and whether a handle has
+// been closed
+type ReadCloser struct {
+	Size     int
+	Closed   bool
+	set      bool
+	FillData func(bool, []byte, int, int)
+}
+
+// Read will call FillData and fill it with whatever data needed.
+// Decrements the size until zero, then return io.EOF.
+func (r *ReadCloser) Read(b []byte) (int, error) {
+	if r.Closed {
+		return 0, io.EOF
+	}
+
+	delta := len(b)
+	if delta > r.Size {
+		delta = r.Size
+	}
+	r.Size -= delta
+
+	for i := 0; i < delta; i++ {
+		b[i] = 'a'
+	}
+
+	if r.FillData != nil {
+		r.FillData(r.set, b, r.Size, delta)
+	}
+	r.set = true
+
+	if r.Size > 0 {
+		return delta, nil
+	}
+	return delta, io.EOF
+}
+
+// Close sets Closed to true and returns no error
+func (r *ReadCloser) Close() error {
+	r.Closed = true
+	return nil
+}
+
+// A FakeContext provides a simple stub implementation of a Context
+type FakeContext struct {
+	Error  error
+	DoneCh chan struct{}
+}
+
+// Deadline always will return not set
+func (c *FakeContext) Deadline() (deadline time.Time, ok bool) {
+	return time.Time{}, false
+}
+
+// Done returns a read channel for listening to the Done event
+func (c *FakeContext) Done() <-chan struct{} {
+	return c.DoneCh
+}
+
+// Err returns the error, is nil if not set.
+func (c *FakeContext) Err() error {
+	return c.Error
+}
+
+// Value ignores the Value and always returns nil
+func (c *FakeContext) Value(key interface{}) interface{} {
+	return nil
+}
+
+// StashEnv stashes the current environment variables except variables listed in envToKeepx
+// Returns an function to pop out old environment
+func StashEnv(envToKeep ...string) []string {
+	if runtime.GOOS == "windows" {
+		envToKeep = append(envToKeep, "ComSpec")
+		envToKeep = append(envToKeep, "SYSTEM32")
+		envToKeep = append(envToKeep, "SYSTEMROOT")
+	}
+	envToKeep = append(envToKeep, "PATH", "HOME", "USERPROFILE")
+	extraEnv := getEnvs(envToKeep)
+	originalEnv := os.Environ()
+	os.Clearenv() // clear env
+	for key, val := range extraEnv {
+		os.Setenv(key, val)
+	}
+	return originalEnv
+}
+
+// PopEnv takes the list of the environment values and injects them into the
+// process's environment variable data. Clears any existing environment values
+// that may already exist.
+func PopEnv(env []string) {
+	os.Clearenv()
+
+	for _, e := range env {
+		p := strings.SplitN(e, "=", 2)
+		k, v := p[0], ""
+		if len(p) > 1 {
+			v = p[1]
+		}
+		os.Setenv(k, v)
+	}
+}
+
+// MockCredentialsProvider is a type that can be used to mock out credentials
+// providers
+type MockCredentialsProvider struct {
+	RetrieveFn   func(ctx context.Context) (aws.Credentials, error)
+	InvalidateFn func()
+}
+
+// Retrieve calls the RetrieveFn
+func (p MockCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	return p.RetrieveFn(ctx)
+}
+
+// Invalidate calls the InvalidateFn
+func (p MockCredentialsProvider) Invalidate() {
+	p.InvalidateFn()
+}
+
+func getEnvs(envs []string) map[string]string {
+	extraEnvs := make(map[string]string)
+	for _, env := range envs {
+		if val, ok := os.LookupEnv(env); ok && len(val) > 0 {
+			extraEnvs[env] = val
+		}
+	}
+	return extraEnvs
+}
+
+const (
+	signaturePreambleSigV4  = "AWS4-HMAC-SHA256"
+	signaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256"
+)
+
+// SigV4Signature represents a parsed sigv4 or sigv4a signature.
+type SigV4Signature struct {
+	Preamble      string   // e.g. AWS4-HMAC-SHA256, AWS4-ECDSA-P256-SHA256
+	SigningName   string   // generally the service name e.g. "s3"
+	SigningRegion string   // for sigv4a this is the region-set header as-is
+	SignedHeaders []string // list of signed headers
+	Signature     string   // calculated signature
+}
+
+// ParseSigV4Signature deconstructs a sigv4 or sigv4a signature from a set of
+// request headers.
+func ParseSigV4Signature(header http.Header) *SigV4Signature {
+	auth := header.Get("Authorization")
+
+	preamble, after, _ := strings.Cut(auth, " ")
+	credential, after, _ := strings.Cut(after, ", ")
+	signedHeaders, signature, _ := strings.Cut(after, ", ")
+
+	credentialParts := strings.Split(credential, "/")
+
+	// sigv4  : AccessKeyID/DateString/SigningRegion/SigningName/SignatureID
+	// sigv4a : AccessKeyID/DateString/SigningName/SignatureID, region set on
+	//          header
+	var signingName, signingRegion string
+	if preamble == signaturePreambleSigV4 {
+		signingName = credentialParts[3]
+		signingRegion = credentialParts[2]
+	} else if preamble == signaturePreambleSigV4A {
+		signingName = credentialParts[2]
+		signingRegion = header.Get("X-Amz-Region-Set")
+	}
+
+	return &SigV4Signature{
+		Preamble:      preamble,
+		SigningName:   signingName,
+		SigningRegion: signingRegion,
+		SignedHeaders: strings.Split(signedHeaders, ";"),
+		Signature:     signature,
+	}
+}

aws/internal/awstesting/util_test.go

@@ -0,0 +1,75 @@
+package awstesting_test
+
+import (
+	"io"
+	"testing"
+
+	"github.com/versity/versitygw/aws/internal/awstesting"
+)
+
+func TestReadCloserClose(t *testing.T) {
+	rc := awstesting.ReadCloser{Size: 1}
+	err := rc.Close()
+
+	if err != nil {
+		t.Errorf("expect nil, got %v", err)
+	}
+	if !rc.Closed {
+		t.Errorf("expect closed, was not")
+	}
+	if e, a := rc.Size, 1; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestReadCloserRead(t *testing.T) {
+	rc := awstesting.ReadCloser{Size: 5}
+	b := make([]byte, 2)
+
+	n, err := rc.Read(b)
+
+	if err != nil {
+		t.Errorf("expect nil, got %v", err)
+	}
+	if e, a := n, 2; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if rc.Closed {
+		t.Errorf("expect not to be closed")
+	}
+	if e, a := rc.Size, 3; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+
+	err = rc.Close()
+	if err != nil {
+		t.Errorf("expect nil, got %v", err)
+	}
+	n, err = rc.Read(b)
+	if e, a := err, io.EOF; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := n, 0; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestReadCloserReadAll(t *testing.T) {
+	rc := awstesting.ReadCloser{Size: 5}
+	b := make([]byte, 5)
+
+	n, err := rc.Read(b)
+
+	if e, a := err, io.EOF; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := n, 5; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if rc.Closed {
+		t.Errorf("expect not to be closed")
+	}
+	if e, a := rc.Size, 0; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}

aws/internal/sdk/interfaces.go

@@ -0,0 +1,9 @@
+package sdk
+
+// Invalidator provides access to a type's invalidate method to make it
+// invalidate it cache.
+//
+// e.g aws.SafeCredentialsProvider's Invalidate method.
+type Invalidator interface {
+	Invalidate()
+}

aws/internal/sdk/time.go

@@ -0,0 +1,74 @@
+package sdk
+
+import (
+	"context"
+	"time"
+)
+
+func init() {
+	NowTime = time.Now
+	Sleep = time.Sleep
+	SleepWithContext = sleepWithContext
+}
+
+// NowTime is a value for getting the current time. This value can be overridden
+// for testing mocking out current time.
+var NowTime func() time.Time
+
+// Sleep is a value for sleeping for a duration. This value can be overridden
+// for testing and mocking out sleep duration.
+var Sleep func(time.Duration)
+
+// SleepWithContext will wait for the timer duration to expire, or the context
+// is canceled. Which ever happens first. If the context is canceled the Context's
+// error will be returned.
+//
+// This value can be overridden for testing and mocking out sleep duration.
+var SleepWithContext func(context.Context, time.Duration) error
+
+// sleepWithContext will wait for the timer duration to expire, or the context
+// is canceled. Which ever happens first. If the context is canceled the
+// Context's error will be returned.
+func sleepWithContext(ctx context.Context, dur time.Duration) error {
+	t := time.NewTimer(dur)
+	defer t.Stop()
+
+	select {
+	case <-t.C:
+		break
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	return nil
+}
+
+// noOpSleepWithContext does nothing, returns immediately.
+func noOpSleepWithContext(context.Context, time.Duration) error {
+	return nil
+}
+
+func noOpSleep(time.Duration) {}
+
+// TestingUseNopSleep is a utility for disabling sleep across the SDK for
+// testing.
+func TestingUseNopSleep() func() {
+	SleepWithContext = noOpSleepWithContext
+	Sleep = noOpSleep
+
+	return func() {
+		SleepWithContext = sleepWithContext
+		Sleep = time.Sleep
+	}
+}
+
+// TestingUseReferenceTime is a utility for swapping the time function across the SDK to return a specific reference time
+// for testing purposes.
+func TestingUseReferenceTime(referenceTime time.Time) func() {
+	NowTime = func() time.Time {
+		return referenceTime
+	}
+	return func() {
+		NowTime = time.Now
+	}
+}

aws/internal/sdk/time_test.go

@@ -0,0 +1,32 @@
+package sdk
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestSleepWithContext(t *testing.T) {
+	ctx, cancelFn := context.WithCancel(context.Background())
+	defer cancelFn()
+
+	err := sleepWithContext(ctx, 1*time.Millisecond)
+	if err != nil {
+		t.Errorf("expect context to not be canceled, got %v", err)
+	}
+}
+
+func TestSleepWithContext_Canceled(t *testing.T) {
+	ctx, cancelFn := context.WithCancel(context.Background())
+	cancelFn()
+
+	err := sleepWithContext(ctx, 10*time.Second)
+	if err == nil {
+		t.Fatalf("expect error, did not get one")
+	}
+
+	if e, a := "context canceled", err.Error(); !strings.Contains(a, e) {
+		t.Errorf("expect %v error, got %v", e, a)
+	}
+}

aws/internal/strings/strings.go

@@ -0,0 +1,11 @@
+package strings
+
+import (
+	"strings"
+)
+
+// HasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
+// under Unicode case-folding.
+func HasPrefixFold(s, prefix string) bool {
+	return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
+}

aws/internal/strings/strings_test.go

@@ -0,0 +1,81 @@
+package strings
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestHasPrefixFold(t *testing.T) {
+	type args struct {
+		s      string
+		prefix string
+	}
+	tests := map[string]struct {
+		args args
+		want bool
+	}{
+		"empty strings and prefix": {
+			args: args{
+				s:      "",
+				prefix: "",
+			},
+			want: true,
+		},
+		"strings starts with prefix": {
+			args: args{
+				s:      "some string",
+				prefix: "some",
+			},
+			want: true,
+		},
+		"prefix longer then string": {
+			args: args{
+				s:      "some",
+				prefix: "some string",
+			},
+		},
+		"equal length string and prefix": {
+			args: args{
+				s:      "short string",
+				prefix: "short string",
+			},
+			want: true,
+		},
+		"different cases": {
+			args: args{
+				s:      "ShOrT StRING",
+				prefix: "short",
+			},
+			want: true,
+		},
+		"empty prefix not empty string": {
+			args: args{
+				s:      "ShOrT StRING",
+				prefix: "",
+			},
+			want: true,
+		},
+		"mixed-case prefixes": {
+			args: args{
+				s:      "SoMe String",
+				prefix: "sOme",
+			},
+			want: true,
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := HasPrefixFold(tt.args.s, tt.args.prefix); got != tt.want {
+				t.Errorf("HasPrefixFold() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func BenchmarkHasPrefixFold(b *testing.B) {
+	HasPrefixFold("SoME string", "sOmE")
+}
+
+func BenchmarkHasPrefix(b *testing.B) {
+	strings.HasPrefix(strings.ToLower("SoME string"), strings.ToLower("sOmE"))
+}

aws/signer/internal/v4/cache.go

@@ -0,0 +1,115 @@
+package v4
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func lookupKey(service, region string) string {
+	var s strings.Builder
+	s.Grow(len(region) + len(service) + 3)
+	s.WriteString(region)
+	s.WriteRune('/')
+	s.WriteString(service)
+	return s.String()
+}
+
+type derivedKey struct {
+	AccessKey  string
+	Date       time.Time
+	Credential []byte
+}
+
+type derivedKeyCache struct {
+	values map[string]derivedKey
+	mutex  sync.RWMutex
+}
+
+func newDerivedKeyCache() derivedKeyCache {
+	return derivedKeyCache{
+		values: make(map[string]derivedKey),
+	}
+}
+
+func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte {
+	key := lookupKey(service, region)
+	s.mutex.RLock()
+	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
+		s.mutex.RUnlock()
+		return cred
+	}
+	s.mutex.RUnlock()
+
+	s.mutex.Lock()
+	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
+		s.mutex.Unlock()
+		return cred
+	}
+	cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime)
+	entry := derivedKey{
+		AccessKey:  credentials.AccessKeyID,
+		Date:       signingTime.Time,
+		Credential: cred,
+	}
+	s.values[key] = entry
+	s.mutex.Unlock()
+
+	return cred
+}
+
+func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) {
+	cacheEntry, ok := s.retrieveFromCache(key)
+	if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) {
+		return cacheEntry.Credential, true
+	}
+	return nil, false
+}
+
+func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) {
+	if v, ok := s.values[key]; ok {
+		return v, true
+	}
+	return derivedKey{}, false
+}
+
+// SigningKeyDeriver derives a signing key from a set of credentials
+type SigningKeyDeriver struct {
+	cache derivedKeyCache
+}
+
+// NewSigningKeyDeriver returns a new SigningKeyDeriver
+func NewSigningKeyDeriver() *SigningKeyDeriver {
+	return &SigningKeyDeriver{
+		cache: newDerivedKeyCache(),
+	}
+}
+
+// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing.
+func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte {
+	return k.cache.Get(credential, service, region, signingTime)
+}
+
+func deriveKey(secret, service, region string, t SigningTime) []byte {
+	hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat()))
+	hmacRegion := HMACSHA256(hmacDate, []byte(region))
+	hmacService := HMACSHA256(hmacRegion, []byte(service))
+	return HMACSHA256(hmacService, []byte("aws4_request"))
+}
+
+func isSameDay(x, y time.Time) bool {
+	xYear, xMonth, xDay := x.Date()
+	yYear, yMonth, yDay := y.Date()
+
+	if xYear != yYear {
+		return false
+	}
+
+	if xMonth != yMonth {
+		return false
+	}
+
+	return xDay == yDay
+}

aws/signer/internal/v4/const.go

@@ -0,0 +1,40 @@
+package v4
+
+// Signature Version 4 (SigV4) Constants
+const (
+	// EmptyStringSHA256 is the hex encoded sha256 value of an empty string
+	EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
+
+	// UnsignedPayload indicates that the request payload body is unsigned
+	UnsignedPayload = "UNSIGNED-PAYLOAD"
+
+	// AmzAlgorithmKey indicates the signing algorithm
+	AmzAlgorithmKey = "X-Amz-Algorithm"
+
+	// AmzSecurityTokenKey indicates the security token to be used with temporary credentials
+	AmzSecurityTokenKey = "X-Amz-Security-Token"
+
+	// AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z'
+	AmzDateKey = "X-Amz-Date"
+
+	// AmzCredentialKey is the access key ID and credential scope
+	AmzCredentialKey = "X-Amz-Credential"
+
+	// AmzSignedHeadersKey is the set of headers signed for the request
+	AmzSignedHeadersKey = "X-Amz-SignedHeaders"
+
+	// AmzSignatureKey is the query parameter to store the SigV4 signature
+	AmzSignatureKey = "X-Amz-Signature"
+
+	// TimeFormat is the time format to be used in the X-Amz-Date header or query parameter
+	TimeFormat = "20060102T150405Z"
+
+	// ShortTimeFormat is the shorten time format used in the credential scope
+	ShortTimeFormat = "20060102"
+
+	// ContentSHAKey is the SHA256 of request body
+	ContentSHAKey = "X-Amz-Content-Sha256"
+
+	// StreamingEventsPayload indicates that the request payload body is a signed event stream.
+	StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS"
+)

aws/signer/internal/v4/header_rules.go

@@ -0,0 +1,82 @@
+package v4
+
+import (
+	sdkstrings "github.com/versity/versitygw/aws/internal/strings"
+)
+
+// Rules houses a set of Rule needed for validation of a
+// string value
+type Rules []Rule
+
+// Rule interface allows for more flexible rules and just simply
+// checks whether or not a value adheres to that Rule
+type Rule interface {
+	IsValid(value string) bool
+}
+
+// IsValid will iterate through all rules and see if any rules
+// apply to the value and supports nested rules
+func (r Rules) IsValid(value string) bool {
+	for _, rule := range r {
+		if rule.IsValid(value) {
+			return true
+		}
+	}
+	return false
+}
+
+// MapRule generic Rule for maps
+type MapRule map[string]struct{}
+
+// IsValid for the map Rule satisfies whether it exists in the map
+func (m MapRule) IsValid(value string) bool {
+	_, ok := m[value]
+	return ok
+}
+
+// AllowList is a generic Rule for include listing
+type AllowList struct {
+	Rule
+}
+
+// IsValid for AllowList checks if the value is within the AllowList
+func (w AllowList) IsValid(value string) bool {
+	return w.Rule.IsValid(value)
+}
+
+// ExcludeList is a generic Rule for exclude listing
+type ExcludeList struct {
+	Rule
+}
+
+// IsValid for AllowList checks if the value is within the AllowList
+func (b ExcludeList) IsValid(value string) bool {
+	return !b.Rule.IsValid(value)
+}
+
+// Patterns is a list of strings to match against
+type Patterns []string
+
+// IsValid for Patterns checks each pattern and returns if a match has
+// been found
+func (p Patterns) IsValid(value string) bool {
+	for _, pattern := range p {
+		if sdkstrings.HasPrefixFold(value, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// InclusiveRules rules allow for rules to depend on one another
+type InclusiveRules []Rule
+
+// IsValid will return true if all rules are true
+func (r InclusiveRules) IsValid(value string) bool {
+	for _, rule := range r {
+		if !rule.IsValid(value) {
+			return false
+		}
+	}
+	return true
+}

aws/signer/internal/v4/headers.go

@@ -0,0 +1,72 @@
+package v4
+
+// IgnoredHeaders is a list of headers that are ignored during signing
+var IgnoredHeaders = Rules{
+	ExcludeList{
+		MapRule{
+			"Authorization": struct{}{},
+			// some clients use user-agent in signed headers
+			// "User-Agent":      struct{}{},
+			"X-Amzn-Trace-Id": struct{}{},
+			"Expect":          struct{}{},
+		},
+	},
+}
+
+// RequiredSignedHeaders is a allow list for Build canonical headers.
+var RequiredSignedHeaders = Rules{
+	AllowList{
+		MapRule{
+			"Cache-Control":                         struct{}{},
+			"Content-Disposition":                   struct{}{},
+			"Content-Encoding":                      struct{}{},
+			"Content-Language":                      struct{}{},
+			"Content-Md5":                           struct{}{},
+			"Content-Type":                          struct{}{},
+			"Expires":                               struct{}{},
+			"If-Match":                              struct{}{},
+			"If-Modified-Since":                     struct{}{},
+			"If-None-Match":                         struct{}{},
+			"If-Unmodified-Since":                   struct{}{},
+			"Range":                                 struct{}{},
+			"X-Amz-Acl":                             struct{}{},
+			"X-Amz-Copy-Source":                     struct{}{},
+			"X-Amz-Copy-Source-If-Match":            struct{}{},
+			"X-Amz-Copy-Source-If-Modified-Since":   struct{}{},
+			"X-Amz-Copy-Source-If-None-Match":       struct{}{},
+			"X-Amz-Copy-Source-If-Unmodified-Since": struct{}{},
+			"X-Amz-Copy-Source-Range":               struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   struct{}{},
+			"X-Amz-Expected-Bucket-Owner":                                 struct{}{},
+			"X-Amz-Grant-Full-control":                                    struct{}{},
+			"X-Amz-Grant-Read":                                            struct{}{},
+			"X-Amz-Grant-Read-Acp":                                        struct{}{},
+			"X-Amz-Grant-Write":                                           struct{}{},
+			"X-Amz-Grant-Write-Acp":                                       struct{}{},
+			"X-Amz-Metadata-Directive":                                    struct{}{},
+			"X-Amz-Mfa":                                                   struct{}{},
+			"X-Amz-Request-Payer":                                         struct{}{},
+			"X-Amz-Server-Side-Encryption":                                struct{}{},
+			"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id":                 struct{}{},
+			"X-Amz-Server-Side-Encryption-Context":                        struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm":             struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Key":                   struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":               struct{}{},
+			"X-Amz-Storage-Class":                                         struct{}{},
+			"X-Amz-Website-Redirect-Location":                             struct{}{},
+			"X-Amz-Content-Sha256":                                        struct{}{},
+			"X-Amz-Tagging":                                               struct{}{},
+		},
+	},
+	Patterns{"X-Amz-Object-Lock-"},
+	Patterns{"X-Amz-Meta-"},
+}
+
+// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value
+// represents whether or not it is a pattern.
+var AllowedQueryHoisting = InclusiveRules{
+	ExcludeList{RequiredSignedHeaders},
+	Patterns{"X-Amz-"},
+}

aws/signer/internal/v4/headers_test.go

@@ -0,0 +1,63 @@
+package v4
+
+import "testing"
+
+func TestAllowedQueryHoisting(t *testing.T) {
+	cases := map[string]struct {
+		Header      string
+		ExpectHoist bool
+	}{
+		"object-lock": {
+			Header:      "X-Amz-Object-Lock-Mode",
+			ExpectHoist: false,
+		},
+		"s3 metadata": {
+			Header:      "X-Amz-Meta-SomeName",
+			ExpectHoist: false,
+		},
+		"another header": {
+			Header:      "X-Amz-SomeOtherHeader",
+			ExpectHoist: true,
+		},
+		"non X-AMZ header": {
+			Header:      "X-SomeOtherHeader",
+			ExpectHoist: false,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a {
+				t.Errorf("expect hoist %v, was %v", e, a)
+			}
+		})
+	}
+}
+
+func TestIgnoredHeaders(t *testing.T) {
+	cases := map[string]struct {
+		Header        string
+		ExpectIgnored bool
+	}{
+		"expect": {
+			Header:        "Expect",
+			ExpectIgnored: true,
+		},
+		"authorization": {
+			Header:        "Authorization",
+			ExpectIgnored: true,
+		},
+		"X-AMZ header": {
+			Header:        "X-Amz-Content-Sha256",
+			ExpectIgnored: false,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a {
+				t.Errorf("expect ignored %v, was %v", e, a)
+			}
+		})
+	}
+}

aws/signer/internal/v4/hmac.go

@@ -0,0 +1,13 @@
+package v4
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+)
+
+// HMACSHA256 computes a HMAC-SHA256 of data given the provided key.
+func HMACSHA256(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}

aws/signer/internal/v4/host.go

@@ -0,0 +1,75 @@
+package v4
+
+import (
+	"net/http"
+	"strings"
+)
+
+// SanitizeHostForHeader removes default port from host and updates request.Host
+func SanitizeHostForHeader(r *http.Request) {
+	host := getHost(r)
+	port := portOnly(host)
+	if port != "" && isDefaultPort(r.URL.Scheme, port) {
+		r.Host = stripPort(host)
+	}
+}
+
+// Returns host from request
+func getHost(r *http.Request) string {
+	if r.Host != "" {
+		return r.Host
+	}
+
+	return r.URL.Host
+}
+
+// Hostname returns u.Host, without any port number.
+//
+// If Host is an IPv6 literal with a port number, Hostname returns the
+// IPv6 literal without the square brackets. IPv6 literals may include
+// a zone identifier.
+//
+// Copied from the Go 1.8 standard library (net/url)
+func stripPort(hostport string) string {
+	colon := strings.IndexByte(hostport, ':')
+	if colon == -1 {
+		return hostport
+	}
+	if i := strings.IndexByte(hostport, ']'); i != -1 {
+		return strings.TrimPrefix(hostport[:i], "[")
+	}
+	return hostport[:colon]
+}
+
+// Port returns the port part of u.Host, without the leading colon.
+// If u.Host doesn't contain a port, Port returns an empty string.
+//
+// Copied from the Go 1.8 standard library (net/url)
+func portOnly(hostport string) string {
+	colon := strings.IndexByte(hostport, ':')
+	if colon == -1 {
+		return ""
+	}
+	if i := strings.Index(hostport, "]:"); i != -1 {
+		return hostport[i+len("]:"):]
+	}
+	if strings.Contains(hostport, "]") {
+		return ""
+	}
+	return hostport[colon+len(":"):]
+}
+
+// Returns true if the specified URI is using the standard port
+// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs)
+func isDefaultPort(scheme, port string) bool {
+	if port == "" {
+		return true
+	}
+
+	lowerCaseScheme := strings.ToLower(scheme)
+	if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") {
+		return true
+	}
+
+	return false
+}

aws/signer/internal/v4/scope.go

@@ -0,0 +1,13 @@
+package v4
+
+import "strings"
+
+// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope
+func BuildCredentialScope(signingTime SigningTime, region, service string) string {
+	return strings.Join([]string{
+		signingTime.ShortTimeFormat(),
+		region,
+		service,
+		"aws4_request",
+	}, "/")
+}

aws/signer/internal/v4/time.go

@@ -0,0 +1,36 @@
+package v4
+
+import "time"
+
+// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing.
+type SigningTime struct {
+	time.Time
+	timeFormat      string
+	shortTimeFormat string
+}
+
+// NewSigningTime creates a new SigningTime given a time.Time
+func NewSigningTime(t time.Time) SigningTime {
+	return SigningTime{
+		Time: t,
+	}
+}
+
+// TimeFormat provides a time formatted in the X-Amz-Date format.
+func (m *SigningTime) TimeFormat() string {
+	return m.format(&m.timeFormat, TimeFormat)
+}
+
+// ShortTimeFormat provides a time formatted of 20060102.
+func (m *SigningTime) ShortTimeFormat() string {
+	return m.format(&m.shortTimeFormat, ShortTimeFormat)
+}
+
+func (m *SigningTime) format(target *string, format string) string {
+	if len(*target) > 0 {
+		return *target
+	}
+	v := m.Time.Format(format)
+	*target = v
+	return v
+}

aws/signer/internal/v4/util.go

@@ -0,0 +1,80 @@
+package v4
+
+import (
+	"net/url"
+	"strings"
+)
+
+const doubleSpace = "  "
+
+// StripExcessSpaces will rewrite the passed in slice's string values to not
+// contain multiple side-by-side spaces.
+func StripExcessSpaces(str string) string {
+	var j, k, l, m, spaces int
+	// Trim trailing spaces
+	for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- {
+	}
+
+	// Trim leading spaces
+	for k = 0; k < j && str[k] == ' '; k++ {
+	}
+	str = str[k : j+1]
+
+	// Strip multiple spaces.
+	j = strings.Index(str, doubleSpace)
+	if j < 0 {
+		return str
+	}
+
+	buf := []byte(str)
+	for k, m, l = j, j, len(buf); k < l; k++ {
+		if buf[k] == ' ' {
+			if spaces == 0 {
+				// First space.
+				buf[m] = buf[k]
+				m++
+			}
+			spaces++
+		} else {
+			// End of multiple spaces.
+			spaces = 0
+			buf[m] = buf[k]
+			m++
+		}
+	}
+
+	return string(buf[:m])
+}
+
+// GetURIPath returns the escaped URI component from the provided URL.
+func GetURIPath(u *url.URL) string {
+	var uriPath string
+
+	if len(u.Opaque) > 0 {
+		const schemeSep, pathSep, queryStart = "//", "/", "?"
+
+		opaque := u.Opaque
+		// Cut off the query string if present.
+		if idx := strings.Index(opaque, queryStart); idx >= 0 {
+			opaque = opaque[:idx]
+		}
+
+		// Cutout the scheme separator if present.
+		if strings.Index(opaque, schemeSep) == 0 {
+			opaque = opaque[len(schemeSep):]
+		}
+
+		// capture URI path starting with first path separator.
+		if idx := strings.Index(opaque, pathSep); idx >= 0 {
+			uriPath = opaque[idx:]
+		}
+	} else {
+		uriPath = u.EscapedPath()
+	}
+
+	if len(uriPath) == 0 {
+		uriPath = "/"
+	}
+
+	return uriPath
+}

aws/signer/internal/v4/util_test.go

@@ -0,0 +1,158 @@
+package v4
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+func lazyURLParse(v string) func() (*url.URL, error) {
+	return func() (*url.URL, error) {
+		return url.Parse(v)
+	}
+}
+
+func TestGetURIPath(t *testing.T) {
+	cases := map[string]struct {
+		getURL func() (*url.URL, error)
+		expect string
+	}{
+		// Cases
+		"with scheme": {
+			getURL: lazyURLParse("https://localhost:9000"),
+			expect: "/",
+		},
+		"no port, with scheme": {
+			getURL: lazyURLParse("https://localhost"),
+			expect: "/",
+		},
+		"without scheme": {
+			getURL: lazyURLParse("localhost:9000"),
+			expect: "/",
+		},
+		"without scheme, with path": {
+			getURL: lazyURLParse("localhost:9000/abc123"),
+			expect: "/abc123",
+		},
+		"without scheme, with separator": {
+			getURL: lazyURLParse("//localhost:9000"),
+			expect: "/",
+		},
+		"no port, without scheme, with separator": {
+			getURL: lazyURLParse("//localhost"),
+			expect: "/",
+		},
+		"without scheme, with separator, with path": {
+			getURL: lazyURLParse("//localhost:9000/abc123"),
+			expect: "/abc123",
+		},
+		"no port, without scheme, with separator, with path": {
+			getURL: lazyURLParse("//localhost/abc123"),
+			expect: "/abc123",
+		},
+		"opaque with query string": {
+			getURL: lazyURLParse("localhost:9000/abc123?efg=456"),
+			expect: "/abc123",
+		},
+		"failing test": {
+			getURL: func() (*url.URL, error) {
+				endpoint := "https://service.region.amazonaws.com"
+				req, _ := http.NewRequest("POST", endpoint, nil)
+				u := req.URL
+
+				u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
+
+				query := u.Query()
+				query.Set("some-query-key", "value")
+				u.RawQuery = query.Encode()
+
+				return u, nil
+			},
+			expect: "/bucket/key-._~,!@#$%^&*()",
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			u, err := c.getURL()
+			if err != nil {
+				t.Fatalf("failed to get URL, %v", err)
+			}
+
+			actual := GetURIPath(u)
+			if e, a := c.expect, actual; e != a {
+				t.Errorf("expect %v path, got %v", e, a)
+			}
+		})
+	}
+}
+
+func TestStripExcessHeaders(t *testing.T) {
+	vals := []string{
+		"",
+		"123",
+		"1 2 3",
+		"1 2 3 ",
+		"  1 2 3",
+		"1  2 3",
+		"1  23",
+		"1  2  3",
+		"1  2  ",
+		" 1  2  ",
+		"12   3",
+		"12   3   1",
+		"12           3     1",
+		"12     3       1abc123",
+	}
+
+	expected := []string{
+		"",
+		"123",
+		"1 2 3",
+		"1 2 3",
+		"1 2 3",
+		"1 2 3",
+		"1 23",
+		"1 2 3",
+		"1 2",
+		"1 2",
+		"12 3",
+		"12 3 1",
+		"12 3 1",
+		"12 3 1abc123",
+	}
+
+	for i := 0; i < len(vals); i++ {
+		r := StripExcessSpaces(vals[i])
+		if e, a := expected[i], r; e != a {
+			t.Errorf("%d, expect %v, got %v", i, e, a)
+		}
+	}
+}
+
+var stripExcessSpaceCases = []string{
+	`AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`,
+	`123   321   123   321`,
+	`   123   321   123   321   `,
+	`   123    321    123          321   `,
+	"123",
+	"1 2 3",
+	"  1 2 3",
+	"1  2 3",
+	"1  23",
+	"1  2  3",
+	"1  2  ",
+	" 1  2  ",
+	"12   3",
+	"12   3   1",
+	"12           3     1",
+	"12     3       1abc123",
+}
+
+func BenchmarkStripExcessSpaces(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		for _, v := range stripExcessSpaceCases {
+			StripExcessSpaces(v)
+		}
+	}
+}

aws/signer/v4/functional_test.go

@@ -0,0 +1,139 @@
+package v4_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/versity/versitygw/aws/internal/awstesting/unit"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+var standaloneSignCases = []struct {
+	OrigURI                    string
+	OrigQuery                  string
+	Region, Service, SubDomain string
+	ExpSig                     string
+	EscapedURI                 string
+}{
+	{
+		OrigURI:   `/logs-*/_search`,
+		OrigQuery: `pretty=true`,
+		Region:    "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
+		EscapedURI: `/logs-%2A/_search`,
+		ExpSig:     `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
+	},
+}
+
+func TestStandaloneSign_CustomURIEscape(t *testing.T) {
+	var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967`
+
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner(func(signer *v4.SignerOptions) {
+		signer.DisableURIPathEscaping = true
+	})
+
+	host := "https://subdomain.us-east-1.es.amazonaws.com"
+	req, err := http.NewRequest("GET", host, nil)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	req.URL.Path = `/log-*/_search`
+	req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search"
+
+	err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0))
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	actual := req.Header.Get("Authorization")
+	if e, a := expectSig, actual; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestStandaloneSign(t *testing.T) {
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner()
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		// URL.EscapedPath() will be used by the signer to get the
+		// escaped form of the request's URI path.
+		req.URL.Path = c.OrigURI
+		req.URL.RawQuery = c.OrigQuery
+
+		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		actual := req.Header.Get("Authorization")
+		if e, a := c.ExpSig, actual; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.OrigURI, req.URL.Path; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+	}
+}
+
+func TestStandaloneSign_RawPath(t *testing.T) {
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner()
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		// URL.EscapedPath() will be used by the signer to get the
+		// escaped form of the request's URI path.
+		req.URL.Path = c.OrigURI
+		req.URL.RawPath = c.EscapedURI
+		req.URL.RawQuery = c.OrigQuery
+
+		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		actual := req.Header.Get("Authorization")
+		if e, a := c.ExpSig, actual; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.OrigURI, req.URL.Path; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+	}
+}

aws/signer/v4/middleware.go

@@ -0,0 +1,443 @@
+package v4
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+	"github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics"
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	internalauth "github.com/versity/versitygw/aws/internal/auth"
+	"github.com/versity/versitygw/aws/internal/sdk"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+const computePayloadHashMiddlewareID = "ComputePayloadHash"
+
+// HashComputationError indicates an error occurred while computing the signing hash
+type HashComputationError struct {
+	Err error
+}
+
+// Error is the error message
+func (e *HashComputationError) Error() string {
+	return fmt.Sprintf("failed to compute payload hash: %v", e.Err)
+}
+
+// Unwrap returns the underlying error if one is set
+func (e *HashComputationError) Unwrap() error {
+	return e.Err
+}
+
+// SigningError indicates an error condition occurred while performing SigV4 signing
+type SigningError struct {
+	Err error
+}
+
+func (e *SigningError) Error() string {
+	return fmt.Sprintf("failed to sign request: %v", e.Err)
+}
+
+// Unwrap returns the underlying error cause
+func (e *SigningError) Unwrap() error {
+	return e.Err
+}
+
+// UseDynamicPayloadSigningMiddleware swaps the compute payload sha256 middleware with a resolver middleware that
+// switches between unsigned and signed payload based on TLS state for request.
+// This middleware should not be used for AWS APIs that do not support unsigned payload signing auth.
+// By default, SDK uses this middleware for known AWS APIs that support such TLS based auth selection .
+//
+// Usage example -
+// S3 PutObject API allows unsigned payload signing auth usage when TLS is enabled, and uses this middleware to
+// dynamically switch between unsigned and signed payload based on TLS state for request.
+func UseDynamicPayloadSigningMiddleware(stack *middleware.Stack) error {
+	_, err := stack.Finalize.Swap(computePayloadHashMiddlewareID, &dynamicPayloadSigningMiddleware{})
+	return err
+}
+
+// dynamicPayloadSigningMiddleware dynamically resolves the middleware that computes and set payload sha256 middleware.
+type dynamicPayloadSigningMiddleware struct {
+}
+
+// ID returns the resolver identifier
+func (m *dynamicPayloadSigningMiddleware) ID() string {
+	return computePayloadHashMiddlewareID
+}
+
+// HandleFinalize delegates SHA256 computation according to whether the request
+// is TLS-enabled.
+func (m *dynamicPayloadSigningMiddleware) HandleFinalize(
+	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	req, ok := in.Request.(*smithyhttp.Request)
+	if !ok {
+		return out, metadata, fmt.Errorf("unknown transport type %T", in.Request)
+	}
+
+	if req.IsHTTPS() {
+		return (&UnsignedPayload{}).HandleFinalize(ctx, in, next)
+	}
+	return (&ComputePayloadSHA256{}).HandleFinalize(ctx, in, next)
+}
+
+// UnsignedPayload sets the SigV4 request payload hash to unsigned.
+//
+// Will not set the Unsigned Payload magic SHA value, if a SHA has already been
+// stored in the context. (e.g. application pre-computed SHA256 before making
+// API call).
+//
+// This middleware does not check the X-Amz-Content-Sha256 header, if that
+// header is serialized a middleware must translate it into the context.
+type UnsignedPayload struct{}
+
+// AddUnsignedPayloadMiddleware adds unsignedPayload to the operation
+// middleware stack
+func AddUnsignedPayloadMiddleware(stack *middleware.Stack) error {
+	return stack.Finalize.Insert(&UnsignedPayload{}, "ResolveEndpointV2", middleware.After)
+}
+
+// ID returns the unsignedPayload identifier
+func (m *UnsignedPayload) ID() string {
+	return computePayloadHashMiddlewareID
+}
+
+// HandleFinalize sets the payload hash magic value to the unsigned sentinel.
+func (m *UnsignedPayload) HandleFinalize(
+	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	if GetPayloadHash(ctx) == "" {
+		ctx = SetPayloadHash(ctx, v4Internal.UnsignedPayload)
+	}
+	return next.HandleFinalize(ctx, in)
+}
+
+// ComputePayloadSHA256 computes SHA256 payload hash to sign.
+//
+// Will not set the Unsigned Payload magic SHA value, if a SHA has already been
+// stored in the context. (e.g. application pre-computed SHA256 before making
+// API call).
+//
+// This middleware does not check the X-Amz-Content-Sha256 header, if that
+// header is serialized a middleware must translate it into the context.
+type ComputePayloadSHA256 struct{}
+
+// AddComputePayloadSHA256Middleware adds computePayloadSHA256 to the
+// operation middleware stack
+func AddComputePayloadSHA256Middleware(stack *middleware.Stack) error {
+	return stack.Finalize.Insert(&ComputePayloadSHA256{}, "ResolveEndpointV2", middleware.After)
+}
+
+// RemoveComputePayloadSHA256Middleware removes computePayloadSHA256 from the
+// operation middleware stack
+func RemoveComputePayloadSHA256Middleware(stack *middleware.Stack) error {
+	_, err := stack.Finalize.Remove(computePayloadHashMiddlewareID)
+	return err
+}
+
+// ID is the middleware name
+func (m *ComputePayloadSHA256) ID() string {
+	return computePayloadHashMiddlewareID
+}
+
+// HandleFinalize computes the payload hash for the request, storing it to the
+// context. This is a no-op if a caller has previously set that value.
+func (m *ComputePayloadSHA256) HandleFinalize(
+	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	if GetPayloadHash(ctx) != "" {
+		return next.HandleFinalize(ctx, in)
+	}
+
+	req, ok := in.Request.(*smithyhttp.Request)
+	if !ok {
+		return out, metadata, &HashComputationError{
+			Err: fmt.Errorf("unexpected request middleware type %T", in.Request),
+		}
+	}
+
+	hash := sha256.New()
+	if stream := req.GetStream(); stream != nil {
+		_, err = io.Copy(hash, stream)
+		if err != nil {
+			return out, metadata, &HashComputationError{
+				Err: fmt.Errorf("failed to compute payload hash, %w", err),
+			}
+		}
+
+		if err := req.RewindStream(); err != nil {
+			return out, metadata, &HashComputationError{
+				Err: fmt.Errorf("failed to seek body to start, %w", err),
+			}
+		}
+	}
+
+	ctx = SetPayloadHash(ctx, hex.EncodeToString(hash.Sum(nil)))
+
+	return next.HandleFinalize(ctx, in)
+}
+
+// SwapComputePayloadSHA256ForUnsignedPayloadMiddleware replaces the
+// ComputePayloadSHA256 middleware with the UnsignedPayload middleware.
+//
+// Use this to disable computing the Payload SHA256 checksum and instead use
+// UNSIGNED-PAYLOAD for the SHA256 value.
+func SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack *middleware.Stack) error {
+	_, err := stack.Finalize.Swap(computePayloadHashMiddlewareID, &UnsignedPayload{})
+	return err
+}
+
+// ContentSHA256Header sets the X-Amz-Content-Sha256 header value to
+// the Payload hash stored in the context.
+type ContentSHA256Header struct{}
+
+// AddContentSHA256HeaderMiddleware adds ContentSHA256Header to the
+// operation middleware stack
+func AddContentSHA256HeaderMiddleware(stack *middleware.Stack) error {
+	return stack.Finalize.Insert(&ContentSHA256Header{}, computePayloadHashMiddlewareID, middleware.After)
+}
+
+// RemoveContentSHA256HeaderMiddleware removes contentSHA256Header middleware
+// from the operation middleware stack
+func RemoveContentSHA256HeaderMiddleware(stack *middleware.Stack) error {
+	_, err := stack.Finalize.Remove((*ContentSHA256Header)(nil).ID())
+	return err
+}
+
+// ID returns the ContentSHA256HeaderMiddleware identifier
+func (m *ContentSHA256Header) ID() string {
+	return "SigV4ContentSHA256Header"
+}
+
+// HandleFinalize sets the X-Amz-Content-Sha256 header value to the Payload hash
+// stored in the context.
+func (m *ContentSHA256Header) HandleFinalize(
+	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	req, ok := in.Request.(*smithyhttp.Request)
+	if !ok {
+		return out, metadata, &HashComputationError{Err: fmt.Errorf("unexpected request middleware type %T", in.Request)}
+	}
+
+	req.Header.Set(v4Internal.ContentSHAKey, GetPayloadHash(ctx))
+	return next.HandleFinalize(ctx, in)
+}
+
+// SignHTTPRequestMiddlewareOptions is the configuration options for
+// [SignHTTPRequestMiddleware].
+//
+// Deprecated: [SignHTTPRequestMiddleware] is deprecated.
+type SignHTTPRequestMiddlewareOptions struct {
+	CredentialsProvider aws.CredentialsProvider
+	Signer              HTTPSigner
+	LogSigning          bool
+}
+
+// SignHTTPRequestMiddleware is a `FinalizeMiddleware` implementation for SigV4
+// HTTP Signing.
+//
+// Deprecated: AWS service clients no longer use this middleware. Signing as an
+// SDK operation is now performed through an internal per-service middleware
+// which opaquely selects and uses the signer from the resolved auth scheme.
+type SignHTTPRequestMiddleware struct {
+	credentialsProvider aws.CredentialsProvider
+	signer              HTTPSigner
+	logSigning          bool
+}
+
+// NewSignHTTPRequestMiddleware constructs a [SignHTTPRequestMiddleware] using
+// the given [Signer] for signing requests.
+//
+// Deprecated: SignHTTPRequestMiddleware is deprecated.
+func NewSignHTTPRequestMiddleware(options SignHTTPRequestMiddlewareOptions) *SignHTTPRequestMiddleware {
+	return &SignHTTPRequestMiddleware{
+		credentialsProvider: options.CredentialsProvider,
+		signer:              options.Signer,
+		logSigning:          options.LogSigning,
+	}
+}
+
+// ID is the SignHTTPRequestMiddleware identifier.
+//
+// Deprecated: SignHTTPRequestMiddleware is deprecated.
+func (s *SignHTTPRequestMiddleware) ID() string {
+	return "Signing"
+}
+
+// HandleFinalize will take the provided input and sign the request using the
+// SigV4 authentication scheme.
+//
+// Deprecated: SignHTTPRequestMiddleware is deprecated.
+func (s *SignHTTPRequestMiddleware) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	if !haveCredentialProvider(s.credentialsProvider) {
+		return next.HandleFinalize(ctx, in)
+	}
+
+	req, ok := in.Request.(*smithyhttp.Request)
+	if !ok {
+		return out, metadata, &SigningError{Err: fmt.Errorf("unexpected request middleware type %T", in.Request)}
+	}
+
+	signingName, signingRegion := awsmiddleware.GetSigningName(ctx), awsmiddleware.GetSigningRegion(ctx)
+	payloadHash := GetPayloadHash(ctx)
+	if len(payloadHash) == 0 {
+		return out, metadata, &SigningError{Err: fmt.Errorf("computed payload hash missing from context")}
+	}
+
+	mctx := metrics.Context(ctx)
+
+	if mctx != nil {
+		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
+			attempt.CredentialFetchStartTime = sdk.NowTime()
+		}
+	}
+
+	credentials, err := s.credentialsProvider.Retrieve(ctx)
+
+	if mctx != nil {
+		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
+			attempt.CredentialFetchEndTime = sdk.NowTime()
+		}
+	}
+
+	if err != nil {
+		return out, metadata, &SigningError{Err: fmt.Errorf("failed to retrieve credentials: %w", err)}
+	}
+
+	signerOptions := []func(o *SignerOptions){
+		func(o *SignerOptions) {
+			o.Logger = middleware.GetLogger(ctx)
+			o.LogSigning = s.logSigning
+		},
+	}
+
+	// existing DisableURIPathEscaping is equivalent in purpose
+	// to authentication scheme property DisableDoubleEncoding
+	disableDoubleEncoding, overridden := internalauth.GetDisableDoubleEncoding(ctx)
+	if overridden {
+		signerOptions = append(signerOptions, func(o *SignerOptions) {
+			o.DisableURIPathEscaping = disableDoubleEncoding
+		})
+	}
+
+	if mctx != nil {
+		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
+			attempt.SignStartTime = sdk.NowTime()
+		}
+	}
+
+	err = s.signer.SignHTTP(ctx, credentials, req.Request, payloadHash, signingName, signingRegion, sdk.NowTime(), signerOptions...)
+
+	if mctx != nil {
+		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
+			attempt.SignEndTime = sdk.NowTime()
+		}
+	}
+
+	if err != nil {
+		return out, metadata, &SigningError{Err: fmt.Errorf("failed to sign http request, %w", err)}
+	}
+
+	ctx = awsmiddleware.SetSigningCredentials(ctx, credentials)
+
+	return next.HandleFinalize(ctx, in)
+}
+
+// StreamingEventsPayload signs input event stream messages.
+type StreamingEventsPayload struct{}
+
+// AddStreamingEventsPayload adds the streamingEventsPayload middleware to the stack.
+func AddStreamingEventsPayload(stack *middleware.Stack) error {
+	return stack.Finalize.Add(&StreamingEventsPayload{}, middleware.Before)
+}
+
+// ID identifies the middleware.
+func (s *StreamingEventsPayload) ID() string {
+	return computePayloadHashMiddlewareID
+}
+
+// HandleFinalize marks the input stream to be signed with SigV4.
+func (s *StreamingEventsPayload) HandleFinalize(
+	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	contentSHA := GetPayloadHash(ctx)
+	if len(contentSHA) == 0 {
+		contentSHA = v4Internal.StreamingEventsPayload
+	}
+
+	ctx = SetPayloadHash(ctx, contentSHA)
+
+	return next.HandleFinalize(ctx, in)
+}
+
+// GetSignedRequestSignature attempts to extract the signature of the request.
+// Returning an error if the request is unsigned, or unable to extract the
+// signature.
+func GetSignedRequestSignature(r *http.Request) ([]byte, error) {
+	const authHeaderSignatureElem = "Signature="
+
+	if auth := r.Header.Get(authorizationHeader); len(auth) != 0 {
+		ps := strings.Split(auth, ", ")
+		for _, p := range ps {
+			if idx := strings.Index(p, authHeaderSignatureElem); idx >= 0 {
+				sig := p[len(authHeaderSignatureElem):]
+				if len(sig) == 0 {
+					return nil, fmt.Errorf("invalid request signature authorization header")
+				}
+				return hex.DecodeString(sig)
+			}
+		}
+	}
+
+	if sig := r.URL.Query().Get("X-Amz-Signature"); len(sig) != 0 {
+		return hex.DecodeString(sig)
+	}
+
+	return nil, fmt.Errorf("request not signed")
+}
+
+func haveCredentialProvider(p aws.CredentialsProvider) bool {
+	if p == nil {
+		return false
+	}
+
+	return !aws.IsCredentialsProvider(p, (*aws.AnonymousCredentials)(nil))
+}
+
+type payloadHashKey struct{}
+
+// GetPayloadHash retrieves the payload hash to use for signing
+//
+// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
+// to clear all stack values.
+func GetPayloadHash(ctx context.Context) (v string) {
+	v, _ = middleware.GetStackValue(ctx, payloadHashKey{}).(string)
+	return v
+}
+
+// SetPayloadHash sets the payload hash to be used for signing the request
+//
+// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
+// to clear all stack values.
+func SetPayloadHash(ctx context.Context, hash string) context.Context {
+	return middleware.WithStackValue(ctx, payloadHashKey{}, hash)
+}

aws/signer/v4/middleware_test.go

@@ -0,0 +1,415 @@
+package v4
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+	"github.com/aws/smithy-go/logging"
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/google/go-cmp/cmp"
+	"github.com/versity/versitygw/aws/internal/awstesting/unit"
+)
+
+func TestComputePayloadHashMiddleware(t *testing.T) {
+	cases := []struct {
+		content      io.Reader
+		expectedHash string
+		expectedErr  interface{}
+	}{
+		0: {
+			content: func() io.Reader {
+				br := bytes.NewReader([]byte("some content"))
+				return br
+			}(),
+			expectedHash: "290f493c44f5d63d06b374d0a5abd292fae38b92cab2fae5efefe1b0e9347f56",
+		},
+		1: {
+			content: func() io.Reader {
+				return &nonSeeker{}
+			}(),
+			expectedErr: &HashComputationError{},
+		},
+		2: {
+			content: func() io.Reader {
+				return &semiSeekable{}
+			}(),
+			expectedErr: &HashComputationError{},
+		},
+	}
+
+	for i, tt := range cases {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			c := &ComputePayloadSHA256{}
+
+			next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+				value := GetPayloadHash(ctx)
+				if len(value) == 0 {
+					t.Fatalf("expected payload hash value to be on context")
+				}
+				if e, a := tt.expectedHash, value; e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+
+				return out, metadata, err
+			})
+
+			stream, err := smithyhttp.NewStackRequest().(*smithyhttp.Request).SetStream(tt.content)
+			if err != nil {
+				t.Fatalf("expected no error, got %v", err)
+			}
+
+			_, _, err = c.HandleFinalize(context.Background(), middleware.FinalizeInput{Request: stream}, next)
+			if err != nil && tt.expectedErr == nil {
+				t.Errorf("expected no error, got %v", err)
+			} else if err != nil && tt.expectedErr != nil {
+				e, a := tt.expectedErr, err
+				if !errors.As(a, &e) {
+					t.Errorf("expected error type %T, got %T", e, a)
+				}
+			} else if err == nil && tt.expectedErr != nil {
+				t.Errorf("expected error, got nil")
+			}
+		})
+	}
+}
+
+type httpSignerFunc func(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
+
+func (f httpSignerFunc) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error {
+	return f(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...)
+}
+
+func TestSignHTTPRequestMiddleware(t *testing.T) {
+	cases := map[string]struct {
+		creds       aws.CredentialsProvider
+		hash        string
+		logSigning  bool
+		expectedErr interface{}
+	}{
+		"success": {
+			creds: unit.StubCredentialsProvider{},
+			hash:  "0123456789abcdef",
+		},
+		"error": {
+			creds:       unit.StubCredentialsProvider{},
+			hash:        "",
+			expectedErr: &SigningError{},
+		},
+		"anonymous creds": {
+			creds: aws.AnonymousCredentials{},
+		},
+		"nil creds": {
+			creds: nil,
+		},
+		"with log signing": {
+			creds:      unit.StubCredentialsProvider{},
+			hash:       "0123456789abcdef",
+			logSigning: true,
+		},
+	}
+
+	const (
+		signingName   = "serviceId"
+		signingRegion = "regionName"
+	)
+
+	for name, tt := range cases {
+		t.Run(name, func(t *testing.T) {
+			c := &SignHTTPRequestMiddleware{
+				credentialsProvider: tt.creds,
+				signer: httpSignerFunc(
+					func(ctx context.Context,
+						credentials aws.Credentials, r *http.Request, payloadHash string,
+						service string, region string, signingTime time.Time,
+						optFns ...func(*SignerOptions),
+					) error {
+						var options SignerOptions
+						for _, fn := range optFns {
+							fn(&options)
+						}
+						if options.Logger == nil {
+							t.Errorf("expect logger, got none")
+						}
+						if options.LogSigning {
+							options.Logger.Logf(logging.Debug, t.Name())
+						}
+
+						expectCreds, _ := unit.StubCredentialsProvider{}.Retrieve(context.Background())
+						if e, a := expectCreds, credentials; e != a {
+							t.Errorf("expected %v, got %v", e, a)
+						}
+						if e, a := tt.hash, payloadHash; e != a {
+							t.Errorf("expected %v, got %v", e, a)
+						}
+						if e, a := signingName, service; e != a {
+							t.Errorf("expected %v, got %v", e, a)
+						}
+						if e, a := signingRegion, region; e != a {
+							t.Errorf("expected %v, got %v", e, a)
+						}
+						return nil
+					}),
+				logSigning: tt.logSigning,
+			}
+
+			next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+				return out, metadata, err
+			})
+
+			ctx := awsmiddleware.SetSigningRegion(
+				awsmiddleware.SetSigningName(context.Background(), signingName),
+				signingRegion)
+
+			var loggerBuf bytes.Buffer
+			logger := logging.NewStandardLogger(&loggerBuf)
+			ctx = middleware.SetLogger(ctx, logger)
+
+			if len(tt.hash) != 0 {
+				ctx = SetPayloadHash(ctx, tt.hash)
+			}
+
+			_, _, err := c.HandleFinalize(ctx, middleware.FinalizeInput{
+				Request: &smithyhttp.Request{Request: &http.Request{}},
+			}, next)
+			if err != nil && tt.expectedErr == nil {
+				t.Errorf("expected no error, got %v", err)
+			} else if err != nil && tt.expectedErr != nil {
+				e, a := tt.expectedErr, err
+				if !errors.As(a, &e) {
+					t.Errorf("expected error type %T, got %T", e, a)
+				}
+			} else if err == nil && tt.expectedErr != nil {
+				t.Errorf("expected error, got nil")
+			}
+
+			if tt.logSigning {
+				if e, a := t.Name(), loggerBuf.String(); !strings.Contains(a, e) {
+					t.Errorf("expect %v logged in %v", e, a)
+				}
+			} else {
+				if loggerBuf.Len() != 0 {
+					t.Errorf("expect no log, got %v", loggerBuf.String())
+				}
+			}
+		})
+	}
+}
+
+func TestSwapComputePayloadSHA256ForUnsignedPayloadMiddleware(t *testing.T) {
+	cases := map[string]struct {
+		InitStep  func(*middleware.Stack) error
+		Mutator   func(*middleware.Stack) error
+		ExpectErr string
+		ExpectIDs []string
+	}{
+		"swap in place": {
+			InitStep: func(s *middleware.Stack) (err error) {
+				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
+				if err != nil {
+					return err
+				}
+				err = AddComputePayloadSHA256Middleware(s)
+				if err != nil {
+					return err
+				}
+				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
+				if err != nil {
+					return err
+				}
+				return nil
+			},
+			Mutator: SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+			ExpectIDs: []string{
+				"ResolveEndpointV2",
+				computePayloadHashMiddlewareID, // should snap to after resolve endpoint
+				"before",
+				"after",
+			},
+		},
+
+		"already unsigned payload exists": {
+			InitStep: func(s *middleware.Stack) (err error) {
+				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
+				if err != nil {
+					return err
+				}
+				err = AddUnsignedPayloadMiddleware(s)
+				if err != nil {
+					return err
+				}
+				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
+				if err != nil {
+					return err
+				}
+				return nil
+			},
+			Mutator: SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+			ExpectIDs: []string{
+				"ResolveEndpointV2",
+				computePayloadHashMiddlewareID,
+				"before",
+				"after",
+			},
+		},
+
+		"no compute payload": {
+			InitStep: func(s *middleware.Stack) (err error) {
+				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
+				if err != nil {
+					return err
+				}
+				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
+				if err != nil {
+					return err
+				}
+				return nil
+			},
+			Mutator:   SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+			ExpectErr: "not found, " + computePayloadHashMiddlewareID,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			stack := middleware.NewStack(t.Name(), smithyhttp.NewStackRequest)
+			stack.Finalize.Add(&nopResolveEndpoint{}, middleware.After)
+			if err := c.InitStep(stack); err != nil {
+				t.Fatalf("expect no error, got %v", err)
+			}
+
+			err := c.Mutator(stack)
+			if len(c.ExpectErr) != 0 {
+				if err == nil {
+					t.Fatalf("expect error, got none")
+				}
+				if e, a := c.ExpectErr, err.Error(); !strings.Contains(a, e) {
+					t.Fatalf("expect error to contain %v, got %v", e, a)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("expect no error, got %v", err)
+			}
+
+			if diff := cmp.Diff(c.ExpectIDs, stack.Finalize.List()); len(diff) != 0 {
+				t.Errorf("expect match\n%v", diff)
+			}
+		})
+	}
+}
+
+func TestUseDynamicPayloadSigningMiddleware(t *testing.T) {
+	cases := map[string]struct {
+		content      io.Reader
+		url          string
+		expectedHash string
+		expectedErr  interface{}
+	}{
+		"TLS disabled": {
+			content: func() io.Reader {
+				br := bytes.NewReader([]byte("some content"))
+				return br
+			}(),
+			url:          "http://localhost.com/",
+			expectedHash: "290f493c44f5d63d06b374d0a5abd292fae38b92cab2fae5efefe1b0e9347f56",
+		},
+		"TLS enabled": {
+			content: func() io.Reader {
+				br := bytes.NewReader([]byte("some content"))
+				return br
+			}(),
+			url:          "https://localhost.com/",
+			expectedHash: "UNSIGNED-PAYLOAD",
+		},
+	}
+
+	for name, tt := range cases {
+		t.Run(name, func(t *testing.T) {
+			c := &dynamicPayloadSigningMiddleware{}
+
+			next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+				value := GetPayloadHash(ctx)
+				if len(value) == 0 {
+					t.Fatalf("expected payload hash value to be on context")
+				}
+				if e, a := tt.expectedHash, value; e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+
+				return out, metadata, err
+			})
+
+			req := smithyhttp.NewStackRequest().(*smithyhttp.Request)
+			req.URL, _ = url.Parse(tt.url)
+			stream, err := req.SetStream(tt.content)
+			if err != nil {
+				t.Fatalf("expected no error, got %v", err)
+			}
+
+			_, _, err = c.HandleFinalize(context.Background(), middleware.FinalizeInput{Request: stream}, next)
+			if err != nil && tt.expectedErr == nil {
+				t.Errorf("expected no error, got %v", err)
+			} else if err != nil && tt.expectedErr != nil {
+				e, a := tt.expectedErr, err
+				if !errors.As(a, &e) {
+					t.Errorf("expected error type %T, got %T", e, a)
+				}
+			} else if err == nil && tt.expectedErr != nil {
+				t.Errorf("expected error, got nil")
+			}
+		})
+	}
+}
+
+type nonSeeker struct{}
+
+func (nonSeeker) Read(p []byte) (n int, err error) {
+	return 0, io.EOF
+}
+
+type semiSeekable struct {
+	hasSeeked bool
+}
+
+func (s *semiSeekable) Seek(offset int64, whence int) (int64, error) {
+	if !s.hasSeeked {
+		s.hasSeeked = true
+		return 0, nil
+	}
+	return 0, fmt.Errorf("io seek error")
+}
+
+func (*semiSeekable) Read(p []byte) (n int, err error) {
+	return 0, io.EOF
+}
+
+type nopResolveEndpoint struct{}
+
+func (*nopResolveEndpoint) ID() string { return "ResolveEndpointV2" }
+
+func (*nopResolveEndpoint) HandleFinalize(
+	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	return out, metadata, err
+}
+
+var (
+	_ middleware.FinalizeMiddleware = &UnsignedPayload{}
+	_ middleware.FinalizeMiddleware = &ComputePayloadSHA256{}
+	_ middleware.FinalizeMiddleware = &ContentSHA256Header{}
+	_ middleware.FinalizeMiddleware = &SignHTTPRequestMiddleware{}
+)

aws/signer/v4/presign_middleware.go

@@ -0,0 +1,127 @@
+package v4
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+	"github.com/aws/smithy-go/middleware"
+	smithyHTTP "github.com/aws/smithy-go/transport/http"
+	"github.com/versity/versitygw/aws/internal/sdk"
+)
+
+// HTTPPresigner is an interface to a SigV4 signer that can sign create a
+// presigned URL for a HTTP requests.
+type HTTPPresigner interface {
+	PresignHTTP(
+		ctx context.Context, credentials aws.Credentials, r *http.Request,
+		payloadHash string, service string, region string, signingTime time.Time,
+		optFns ...func(*SignerOptions),
+	) (url string, signedHeader http.Header, err error)
+}
+
+// PresignedHTTPRequest provides the URL and signed headers that are included
+// in the presigned URL.
+type PresignedHTTPRequest struct {
+	URL          string
+	Method       string
+	SignedHeader http.Header
+}
+
+// PresignHTTPRequestMiddlewareOptions is the options for the PresignHTTPRequestMiddleware middleware.
+type PresignHTTPRequestMiddlewareOptions struct {
+	CredentialsProvider aws.CredentialsProvider
+	Presigner           HTTPPresigner
+	LogSigning          bool
+}
+
+// PresignHTTPRequestMiddleware provides the Finalize middleware for creating a
+// presigned URL for an HTTP request.
+//
+// Will short circuit the middleware stack and not forward onto the next
+// Finalize handler.
+type PresignHTTPRequestMiddleware struct {
+	credentialsProvider aws.CredentialsProvider
+	presigner           HTTPPresigner
+	logSigning          bool
+}
+
+// NewPresignHTTPRequestMiddleware returns a new PresignHTTPRequestMiddleware
+// initialized with the presigner.
+func NewPresignHTTPRequestMiddleware(options PresignHTTPRequestMiddlewareOptions) *PresignHTTPRequestMiddleware {
+	return &PresignHTTPRequestMiddleware{
+		credentialsProvider: options.CredentialsProvider,
+		presigner:           options.Presigner,
+		logSigning:          options.LogSigning,
+	}
+}
+
+// ID provides the middleware ID.
+func (*PresignHTTPRequestMiddleware) ID() string { return "PresignHTTPRequest" }
+
+// HandleFinalize will take the provided input and create a presigned url for
+// the http request using the SigV4 presign authentication scheme.
+//
+// Since the signed request is not a valid HTTP request
+func (s *PresignHTTPRequestMiddleware) HandleFinalize(
+	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	req, ok := in.Request.(*smithyHTTP.Request)
+	if !ok {
+		return out, metadata, &SigningError{
+			Err: fmt.Errorf("unexpected request middleware type %T", in.Request),
+		}
+	}
+
+	httpReq := req.Build(ctx)
+	if !haveCredentialProvider(s.credentialsProvider) {
+		out.Result = &PresignedHTTPRequest{
+			URL:          httpReq.URL.String(),
+			Method:       httpReq.Method,
+			SignedHeader: http.Header{},
+		}
+
+		return out, metadata, nil
+	}
+
+	signingName := awsmiddleware.GetSigningName(ctx)
+	signingRegion := awsmiddleware.GetSigningRegion(ctx)
+	payloadHash := GetPayloadHash(ctx)
+	if len(payloadHash) == 0 {
+		return out, metadata, &SigningError{
+			Err: fmt.Errorf("computed payload hash missing from context"),
+		}
+	}
+
+	credentials, err := s.credentialsProvider.Retrieve(ctx)
+	if err != nil {
+		return out, metadata, &SigningError{
+			Err: fmt.Errorf("failed to retrieve credentials: %w", err),
+		}
+	}
+
+	u, h, err := s.presigner.PresignHTTP(ctx, credentials,
+		httpReq, payloadHash, signingName, signingRegion, sdk.NowTime(),
+		func(o *SignerOptions) {
+			o.Logger = middleware.GetLogger(ctx)
+			o.LogSigning = s.logSigning
+		})
+	if err != nil {
+		return out, metadata, &SigningError{
+			Err: fmt.Errorf("failed to sign http request, %w", err),
+		}
+	}
+
+	out.Result = &PresignedHTTPRequest{
+		URL:          u,
+		Method:       httpReq.Method,
+		SignedHeader: h,
+	}
+
+	return out, metadata, nil
+}

aws/signer/v4/presign_middleware_test.go

@@ -0,0 +1,224 @@
+package v4
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+	"github.com/aws/smithy-go/logging"
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/google/go-cmp/cmp"
+	"github.com/versity/versitygw/aws/internal/awstesting/unit"
+)
+
+type httpPresignerFunc func(
+	ctx context.Context, credentials aws.Credentials, r *http.Request,
+	payloadHash string, service string, region string, signingTime time.Time,
+	optFns ...func(*SignerOptions),
+) (url string, signedHeader http.Header, err error)
+
+func (f httpPresignerFunc) PresignHTTP(
+	ctx context.Context, credentials aws.Credentials, r *http.Request,
+	payloadHash string, service string, region string, signingTime time.Time,
+	optFns ...func(*SignerOptions),
+) (
+	url string, signedHeader http.Header, err error,
+) {
+	return f(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...)
+}
+
+func TestPresignHTTPRequestMiddleware(t *testing.T) {
+	cases := map[string]struct {
+		Request      *http.Request
+		Creds        aws.CredentialsProvider
+		PayloadHash  string
+		LogSigning   bool
+		ExpectResult *PresignedHTTPRequest
+		ExpectErr    string
+	}{
+		"success": {
+			Request: &http.Request{
+				URL: func() *url.URL {
+					u, _ := url.Parse("https://example.aws/path?query=foo")
+					return u
+				}(),
+				Header: http.Header{},
+			},
+			Creds:       unit.StubCredentialsProvider{},
+			PayloadHash: "0123456789abcdef",
+			ExpectResult: &PresignedHTTPRequest{
+				URL:          "https://example.aws/path?query=foo",
+				SignedHeader: http.Header{},
+			},
+		},
+		"error": {
+			Request: func() *http.Request {
+				return &http.Request{}
+			}(),
+			Creds:       unit.StubCredentialsProvider{},
+			PayloadHash: "",
+			ExpectErr:   "failed to sign request",
+		},
+		"anonymous creds": {
+			Request: &http.Request{
+				URL: func() *url.URL {
+					u, _ := url.Parse("https://example.aws/path?query=foo")
+					return u
+				}(),
+				Header: http.Header{},
+			},
+			Creds:       unit.StubCredentialsProvider{},
+			PayloadHash: "",
+			ExpectErr:   "failed to sign request",
+			ExpectResult: &PresignedHTTPRequest{
+				URL:          "https://example.aws/path?query=foo",
+				SignedHeader: http.Header{},
+			},
+		},
+		"nil creds": {
+			Request: &http.Request{
+				URL: func() *url.URL {
+					u, _ := url.Parse("https://example.aws/path?query=foo")
+					return u
+				}(),
+				Header: http.Header{},
+			},
+			Creds: nil,
+			ExpectResult: &PresignedHTTPRequest{
+				URL:          "https://example.aws/path?query=foo",
+				SignedHeader: http.Header{},
+			},
+		},
+		"with log signing": {
+			Request: &http.Request{
+				URL: func() *url.URL {
+					u, _ := url.Parse("https://example.aws/path?query=foo")
+					return u
+				}(),
+				Header: http.Header{},
+			},
+			Creds:       unit.StubCredentialsProvider{},
+			PayloadHash: "0123456789abcdef",
+			ExpectResult: &PresignedHTTPRequest{
+				URL:          "https://example.aws/path?query=foo",
+				SignedHeader: http.Header{},
+			},
+
+			LogSigning: true,
+		},
+	}
+
+	const (
+		signingName   = "serviceId"
+		signingRegion = "regionName"
+	)
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			m := &PresignHTTPRequestMiddleware{
+				credentialsProvider: c.Creds,
+
+				presigner: httpPresignerFunc(func(
+					ctx context.Context, credentials aws.Credentials, r *http.Request,
+					payloadHash string, service string, region string, signingTime time.Time,
+					optFns ...func(*SignerOptions),
+				) (url string, signedHeader http.Header, err error) {
+					var options SignerOptions
+					for _, fn := range optFns {
+						fn(&options)
+					}
+					if options.Logger == nil {
+						t.Errorf("expect logger, got none")
+					}
+					if options.LogSigning {
+						options.Logger.Logf(logging.Debug, t.Name())
+					}
+
+					if !haveCredentialProvider(c.Creds) {
+						t.Errorf("expect presigner not to be called for not credentials provider")
+					}
+
+					expectCreds, _ := unit.StubCredentialsProvider{}.Retrieve(context.Background())
+					if e, a := expectCreds, credentials; e != a {
+						t.Errorf("expected %v, got %v", e, a)
+					}
+					if e, a := c.PayloadHash, payloadHash; e != a {
+						t.Errorf("expected %v, got %v", e, a)
+					}
+					if e, a := signingName, service; e != a {
+						t.Errorf("expected %v, got %v", e, a)
+					}
+					if e, a := signingRegion, region; e != a {
+						t.Errorf("expected %v, got %v", e, a)
+					}
+
+					return c.ExpectResult.URL, c.ExpectResult.SignedHeader, nil
+				}),
+				logSigning: c.LogSigning,
+			}
+
+			next := middleware.FinalizeHandlerFunc(
+				func(ctx context.Context, in middleware.FinalizeInput) (
+					out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+				) {
+					t.Errorf("expect next handler not to be called")
+					return out, metadata, err
+				})
+
+			ctx := awsmiddleware.SetSigningRegion(
+				awsmiddleware.SetSigningName(context.Background(), signingName),
+				signingRegion)
+
+			var loggerBuf bytes.Buffer
+			logger := logging.NewStandardLogger(&loggerBuf)
+			ctx = middleware.SetLogger(ctx, logger)
+
+			if len(c.PayloadHash) != 0 {
+				ctx = SetPayloadHash(ctx, c.PayloadHash)
+			}
+
+			result, _, err := m.HandleFinalize(ctx, middleware.FinalizeInput{
+				Request: &smithyhttp.Request{
+					Request: c.Request,
+				},
+			}, next)
+			if len(c.ExpectErr) != 0 {
+				if err == nil {
+					t.Fatalf("expect error, got none")
+				}
+				if e, a := c.ExpectErr, err.Error(); !strings.Contains(a, e) {
+					t.Fatalf("expect error to contain %v, got %v", e, a)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("expect no error, got %v", err)
+			}
+
+			if diff := cmp.Diff(c.ExpectResult, result.Result); len(diff) != 0 {
+				t.Errorf("expect result match\n%v", diff)
+			}
+
+			if c.LogSigning {
+				if e, a := t.Name(), loggerBuf.String(); !strings.Contains(a, e) {
+					t.Errorf("expect %v logged in %v", e, a)
+				}
+			} else {
+				if loggerBuf.Len() != 0 {
+					t.Errorf("expect no log, got %v", loggerBuf.String())
+				}
+			}
+		})
+	}
+}
+
+var (
+	_ middleware.FinalizeMiddleware = &PresignHTTPRequestMiddleware{}
+)

aws/signer/v4/stream.go

@@ -0,0 +1,87 @@
+package v4
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+// EventStreamSigner is an AWS EventStream protocol signer.
+type EventStreamSigner interface {
+	GetSignature(ctx context.Context, headers, payload []byte, signingTime time.Time, optFns ...func(*StreamSignerOptions)) ([]byte, error)
+}
+
+// StreamSignerOptions is the configuration options for StreamSigner.
+type StreamSignerOptions struct{}
+
+// StreamSigner implements Signature Version 4 (SigV4) signing of event stream encoded payloads.
+type StreamSigner struct {
+	options StreamSignerOptions
+
+	credentials aws.Credentials
+	service     string
+	region      string
+
+	prevSignature []byte
+
+	signingKeyDeriver *v4Internal.SigningKeyDeriver
+}
+
+// NewStreamSigner returns a new AWS EventStream protocol signer.
+func NewStreamSigner(credentials aws.Credentials, service, region string, seedSignature []byte, optFns ...func(*StreamSignerOptions)) *StreamSigner {
+	o := StreamSignerOptions{}
+
+	for _, fn := range optFns {
+		fn(&o)
+	}
+
+	return &StreamSigner{
+		options:           o,
+		credentials:       credentials,
+		service:           service,
+		region:            region,
+		signingKeyDeriver: v4Internal.NewSigningKeyDeriver(),
+		prevSignature:     seedSignature,
+	}
+}
+
+// GetSignature signs the provided header and payload bytes.
+func (s *StreamSigner) GetSignature(ctx context.Context, headers, payload []byte, signingTime time.Time, optFns ...func(*StreamSignerOptions)) ([]byte, error) {
+	options := s.options
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	prevSignature := s.prevSignature
+
+	st := v4Internal.NewSigningTime(signingTime)
+
+	sigKey := s.signingKeyDeriver.DeriveKey(s.credentials, s.service, s.region, st)
+
+	scope := v4Internal.BuildCredentialScope(st, s.region, s.service)
+
+	stringToSign := s.buildEventStreamStringToSign(headers, payload, prevSignature, scope, &st)
+
+	signature := v4Internal.HMACSHA256(sigKey, []byte(stringToSign))
+	s.prevSignature = signature
+
+	return signature, nil
+}
+
+func (s *StreamSigner) buildEventStreamStringToSign(headers, payload, previousSignature []byte, credentialScope string, signingTime *v4Internal.SigningTime) string {
+	hash := sha256.New()
+	return strings.Join([]string{
+		"AWS4-HMAC-SHA256-PAYLOAD",
+		signingTime.TimeFormat(),
+		credentialScope,
+		hex.EncodeToString(previousSignature),
+		hex.EncodeToString(makeHash(hash, headers)),
+		hex.EncodeToString(makeHash(hash, payload)),
+	}, "\n")
+}

aws/signer/v4/v4.go

@@ -0,0 +1,565 @@
+// Package v4 implements signing for AWS V4 signer
+//
+// Provides request signing for request that need to be signed with
+// AWS V4 Signatures.
+//
+// # Standalone Signer
+//
+// Generally using the signer outside of the SDK should not require any additional
+//
+//	The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires
+//
+// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent
+// to the service as.
+//
+// The signer will first check the URL.Opaque field, and use its value if set.
+// The signer does require the URL.Opaque field to be set in the form of:
+//
+//	"//<hostname>/<path>"
+//
+//	// e.g.
+//	"//example.com/some/path"
+//
+// The leading "//" and hostname are required or the URL.Opaque escaping will
+// not work correctly.
+//
+// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath()
+// method and using the returned value.
+//
+// AWS v4 signature validation requires that the canonical string's URI path
+// element must be the URI escaped form of the HTTP request's path.
+// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+//
+// The Go HTTP client will perform escaping automatically on the request. Some
+// of these escaping may cause signature validation errors because the HTTP
+// request differs from the URI path or query that the signature was generated.
+// https://golang.org/pkg/net/url/#URL.EscapedPath
+//
+// Because of this, it is recommended that when using the signer outside of the
+// SDK that explicitly escaping the request prior to being signed is preferable,
+// and will help prevent signature validation errors. This can be done by setting
+// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then
+// call URL.EscapedPath() if Opaque is not set.
+//
+// Test `TestStandaloneSign` provides a complete example of using the signer
+// outside of the SDK and pre-escaping the URI path.
+package v4
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"net/http"
+	"net/textproto"
+	"net/url"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go/encoding/httpbinding"
+	"github.com/aws/smithy-go/logging"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+const (
+	signingAlgorithm    = "AWS4-HMAC-SHA256"
+	authorizationHeader = "Authorization"
+
+	// Version of signing v4
+	Version = "SigV4"
+)
+
+// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests
+type HTTPSigner interface {
+	SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
+}
+
+type keyDerivator interface {
+	DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte
+}
+
+// SignerOptions is the SigV4 Signer options.
+type SignerOptions struct {
+	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
+	// request header to the request's query string. This is most commonly used
+	// with pre-signed requests preventing headers from being added to the
+	// request's query string.
+	DisableHeaderHoisting bool
+
+	// Disables the automatic escaping of the URI path of the request for the
+	// siganture's canonical string's path. For services that do not need additional
+	// escaping then use this to disable the signer escaping the path.
+	//
+	// S3 is an example of a service that does not need additional escaping.
+	//
+	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+	DisableURIPathEscaping bool
+
+	// The logger to send log messages to.
+	Logger logging.Logger
+
+	// Enable logging of signed requests.
+	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
+	// presigned URL.
+	LogSigning bool
+
+	// Disables setting the session token on the request as part of signing
+	// through X-Amz-Security-Token. This is needed for variations of v4 that
+	// present the token elsewhere.
+	DisableSessionToken bool
+}
+
+// Signer applies AWS v4 signing to given request. Use this to sign requests
+// that need to be signed with AWS V4 Signatures.
+type Signer struct {
+	options      SignerOptions
+	keyDerivator keyDerivator
+}
+
+// NewSigner returns a new SigV4 Signer
+func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
+	options := SignerOptions{}
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()}
+}
+
+type httpSigner struct {
+	Request      *http.Request
+	ServiceName  string
+	Region       string
+	Time         v4Internal.SigningTime
+	Credentials  aws.Credentials
+	KeyDerivator keyDerivator
+	IsPreSign    bool
+	SignedHdrs   []string
+
+	PayloadHash string
+
+	DisableHeaderHoisting  bool
+	DisableURIPathEscaping bool
+	DisableSessionToken    bool
+}
+
+func (s *httpSigner) Build() (signedRequest, error) {
+	req := s.Request
+
+	query := req.URL.Query()
+	headers := req.Header
+
+	s.setRequiredSigningFields(headers, query)
+
+	// Sort Each Query Key's Values
+	for key := range query {
+		sort.Strings(query[key])
+	}
+
+	v4Internal.SanitizeHostForHeader(req)
+
+	credentialScope := s.buildCredentialScope()
+	credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzCredentialKey, credentialStr)
+	}
+
+	unsignedHeaders := headers
+	if s.IsPreSign && !s.DisableHeaderHoisting {
+		var urlValues url.Values
+		urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers)
+		for k := range urlValues {
+			query[k] = urlValues[k]
+		}
+	}
+
+	host := req.URL.Host
+	if len(req.Host) > 0 {
+		host = req.Host
+	}
+
+	signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength)
+
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr)
+	}
+
+	var rawQuery strings.Builder
+	rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1))
+
+	canonicalURI := v4Internal.GetURIPath(req.URL)
+	if !s.DisableURIPathEscaping {
+		canonicalURI = httpbinding.EscapePath(canonicalURI, false)
+	}
+
+	canonicalString := s.buildCanonicalString(
+		req.Method,
+		canonicalURI,
+		rawQuery.String(),
+		signedHeadersStr,
+		canonicalHeaderStr,
+	)
+
+	strToSign := s.buildStringToSign(credentialScope, canonicalString)
+	signingSignature, err := s.buildSignature(strToSign)
+	if err != nil {
+		return signedRequest{}, err
+	}
+
+	if s.IsPreSign {
+		rawQuery.WriteString("&X-Amz-Signature=")
+		rawQuery.WriteString(signingSignature)
+	} else {
+		headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature))
+	}
+
+	req.URL.RawQuery = rawQuery.String()
+
+	return signedRequest{
+		Request:         req,
+		SignedHeaders:   signedHeaders,
+		CanonicalString: canonicalString,
+		StringToSign:    strToSign,
+		PreSigned:       s.IsPreSign,
+	}, nil
+}
+
+func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string {
+	const credential = "Credential="
+	const signedHeaders = "SignedHeaders="
+	const signature = "Signature="
+	const commaSpace = ", "
+
+	var parts strings.Builder
+	parts.Grow(len(signingAlgorithm) + 1 +
+		len(credential) + len(credentialStr) + 2 +
+		len(signedHeaders) + len(signedHeadersStr) + 2 +
+		len(signature) + len(signingSignature),
+	)
+	parts.WriteString(signingAlgorithm)
+	parts.WriteRune(' ')
+	parts.WriteString(credential)
+	parts.WriteString(credentialStr)
+	parts.WriteString(commaSpace)
+	parts.WriteString(signedHeaders)
+	parts.WriteString(signedHeadersStr)
+	parts.WriteString(commaSpace)
+	parts.WriteString(signature)
+	parts.WriteString(signingSignature)
+	return parts.String()
+}
+
+// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the
+// request is made to, and time the request is signed at. The signTime allows
+// you to specify that a request is signed for the future, and cannot be
+// used until then.
+//
+// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
+// must be provided. Even if the request has no payload (aka body). If the
+// request has no payload you should use the hex encoded SHA-256 of an empty
+// string as the payloadHash value.
+//
+//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+//
+// Some services such as Amazon S3 accept alternative values for the payload
+// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
+// included in the request signature.
+//
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+//
+// Sign differs from Presign in that it will sign the request using HTTP
+// header values. This type of signing is intended for http.Request values that
+// will not be shared, or are shared in a way the header values on the request
+// will not be lost.
+//
+// The passed in request will be modified in place.
+func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) error {
+	options := s.options
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	signer := &httpSigner{
+		Request:                r,
+		PayloadHash:            payloadHash,
+		ServiceName:            service,
+		Region:                 region,
+		Credentials:            credentials,
+		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
+		DisableHeaderHoisting:  options.DisableHeaderHoisting,
+		DisableURIPathEscaping: options.DisableURIPathEscaping,
+		DisableSessionToken:    options.DisableSessionToken,
+		KeyDerivator:           s.keyDerivator,
+		SignedHdrs:             signedHdrs,
+	}
+
+	signedRequest, err := signer.Build()
+	if err != nil {
+		return err
+	}
+
+	logSigningInfo(ctx, options, &signedRequest, false)
+
+	return nil
+}
+
+// PresignHTTP signs AWS v4 requests with the payload hash, service name, region
+// the request is made to, and time the request is signed at. The signTime
+// allows you to specify that a request is signed for the future, and cannot
+// be used until then.
+//
+// Returns the signed URL and the map of HTTP headers that were included in the
+// signature or an error if signing the request failed. For presigned requests
+// these headers and their values must be included on the HTTP request when it
+// is made. This is helpful to know what header values need to be shared with
+// the party the presigned request will be distributed to.
+//
+// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
+// must be provided. Even if the request has no payload (aka body). If the
+// request has no payload you should use the hex encoded SHA-256 of an empty
+// string as the payloadHash value.
+//
+//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+//
+// Some services such as Amazon S3 accept alternative values for the payload
+// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
+// included in the request signature.
+//
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+//
+// PresignHTTP differs from SignHTTP in that it will sign the request using
+// query string instead of header values. This allows you to share the
+// Presigned Request's URL with third parties, or distribute it throughout your
+// system with minimal dependencies.
+//
+// PresignHTTP will not set the expires time of the presigned request
+// automatically. To specify the expire duration for a request add the
+// "X-Amz-Expires" query parameter on the request with the value as the
+// duration in seconds the presigned URL should be considered valid for. This
+// parameter is not used by all AWS services, and is most notable used by
+// Amazon S3 APIs.
+//
+//	expires := 20 * time.Minute
+//	query := req.URL.Query()
+//	query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10))
+//	req.URL.RawQuery = query.Encode()
+//
+// This method does not modify the provided request.
+func (s *Signer) PresignHTTP(
+	ctx context.Context, credentials aws.Credentials, r *http.Request,
+	payloadHash string, service string, region string, signingTime time.Time,
+	signedHdrs []string,
+	optFns ...func(*SignerOptions),
+) (signedURI string, signedHeaders http.Header, err error) {
+	options := s.options
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	signer := &httpSigner{
+		Request:                r.Clone(r.Context()),
+		PayloadHash:            payloadHash,
+		ServiceName:            service,
+		Region:                 region,
+		Credentials:            credentials,
+		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
+		IsPreSign:              true,
+		DisableHeaderHoisting:  options.DisableHeaderHoisting,
+		DisableURIPathEscaping: options.DisableURIPathEscaping,
+		DisableSessionToken:    options.DisableSessionToken,
+		KeyDerivator:           s.keyDerivator,
+		SignedHdrs:             signedHdrs,
+	}
+
+	signedRequest, err := signer.Build()
+	if err != nil {
+		return "", nil, err
+	}
+
+	logSigningInfo(ctx, options, &signedRequest, true)
+
+	signedHeaders = make(http.Header)
+
+	// For the signed headers we canonicalize the header keys in the returned map.
+	// This avoids situations where can standard library double headers like host header. For example the standard
+	// library will set the Host header, even if it is present in lower-case form.
+	for k, v := range signedRequest.SignedHeaders {
+		key := textproto.CanonicalMIMEHeaderKey(k)
+		signedHeaders[key] = append(signedHeaders[key], v...)
+	}
+
+	return signedRequest.Request.URL.String(), signedHeaders, nil
+}
+
+func (s *httpSigner) buildCredentialScope() string {
+	return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName)
+}
+
+func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) {
+	query := url.Values{}
+	unsignedHeaders := http.Header{}
+	for k, h := range header {
+		if r.IsValid(k) {
+			query[k] = h
+		} else {
+			unsignedHeaders[k] = h
+		}
+	}
+
+	return query, unsignedHeaders
+}
+
+func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) {
+	signed = make(http.Header)
+
+	var headers []string
+	const hostHeader = "host"
+	headers = append(headers, hostHeader)
+	signed[hostHeader] = append(signed[hostHeader], host)
+
+	const contentLengthHeader = "content-length"
+	if slices.Contains(s.SignedHdrs, contentLengthHeader) {
+		headers = append(headers, contentLengthHeader)
+		signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10))
+	}
+
+	for k, v := range header {
+		if !rule.IsValid(k) {
+			continue // ignored header
+		}
+		if strings.EqualFold(k, contentLengthHeader) {
+			// prevent signing already handled content-length header.
+			continue
+		}
+
+		lowerCaseKey := strings.ToLower(k)
+		if _, ok := signed[lowerCaseKey]; ok {
+			// include additional values
+			signed[lowerCaseKey] = append(signed[lowerCaseKey], v...)
+			continue
+		}
+
+		headers = append(headers, lowerCaseKey)
+		signed[lowerCaseKey] = v
+	}
+	sort.Strings(headers)
+
+	signedHeaders = strings.Join(headers, ";")
+
+	var canonicalHeaders strings.Builder
+	n := len(headers)
+	const colon = ':'
+	for i := 0; i < n; i++ {
+		if headers[i] == hostHeader {
+			canonicalHeaders.WriteString(hostHeader)
+			canonicalHeaders.WriteRune(colon)
+			canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host))
+		} else {
+			canonicalHeaders.WriteString(headers[i])
+			canonicalHeaders.WriteRune(colon)
+			// Trim out leading, trailing, and dedup inner spaces from signed header values.
+			values := signed[headers[i]]
+			for j, v := range values {
+				cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v))
+				canonicalHeaders.WriteString(cleanedValue)
+				if j < len(values)-1 {
+					canonicalHeaders.WriteRune(',')
+				}
+			}
+		}
+		canonicalHeaders.WriteRune('\n')
+	}
+	canonicalHeadersStr = canonicalHeaders.String()
+
+	return signed, signedHeaders, canonicalHeadersStr
+}
+
+func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string {
+	return strings.Join([]string{
+		method,
+		uri,
+		query,
+		canonicalHeaders,
+		signedHeaders,
+		s.PayloadHash,
+	}, "\n")
+}
+
+func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string {
+	return strings.Join([]string{
+		signingAlgorithm,
+		s.Time.TimeFormat(),
+		credentialScope,
+		hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))),
+	}, "\n")
+}
+
+func makeHash(hash hash.Hash, b []byte) []byte {
+	hash.Reset()
+	hash.Write(b)
+	return hash.Sum(nil)
+}
+
+func (s *httpSigner) buildSignature(strToSign string) (string, error) {
+	key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time)
+	return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil
+}
+
+func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) {
+	amzDate := s.Time.TimeFormat()
+
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm)
+		sessionToken := s.Credentials.SessionToken
+		if !s.DisableSessionToken && len(sessionToken) > 0 {
+			query.Set("X-Amz-Security-Token", sessionToken)
+		}
+
+		query.Set(v4Internal.AmzDateKey, amzDate)
+		return
+	}
+
+	headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate)
+
+	if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 {
+		headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken)
+	}
+}
+
+func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) {
+	if !options.LogSigning {
+		return
+	}
+	signedURLMsg := ""
+	if isPresign {
+		signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String())
+	}
+	logger := logging.WithContext(ctx, options.Logger)
+	logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg)
+}
+
+type signedRequest struct {
+	Request         *http.Request
+	SignedHeaders   http.Header
+	CanonicalString string
+	StringToSign    string
+	PreSigned       bool
+}
+
+const logSignInfoMsg = `Request Signature:
+---[ CANONICAL STRING  ]-----------------------------
+%s
+---[ STRING TO SIGN ]--------------------------------
+%s%s
+-----------------------------------------------------`
+const logSignedURLMsg = `
+---[ SIGNED URL ]------------------------------------
+%s`

aws/signer/v4/v4_test.go

@@ -0,0 +1,359 @@
+package v4
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/google/go-cmp/cmp"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"}
+
+func buildRequest(serviceName, region, body string) (*http.Request, string) {
+	reader := strings.NewReader(body)
+	return buildRequestWithBodyReader(serviceName, region, reader)
+}
+
+func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) {
+	var bodyLen int
+
+	type lenner interface {
+		Len() int
+	}
+	if lr, ok := body.(lenner); ok {
+		bodyLen = lr.Len()
+	}
+
+	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
+	req, _ := http.NewRequest("POST", endpoint, body)
+	req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
+	req.Header.Set("X-Amz-Target", "prefix.Operation")
+	req.Header.Set("Content-Type", "application/x-amz-json-1.0")
+
+	if bodyLen > 0 {
+		req.ContentLength = int64(bodyLen)
+	}
+
+	req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)")
+	req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
+	req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
+
+	h := sha256.New()
+	_, _ = io.Copy(h, body)
+	payloadHash := hex.EncodeToString(h.Sum(nil))
+
+	return req, payloadHash
+}
+
+func TestPresignRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "300")
+	req.URL.RawQuery = query.Encode()
+
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	signer := NewSigner()
+	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
+	expectedSig := "122f0b9e091e4ba84286097e2b3404a1f1f4c4aad479adda95b7dff0ccbe5581"
+	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
+	expectedTarget := "prefix.Operation"
+
+	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
+	if err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+
+	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
+		t.Errorf("expect %v to be empty", a)
+	}
+	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+
+	for _, h := range strings.Split(expectedHeaders, ";") {
+		v := headers.Get(h)
+		if len(v) == 0 {
+			t.Errorf("expect %v, to be present in header map", h)
+		}
+	}
+}
+
+func TestPresignBodyWithArrayRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "300")
+	req.URL.RawQuery = query.Encode()
+
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	signer := NewSigner()
+	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
+	if err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
+	expectedSig := "e3ac55addee8711b76c6d608d762cff285fe8b627a057f8b5ec9268cf82c08b1"
+	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
+	expectedTarget := "prefix.Operation"
+
+	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
+		t.Errorf("expect %v to be empty, was not", a)
+	}
+	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+
+	for _, h := range strings.Split(expectedHeaders, ";") {
+		v := headers.Get(h)
+		if len(v) == 0 {
+			t.Errorf("expect %v, to be present in header map", h)
+		}
+	}
+}
+
+func TestSignRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+	signer := NewSigner()
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9"
+
+	q := req.Header
+	if e, a := expectedSig, q.Get("Authorization"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestBuildCanonicalRequest(t *testing.T) {
+	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z"
+	if e, a := expected, build.Request.URL.String(); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+	req.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
+
+	s := NewSigner()
+
+	origBody := req.Body
+
+	err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	if req.Body != origBody {
+		t.Errorf("expect request body to not be chagned")
+	}
+}
+
+func TestRequestHost(t *testing.T) {
+	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+	req.Host = "myhost"
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if !strings.Contains(build.CanonicalString, "host:"+req.Host) {
+		t.Errorf("canonical host header invalid")
+	}
+}
+
+func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) {
+	body := `{"description": "this is a test"}`
+	req, _ := buildRequest("dynamodb", "us-east-1", body)
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+	req.Host = "myhost"
+
+	contentLength := fmt.Sprintf("%d", len([]byte(body)))
+	req.Header.Add("Content-Length", contentLength)
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	_, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	//if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") {
+	//	t.Errorf("canonical header content-length invalid")
+	//}
+}
+
+func TestSign_buildCanonicalHeaders(t *testing.T) {
+	serviceName := "mockAPI"
+	region := "mock-region"
+	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
+
+	req, err := http.NewRequest("POST", endpoint, nil)
+	if err != nil {
+		t.Fatalf("failed to create request, %v", err)
+	}
+
+	req.Header.Set("FooInnerSpace", "   inner      space    ")
+	req.Header.Set("FooLeadingSpace", "    leading-space")
+	req.Header.Add("FooMultipleSpace", "no-space")
+	req.Header.Add("FooMultipleSpace", "\ttab-space")
+	req.Header.Add("FooMultipleSpace", "trailing-space    ")
+	req.Header.Set("FooNoSpace", "no-space")
+	req.Header.Set("FooTabSpace", "\ttab-space\t")
+	req.Header.Set("FooTrailingSpace", "trailing-space    ")
+	req.Header.Set("FooWrappedSpace", "   wrapped-space    ")
+
+	ctx := &httpSigner{
+		ServiceName:  serviceName,
+		Region:       region,
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectCanonicalString := strings.Join([]string{
+		`POST`,
+		`/`,
+		``,
+		`fooinnerspace:inner space`,
+		`fooleadingspace:leading-space`,
+		`foomultiplespace:no-space,tab-space,trailing-space`,
+		`foonospace:no-space`,
+		`footabspace:tab-space`,
+		`footrailingspace:trailing-space`,
+		`foowrappedspace:wrapped-space`,
+		`host:mockAPI.mock-region.amazonaws.com`,
+		`x-amz-date:20211020T124200Z`,
+		``,
+		`fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`,
+		``,
+	}, "\n")
+	if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" {
+		t.Errorf("expect match, got\n%s", diff)
+	}
+}
+
+func BenchmarkPresignRequest(b *testing.B) {
+	signer := NewSigner()
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	for i := 0; i < b.N; i++ {
+		signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	}
+}
+
+func BenchmarkSignRequest(b *testing.B) {
+	signer := NewSigner()
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+	for i := 0; i < b.N; i++ {
+		signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	}
+}

aws/staticcheck.conf

@@ -0,0 +1 @@
+checks = []

backend/azure/err.go

@@ -0,0 +1,63 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package azure
+
+import (
+	"errors"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/versity/versitygw/s3err"
+)
+
+// Parses azure ResponseError into AWS APIError
+func azureErrToS3Err(apiErr error) error {
+	var azErr *azcore.ResponseError
+	if !errors.As(apiErr, &azErr) {
+		return apiErr
+	}
+
+	return azErrToS3err(azErr)
+}
+
+func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
+	switch azErr.ErrorCode {
+	case "ContainerAlreadyExists":
+		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
+	case "InvalidResourceName", "ContainerNotFound":
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	case "BlobNotFound":
+		return s3err.GetAPIError(s3err.ErrNoSuchKey)
+	case "TagsTooLarge":
+		return s3err.GetAPIError(s3err.ErrInvalidTag)
+	case "Requested Range Not Satisfiable":
+		return s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+	return s3err.APIError{
+		Code:           azErr.ErrorCode,
+		Description:    azErr.RawResponse.Status,
+		HTTPStatusCode: azErr.StatusCode,
+	}
+}
+
+func parseMpError(mpErr error) error {
+	err := azureErrToS3Err(mpErr)
+
+	serr, ok := err.(s3err.APIError)
+	if !ok || serr.Code != "NoSuchKey" {
+		return mpErr
+	}
+
+	return s3err.GetAPIError(s3err.ErrNoSuchUpload)
+}

backend/backend.go

@@ -1,55 +1,86 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package backend
 
 import (
+	"bufio"
+	"context"
 	"fmt"
 	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/scoutgw/s3err"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+	"github.com/versity/versitygw/s3select"
 )
 
-//go:generate moq -out backend_moq_test.go . Backend
-//go:generate moq -out ../s3api/controllers/backend_moq_test.go . Backend
+//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
 type Backend interface {
 	fmt.Stringer
-	GetIAMConfig() ([]byte, error)
 	Shutdown()
 
-	ListBuckets() (*s3.ListBucketsOutput, error)
-	HeadBucket(bucket string) (*s3.HeadBucketOutput, error)
-	GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, error)
-	PutBucket(bucket string) error
-	DeleteBucket(bucket string) error
+	// bucket operations
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
+	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
+	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
+	PutBucketAcl(_ context.Context, bucket string, data []byte) error
+	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
+	PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
+	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 
-	CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
-	CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error)
-	AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
-	ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error)
-	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error)
-	CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error)
-	PutObjectPart(bucket, object, uploadID string, part int, r io.Reader) (etag string, err error)
+	// multipart operations
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
+	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
+	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
 
-	PutObject(bucket, object string, r io.Reader) (string, error)
-	HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error)
-	GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3.GetObjectOutput, error)
-	GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
-	CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error)
-	ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
-	ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
-	DeleteObject(bucket, object string) error
-	DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error
-	PutBucketAcl(*s3.PutBucketAclInput) error
-	PutObjectAcl(*s3.PutObjectAclInput) error
-	RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error
-	UploadPart(bucket, object, uploadId string, Body io.ReadSeeker) (*s3.UploadPartOutput, error)
-	UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error)
+	// standard object operations
+	PutObject(context.Context, *s3.PutObjectInput) (string, error)
+	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
+	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
 
-	IsTaggingSupported() bool
-	GetTags(bucket, object string) (map[string]string, error)
-	SetTags(bucket, object string, tags map[string]string) error
-	RemoveTags(bucket, object string) error
+	// special case object operations
+	RestoreObject(context.Context, *s3.RestoreObjectInput) error
+	SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
+
+	// bucket tagging operations
+	GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
+	PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
+	DeleteBucketTagging(_ context.Context, bucket string) error
+
+	// object tagging operations
+	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
+	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
+	DeleteObjectTagging(_ context.Context, bucket, object string) error
+
+	// non AWS actions
+	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
 type BackendUnsupported struct{}
@@ -59,105 +90,136 @@ var _ Backend = &BackendUnsupported{}
 func New() Backend {
 	return &BackendUnsupported{}
 }
-
-func (BackendUnsupported) GetIAMConfig() ([]byte, error) {
-	return nil, fmt.Errorf("not supported")
-}
 func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets() (*s3.ListBucketsOutput, error) {
+func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketAcl(*s3.PutBucketAclInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
+func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPart(bucket, object, uploadId string, Body io.ReadSeeker) (*s3.UploadPartOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucket(bucket string) error {
+func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(bucket string) error {
+func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) CreateMultipartUpload(input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
+func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) error {
+func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, r io.Reader) (etag string, err error) {
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
 	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) PutObject(bucket, object string, r io.Reader) (string, error) {
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
 	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObject(bucket, object string) error {
+func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error {
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3.GetObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+
+func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
+	return func(w *bufio.Writer) {
+		var getProgress s3select.GetProgress
+		progress := input.RequestProgress
+		if progress != nil && *progress.Enabled {
+			getProgress = func() (bytesScanned int64, bytesProcessed int64) {
+				return -1, -1
+			}
+		}
+		mh := s3select.NewMessageHandler(ctx, w, getProgress)
+		apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
+		mh.FinishWithError(apiErr.Code, apiErr.Description)
+	}
 }
-func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
+
+func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) IsTaggingSupported() bool { return false }
-func (BackendUnsupported) GetTags(bucket, object string) (map[string]string, error) {
-	return nil, fmt.Errorf("not supported")
+func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) SetTags(bucket, object string, tags map[string]string) error {
-	return fmt.Errorf("not supported")
+func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) RemoveTags(bucket, object string) error {
-	return fmt.Errorf("not supported")
+func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
+	return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }

backend/backend_test.go

@@ -1,204 +0,0 @@
-package backend
-
-import (
-	"context"
-	"testing"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/scoutgw/s3err"
-)
-
-func TestBackend_ListBuckets(t *testing.T) {
-	type args struct {
-		ctx context.Context
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "list-Bucket",
-		c: &BackendMock{
-			ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
-				return &s3.ListBucketsOutput{
-					Buckets: []types.Bucket{
-						{
-							Name: aws.String("t1"),
-						},
-					},
-				}, s3err.GetAPIError(0)
-			},
-		},
-		args: args{
-			ctx: context.Background(),
-		},
-		wantErr: false,
-	}, test{
-		name: "list-Bucket-error",
-		c: &BackendMock{
-			ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx: context.Background(),
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := tt.c.ListBuckets(); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_HeadBucket(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		BucketName string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "head-buckets-error",
-		c: &BackendMock{
-			HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			BucketName: "b1",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := tt.c.HeadBucket(tt.args.BucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.HeadBucket() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_GetBucketAcl(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		bucketName string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "get bucket acl error",
-		c: &BackendMock{
-			GetBucketAclFunc: func(bucket string) (*s3.GetBucketAclOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			bucketName: "b1",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := tt.c.GetBucketAcl(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.GetBucketAcl() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_PutBucket(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		bucketName string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "put bucket ",
-		c: &BackendMock{
-			PutBucketFunc: func(bucket string) error {
-				return s3err.GetAPIError(0)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			bucketName: "b1",
-		},
-		wantErr: false,
-	}, test{
-		name: "put bucket error",
-		c: &BackendMock{
-			PutBucketFunc: func(bucket string) error {
-				return s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			bucketName: "b2",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.c.PutBucket(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.PutBucket() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_DeleteBucket(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		bucketName string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "Delete Bucket Error",
-		c: &BackendMock{
-			DeleteBucketFunc: func(bucket string) error {
-				return s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			bucketName: "b1",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.c.DeleteBucket(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.DeleteBucket() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}

backend/common.go

@@ -1,18 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package backend
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/fs"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	// RFC3339TimeFormat RFC3339 time format
+	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
 )
 
 func IsValidBucketName(name string) bool { return true }
 
-type ByBucketName []types.Bucket
+type ByBucketName []s3response.ListAllMyBucketsEntry
 
 func (d ByBucketName) Len() int           { return len(d) }
 func (d ByBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
-func (d ByBucketName) Less(i, j int) bool { return *d[i].Name < *d[j].Name }
+func (d ByBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
+
+type ByObjectName []types.Object
+
+func (d ByObjectName) Len() int           { return len(d) }
+func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
 
 func GetStringPtr(s string) *string {
 	return &s
@@ -21,3 +54,69 @@ func GetStringPtr(s string) *string {
 func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
+
+var (
+	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
+)
+
+// ParseRange parses input range header and returns startoffset, length, and
+// error. If no endoffset specified, then length is set to -1.
+func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
+	if acceptRange == "" {
+		return 0, fi.Size(), nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) < 2 {
+		return 0, 0, errInvalidRange
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) < 1 || len(bRange) > 2 {
+		return 0, 0, errInvalidRange
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, 0, errInvalidRange
+	}
+
+	endOffset := int64(-1)
+	if len(bRange) == 1 || bRange[1] == "" {
+		return startOffset, endOffset, nil
+	}
+
+	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, 0, errInvalidRange
+	}
+
+	if endOffset < startOffset {
+		return 0, 0, errInvalidRange
+	}
+
+	return startOffset, endOffset - startOffset + 1, nil
+}
+
+func GetMultipartMD5(parts []types.CompletedPart) string {
+	var partsEtagBytes []byte
+	for _, part := range parts {
+		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
+	}
+	s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
+	return s3MD5
+}
+
+func getEtagBytes(etag string) []byte {
+	decode, err := hex.DecodeString(strings.ReplaceAll(etag, string('"'), ""))
+	if err != nil {
+		return []byte(etag)
+	}
+	return decode
+}
+
+func md5String(data []byte) string {
+	sum := md5.Sum(data)
+	return hex.EncodeToString(sum[:])
+}

backend/posix/posix_darwin.go

@@ -1,14 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package posix
 
 import (
+	"crypto/sha256"
+	"errors"
 	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 )
 
-func openTmpFile(dir string) (*os.File, error) {
-	return nil, fmt.Errorf("not implemented")
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
 }
 
-func linkTmpFile(f *os.File, path string) error {
-	return fmt.Errorf("not implemented")
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
 }

backend/posix/posix_linux.go

@@ -1,31 +1,105 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package posix
 
 import (
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 
 	"golang.org/x/sys/unix"
 )
 
 const procfddir = "/proc/self/fd"
 
-func openTmpFile(dir string) (*os.File, error) {
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
-	if err != nil {
-		return nil, err
-	}
-
-	return os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd))), nil
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
 }
 
-func linkTmpFile(f *os.File, path string) error {
-	err := os.Remove(path)
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
+	// This can help reduce contention within the namespace (parent directories),
+	// etc. And will auto cleanup the inode on close if we never link this
+	// file descriptor into the namespace.
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	if err != nil {
+		// O_TMPFILE not supported, try fallback
+		err := os.MkdirAll(dir, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("make temp dir: %w", err)
+		}
+		f, err := os.CreateTemp(dir,
+			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+		if err != nil {
+			return nil, err
+		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		// falloc is best effort, its fine if this fails
+		if size > 0 {
+			tmp.falloc()
+		}
+		return tmp, nil
+	}
+
+	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
+	// later to link file into namespace
+	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
+
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	// falloc is best effort, its fine if this fails
+	if size > 0 {
+		tmp.falloc()
+	}
+	return tmp, nil
+}
+
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
+func (tmp *tmpfile) link() error {
+	// We use Linkat/Rename as the atomic operation for object puts. The
+	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
+	// with any other simultaneous uploads. The final operation is to move the
+	// temp file into place for the object. This ensures the object semantics
+	// of last upload completed wins and is not some combination of writes
+	// from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
 	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove stale part: %w", err)
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
 	}
 
 	procdir, err := os.Open(procfddir)
@@ -34,17 +108,57 @@ func linkTmpFile(f *os.File, path string) error {
 	}
 	defer procdir.Close()
 
-	dir, err := os.Open(filepath.Dir(path))
+	dir, err := os.Open(filepath.Dir(objPath))
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
 	defer dir.Close()
 
-	err = unix.Linkat(int(procdir.Fd()), filepath.Base(f.Name()),
-		int(dir.Fd()), filepath.Base(path), unix.AT_SYMLINK_FOLLOW)
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
-		return fmt.Errorf("link tmpfile: %w", err)
+		return fmt.Errorf("link tmpfile (%q in %q): %w",
+			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
 	}
 
 	return nil
 }
+
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/posix/posix_windows.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length")
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/s3proxy/client.go

@@ -0,0 +1,79 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+)
+
+func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
+	cfg, err := s.getConfig(ctx, s.access, s.secret)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg), nil
+}
+
+func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.disableChecksum {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(ctx, opts...)
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}

backend/s3proxy/s3.go

@@ -0,0 +1,545 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+const aclKey string = "versitygwAcl"
+
+type S3Proxy struct {
+	backend.BackendUnsupported
+
+	client *s3.Client
+
+	access          string
+	secret          string
+	endpoint        string
+	awsRegion       string
+	disableChecksum bool
+	sslSkipVerify   bool
+	debug           bool
+}
+
+func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
+	s := &S3Proxy{
+		access:          access,
+		secret:          secret,
+		endpoint:        endpoint,
+		awsRegion:       region,
+		disableChecksum: disableChecksum,
+		sslSkipVerify:   sslSkipVerify,
+		debug:           debug,
+	}
+	client, err := s.getClientWithCtx(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	s.client = client
+	return s, nil
+}
+
+func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, handleError(err)
+	}
+
+	var buckets []s3response.ListAllMyBucketsEntry
+	for _, b := range output.Buckets {
+		buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+			Name:         *b.Name,
+			CreationDate: *b.CreationDate,
+		})
+	}
+
+	return s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID: *output.Owner.ID,
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: buckets,
+		},
+	}, nil
+}
+
+func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	out, err := s.client.HeadBucket(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
+	_, err := s.client.CreateBucket(ctx, input)
+	if err != nil {
+		return handleError(err)
+	}
+
+	var tagSet []types.Tag
+	tagSet = append(tagSet, types.Tag{
+		Key:   backend.GetStringPtr(aclKey),
+		Value: backend.GetStringPtr(base64Encode(acl)),
+	})
+
+	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
+		Bucket: input.Bucket,
+		Tagging: &types.Tagging{
+			TagSet: tagSet,
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	_, err := s.client.DeleteBucket(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	out, err := s.client.CreateMultipartUpload(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	out, err := s.client.CompleteMultipartUpload(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	_, err := s.client.AbortMultipartUpload(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	output, err := s.client.ListMultipartUploads(ctx, input)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, handleError(err)
+	}
+
+	var uploads []s3response.Upload
+	for _, u := range output.Uploads {
+		uploads = append(uploads, s3response.Upload{
+			Key:      *u.Key,
+			UploadID: *u.UploadId,
+			Initiator: s3response.Initiator{
+				ID:          *u.Initiator.ID,
+				DisplayName: *u.Initiator.DisplayName,
+			},
+			Owner: s3response.Owner{
+				ID:          *u.Owner.ID,
+				DisplayName: *u.Owner.DisplayName,
+			},
+			StorageClass: string(u.StorageClass),
+			Initiated:    u.Initiated.Format(backend.RFC3339TimeFormat),
+		})
+	}
+
+	var cps []s3response.CommonPrefix
+	for _, c := range output.CommonPrefixes {
+		cps = append(cps, s3response.CommonPrefix{
+			Prefix: *c.Prefix,
+		})
+	}
+
+	return s3response.ListMultipartUploadsResult{
+		Bucket:             *output.Bucket,
+		KeyMarker:          *output.KeyMarker,
+		UploadIDMarker:     *output.UploadIdMarker,
+		NextKeyMarker:      *output.NextKeyMarker,
+		NextUploadIDMarker: *output.NextUploadIdMarker,
+		Delimiter:          *output.Delimiter,
+		Prefix:             *output.Prefix,
+		EncodingType:       string(output.EncodingType),
+		MaxUploads:         int(*output.MaxUploads),
+		IsTruncated:        *output.IsTruncated,
+		Uploads:            uploads,
+		CommonPrefixes:     cps,
+	}, nil
+}
+
+func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	output, err := s.client.ListParts(ctx, input)
+	if err != nil {
+		return s3response.ListPartsResult{}, handleError(err)
+	}
+
+	var parts []s3response.Part
+	for _, p := range output.Parts {
+		parts = append(parts, s3response.Part{
+			PartNumber:   int(*p.PartNumber),
+			LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
+			ETag:         *p.ETag,
+			Size:         *p.Size,
+		})
+	}
+	pnm, err := strconv.Atoi(*output.PartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse part number marker: %w", err)
+	}
+
+	npmn, err := strconv.Atoi(*output.NextPartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse next part number marker: %w", err)
+	}
+
+	return s3response.ListPartsResult{
+		Bucket:   *output.Bucket,
+		Key:      *output.Key,
+		UploadID: *output.UploadId,
+		Initiator: s3response.Initiator{
+			ID:          *output.Initiator.ID,
+			DisplayName: *output.Initiator.DisplayName,
+		},
+		Owner: s3response.Owner{
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
+		},
+		StorageClass:         string(output.StorageClass),
+		PartNumberMarker:     pnm,
+		NextPartNumberMarker: npmn,
+		MaxParts:             int(*output.MaxParts),
+		IsTruncated:          *output.IsTruncated,
+		Parts:                parts,
+	}, nil
+}
+
+func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
+	if err != nil {
+		return "", handleError(err)
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	output, err := s.client.UploadPartCopy(ctx, input)
+	if err != nil {
+		return s3response.CopyObjectResult{}, handleError(err)
+	}
+
+	return s3response.CopyObjectResult{
+		LastModified: *output.CopyPartResult.LastModified,
+		ETag:         *output.CopyPartResult.ETag,
+	}, nil
+}
+
+func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
+	if err != nil {
+		return "", handleError(err)
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	out, err := s.client.HeadObject(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+	output, err := s.client.GetObject(ctx, input)
+	if err != nil {
+		return nil, handleError(err)
+	}
+	defer output.Body.Close()
+
+	_, err = io.Copy(w, output.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}
+
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	out, err := s.client.GetObjectAttributes(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	out, err := s.client.CopyObject(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	out, err := s.client.ListObjects(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	out, err := s.client.ListObjectsV2(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	_, err := s.client.DeleteObject(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	if len(input.Delete.Objects) == 0 {
+		input.Delete.Objects = []types.ObjectIdentifier{}
+	}
+
+	output, err := s.client.DeleteObjects(ctx, input)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, handleError(err)
+	}
+
+	return s3response.DeleteObjectsResult{
+		Deleted: output.Deleted,
+		Error:   output.Errors,
+	}, nil
+}
+
+func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+		Bucket: input.Bucket,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	for _, tag := range tagout.TagSet {
+		if *tag.Key == aclKey {
+			acl, err := base64Decode(*tag.Value)
+			if err != nil {
+				return nil, handleError(err)
+			}
+			return acl, nil
+		}
+	}
+
+	return []byte{}, nil
+}
+
+func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
+	tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return handleError(err)
+	}
+
+	var found bool
+	for i, tag := range tagout.TagSet {
+		if *tag.Key == aclKey {
+			tagout.TagSet[i] = types.Tag{
+				Key:   backend.GetStringPtr(aclKey),
+				Value: backend.GetStringPtr(base64Encode(data)),
+			}
+			found = true
+			break
+		}
+	}
+	if !found {
+		tagout.TagSet = append(tagout.TagSet, types.Tag{
+			Key:   backend.GetStringPtr(aclKey),
+			Value: backend.GetStringPtr(base64Encode(data)),
+		})
+	}
+
+	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
+		Bucket: &bucket,
+		Tagging: &types.Tagging{
+			TagSet: tagout.TagSet,
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	tagging := &types.Tagging{
+		TagSet: []types.Tag{},
+	}
+	for key, val := range tags {
+		tagging.TagSet = append(tagging.TagSet, types.Tag{
+			Key:   &key,
+			Value: &val,
+		})
+	}
+
+	_, err := s.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
+		Bucket:  &bucket,
+		Key:     &object,
+		Tagging: tagging,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	tags := make(map[string]string)
+	for _, el := range output.TagSet {
+		tags[*el.Key] = *el.Value
+	}
+
+	return tags, nil
+}
+
+func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	if resp.StatusCode > 300 {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		return fmt.Errorf(string(body))
+	}
+
+	return nil
+}
+
+func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return []s3response.Bucket{}, err
+	}
+	defer resp.Body.Close()
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return []s3response.Bucket{}, err
+	}
+
+	return buckets, nil
+}
+
+func handleError(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		apiErr := s3err.APIError{
+			Code:        ae.ErrorCode(),
+			Description: ae.ErrorMessage(),
+		}
+		var re *awshttp.ResponseError
+		if errors.As(err, &re) {
+			apiErr.HTTPStatusCode = re.Response.StatusCode
+		}
+		return apiErr
+	}
+	return err
+}
+
+func base64Encode(input []byte) string {
+	return base64.StdEncoding.EncodeToString(input)
+}
+
+func base64Decode(encoded string) ([]byte, error) {
+	decoded, err := base64.StdEncoding.DecodeString(encoded)
+	if err != nil {
+		return nil, err
+	}
+	return decoded, nil
+}

backend/scoutfs/scoutfs.go

@@ -1,12 +1,813 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package scoutfs
 
 import (
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/backend/posix"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/s3err"
 )
 
 type ScoutFS struct {
 	*posix.Posix
+	rootfd  *os.File
+	rootdir string
+
+	// glaciermode enables the following behavior:
+	// GET object:  if file offline, return invalid object state
+	// HEAD object: if file offline, set obj storage class to GLACIER
+	//              if file offline and staging, x-amz-restore: ongoing-request="true"
+	//              if file offline and not staging, x-amz-restore: ongoing-request="false"
+	//              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
+	//              note: this expiry-date is not used but provided for client glacier compatibility
+	// ListObjects: if file offline, set obj storage class to GLACIER
+	// RestoreObject: add batch stage request to file
+	glaciermode bool
 }
 
-var _ backend.Backend = ScoutFS{}
+var _ backend.Backend = &ScoutFS{}
+
+const (
+	metaTmpDir          = ".sgwtmp"
+	metaTmpMultipartDir = metaTmpDir + "/multipart"
+	tagHdr              = "X-Amz-Tagging"
+	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
+	etagkey             = "user.etag"
+)
+
+var (
+	stageComplete      = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\""
+	stageInProgress    = "true"
+	stageNotInProgress = "false"
+)
+
+const (
+	// ScoutFS special xattr types
+
+	systemPrefix = "scoutfs.hide."
+	onameAttr    = systemPrefix + "objname"
+	flagskey     = systemPrefix + "sam_flags"
+	stagecopykey = systemPrefix + "sam_stagereq"
+)
+
+const (
+	// ScoutAM Flags
+
+	// Staging - file requested stage
+	Staging uint64 = 1 << iota
+	// StageFail - all copies failed to stage
+	StageFail
+	// NoArchive - no archive copies of file should be made
+	NoArchive
+	// ExtCacheRequested means file policy requests Ext Cache
+	ExtCacheRequested
+	// ExtCacheDone means this file ext cache copy has been
+	// created already (and possibly pruned, so may not exist)
+	ExtCacheDone
+)
+
+// Option sets various options for scoutfs
+type Option func(s *ScoutFS)
+
+// WithGlacierEmulation sets glacier mode emulation
+func WithGlacierEmulation() Option {
+	return func(s *ScoutFS) { s.glaciermode = true }
+}
+
+func (s *ScoutFS) Shutdown() {
+	s.Posix.Shutdown()
+	s.rootfd.Close()
+	_ = s.rootdir
+}
+
+func (*ScoutFS) String() string {
+	return "ScoutFS Gateway"
+}
+
+// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
+// ioctl to not have to read and copy the part data to the final object. This
+// saves a read and write cycle for all mutlipart uploads.
+func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+	uploadID := *input.UploadId
+	parts := input.MultipartUpload.Parts
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	sum, err := s.checkUploadIDExists(bucket, object, uploadID)
+	if err != nil {
+		return nil, err
+	}
+
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	// check all parts ok
+	last := len(parts) - 1
+	partsize := int64(0)
+	var totalsize int64
+	for i, p := range parts {
+		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
+		fi, err := os.Lstat(partPath)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		if i == 0 {
+			partsize = fi.Size()
+		}
+		totalsize += fi.Size()
+		// all parts except the last need to be the same size
+		if i < last && partsize != fi.Size() {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		// non-last part sizes need to be multiples of 4k for move blocks
+		// TODO: fallback to no move blocks if not 4k aligned?
+		if i == 0 && i < last && fi.Size()%4096 != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		b, err := xattr.Get(partPath, "user.etag")
+		etag := string(b)
+		if err != nil {
+			etag = ""
+		}
+		if etag != *parts[i].ETag {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+	}
+
+	// use totalsize=0 because we wont be writing to the file, only moving
+	// extents around.  so we dont want to fallocate this.
+	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
+	if err != nil {
+		return nil, fmt.Errorf("open temp file: %w", err)
+	}
+	defer f.cleanup()
+
+	for _, p := range parts {
+		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
+		if err != nil {
+			return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
+		}
+
+		// scoutfs move data is a metadata only operation that moves the data
+		// extent references from the source, appeding to the destination.
+		// this needs to be 4k aligned.
+		err = moveData(pf, f.f)
+		pf.Close()
+		if err != nil {
+			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
+		}
+	}
+
+	userMetaData := make(map[string]string)
+	upiddir := filepath.Join(objdir, uploadID)
+	loadUserMetaData(upiddir, userMetaData)
+
+	objname := filepath.Join(bucket, object)
+	dir := filepath.Dir(objname)
+	if dir != "" {
+		if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
+			if err != nil {
+				return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			}
+		}
+	}
+	err = f.link()
+	if err != nil {
+		return nil, fmt.Errorf("link object in namespace: %w", err)
+	}
+
+	for k, v := range userMetaData {
+		err = xattr.Set(objname, "user."+k, []byte(v))
+		if err != nil {
+			// cleanup object if returning error
+			os.Remove(objname)
+			return nil, fmt.Errorf("set user attr %q: %w", k, err)
+		}
+	}
+
+	// Calculate s3 compatible md5sum for complete multipart.
+	s3MD5 := backend.GetMultipartMD5(parts)
+
+	err = xattr.Set(objname, "user.etag", []byte(s3MD5))
+	if err != nil {
+		// cleanup object if returning error
+		os.Remove(objname)
+		return nil, fmt.Errorf("set etag attr: %w", err)
+	}
+
+	// cleanup tmp dirs
+	os.RemoveAll(upiddir)
+	// use Remove for objdir in case there are still other uploads
+	// for same object name outstanding
+	os.Remove(objdir)
+
+	return &s3.CompleteMultipartUploadOutput{
+		Bucket: &bucket,
+		ETag:   &s3MD5,
+		Key:    &object,
+	}, nil
+}
+
+func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) {
+	sum := sha256.Sum256([]byte(object))
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	_, err := os.Stat(filepath.Join(objdir, uploadID))
+	if errors.Is(err, fs.ErrNotExist) {
+		return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
+	}
+	if err != nil {
+		return [32]byte{}, fmt.Errorf("stat upload: %w", err)
+	}
+	return sum, nil
+}
+
+func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
+	ents, err := xattr.List(path)
+	if err != nil || len(ents) == 0 {
+		return
+	}
+	for _, e := range ents {
+		if !isValidMeta(e) {
+			continue
+		}
+		b, err := xattr.Get(path, e)
+		if err == syscall.ENODATA {
+			m[strings.TrimPrefix(e, "user.")] = ""
+			continue
+		}
+		if err != nil {
+			continue
+		}
+		m[strings.TrimPrefix(e, "user.")] = string(b)
+	}
+
+	b, err := xattr.Get(path, "user.content-type")
+	contentType = string(b)
+	if err != nil {
+		contentType = ""
+	}
+	if contentType != "" {
+		m["content-type"] = contentType
+	}
+
+	b, err = xattr.Get(path, "user.content-encoding")
+	contentEncoding = string(b)
+	if err != nil {
+		contentEncoding = ""
+	}
+	if contentEncoding != "" {
+		m["content-encoding"] = contentEncoding
+	}
+
+	return
+}
+
+func isValidMeta(val string) bool {
+	if strings.HasPrefix(val, "user.X-Amz-Meta") {
+		return true
+	}
+	if strings.EqualFold(val, "user.Expires") {
+		return true
+	}
+	return false
+}
+
+// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
+// when appropriate
+func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = mkdirAll(path[:j-1], perm, bucket, object)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, perm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+	return nil
+}
+
+func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	objPath := filepath.Join(bucket, object)
+	fi, err := os.Stat(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat object: %w", err)
+	}
+
+	userMetaData := make(map[string]string)
+	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+
+	b, err := xattr.Get(objPath, etagkey)
+	etag := string(b)
+	if err != nil {
+		etag = ""
+	}
+
+	stclass := types.StorageClassStandard
+	requestOngoing := ""
+	if s.glaciermode {
+		requestOngoing = stageComplete
+
+		// Check if there are any offline exents associated with this file.
+		// If so, we will set storage class to glacier.
+		st, err := statMore(objPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat more: %w", err)
+		}
+		if st.Offline_blocks != 0 {
+			stclass = types.StorageClassGlacier
+			requestOngoing = stageNotInProgress
+
+			ok, err := isStaging(objPath)
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+			}
+			if err != nil {
+				return nil, fmt.Errorf("check stage status: %w", err)
+			}
+			if ok {
+				requestOngoing = stageInProgress
+			}
+		}
+	}
+
+	contentLength := fi.Size()
+
+	return &s3.HeadObjectOutput{
+		ContentLength:   &contentLength,
+		ContentType:     &contentType,
+		ContentEncoding: &contentEncoding,
+		ETag:            &etag,
+		LastModified:    backend.GetTimePtr(fi.ModTime()),
+		Metadata:        userMetaData,
+		StorageClass:    stclass,
+		Restore:         &requestOngoing,
+	}, nil
+}
+
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+	acceptRange := *input.Range
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	objPath := filepath.Join(bucket, object)
+	fi, err := os.Stat(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat object: %w", err)
+	}
+
+	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	if err != nil {
+		return nil, err
+	}
+
+	if length == -1 {
+		length = fi.Size() - startOffset + 1
+	}
+
+	if startOffset+length > fi.Size() {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	if s.glaciermode {
+		// Check if there are any offline exents associated with this file.
+		// If so, we will return the InvalidObjectState error.
+		st, err := statMore(objPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat more: %w", err)
+		}
+		if st.Offline_blocks != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidObjectState)
+		}
+	}
+
+	f, err := os.Open(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("open object: %w", err)
+	}
+	defer f.Close()
+
+	rdr := io.NewSectionReader(f, startOffset, length)
+	_, err = io.Copy(writer, rdr)
+	if err != nil {
+		return nil, fmt.Errorf("copy data: %w", err)
+	}
+
+	userMetaData := make(map[string]string)
+
+	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+
+	b, err := xattr.Get(objPath, etagkey)
+	etag := string(b)
+	if err != nil {
+		etag = ""
+	}
+
+	tags, err := s.getXattrTags(bucket, object)
+	if err != nil {
+		return nil, fmt.Errorf("get object tags: %w", err)
+	}
+
+	tagCount := int32(len(tags))
+
+	return &s3.GetObjectOutput{
+		AcceptRanges:    &acceptRange,
+		ContentLength:   &length,
+		ContentEncoding: &contentEncoding,
+		ContentType:     &contentType,
+		ETag:            &etag,
+		LastModified:    backend.GetTimePtr(fi.ModTime()),
+		Metadata:        userMetaData,
+		TagCount:        &tagCount,
+		StorageClass:    types.StorageClassStandard,
+	}, nil
+}
+
+func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) {
+	tags := make(map[string]string)
+	b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if isNoAttr(err) {
+		return tags, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get tags: %w", err)
+	}
+
+	err = json.Unmarshal(b, &tags)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal tags: %w", err)
+	}
+
+	return tags, nil
+}
+
+func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	bucket := *input.Bucket
+	prefix := ""
+	if input.Prefix != nil {
+		prefix = *input.Prefix
+	}
+	marker := ""
+	if input.Marker != nil {
+		marker = *input.Marker
+	}
+	delim := ""
+	if input.Delimiter != nil {
+		delim = *input.Delimiter
+	}
+	maxkeys := int32(0)
+	if input.MaxKeys != nil {
+		maxkeys = *input.MaxKeys
+	}
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	fileSystem := os.DirFS(bucket)
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
+		s.fileToObj(bucket), []string{metaTmpDir})
+	if err != nil {
+		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+	}
+
+	return &s3.ListObjectsOutput{
+		CommonPrefixes: results.CommonPrefixes,
+		Contents:       results.Objects,
+		Delimiter:      &delim,
+		IsTruncated:    &results.Truncated,
+		Marker:         &marker,
+		MaxKeys:        &maxkeys,
+		Name:           &bucket,
+		NextMarker:     &results.NextMarker,
+		Prefix:         &prefix,
+	}, nil
+}
+
+func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	bucket := *input.Bucket
+	prefix := ""
+	if input.Prefix != nil {
+		prefix = *input.Prefix
+	}
+	marker := ""
+	if input.ContinuationToken != nil {
+		marker = *input.ContinuationToken
+	}
+	delim := ""
+	if input.Delimiter != nil {
+		delim = *input.Delimiter
+	}
+	maxkeys := int32(0)
+	if input.MaxKeys != nil {
+		maxkeys = *input.MaxKeys
+	}
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	fileSystem := os.DirFS(bucket)
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
+		s.fileToObj(bucket), []string{metaTmpDir})
+	if err != nil {
+		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+	}
+
+	return &s3.ListObjectsV2Output{
+		CommonPrefixes:        results.CommonPrefixes,
+		Contents:              results.Objects,
+		Delimiter:             &delim,
+		IsTruncated:           &results.Truncated,
+		ContinuationToken:     &marker,
+		MaxKeys:               &maxkeys,
+		Name:                  &bucket,
+		NextContinuationToken: &results.NextMarker,
+		Prefix:                &prefix,
+	}, nil
+}
+
+func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
+	return func(path string, d fs.DirEntry) (types.Object, error) {
+		objPath := filepath.Join(bucket, path)
+		if d.IsDir() {
+			// directory object only happens if directory empty
+			// check to see if this is a directory object by checking etag
+			etagBytes, err := xattr.Get(objPath, etagkey)
+			if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("get etag: %w", err)
+			}
+			etag := string(etagBytes)
+
+			fi, err := d.Info()
+			if errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			}
+
+			key := path + "/"
+
+			return types.Object{
+				ETag:         &etag,
+				Key:          &key,
+				LastModified: backend.GetTimePtr(fi.ModTime()),
+			}, nil
+		}
+
+		// file object, get object info and fill out object data
+		etagBytes, err := xattr.Get(objPath, etagkey)
+		if errors.Is(err, fs.ErrNotExist) {
+			return types.Object{}, backend.ErrSkipObj
+		}
+		if err != nil && !isNoAttr(err) {
+			return types.Object{}, fmt.Errorf("get etag: %w", err)
+		}
+		etag := string(etagBytes)
+
+		fi, err := d.Info()
+		if errors.Is(err, fs.ErrNotExist) {
+			return types.Object{}, backend.ErrSkipObj
+		}
+		if err != nil {
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		}
+
+		sc := types.ObjectStorageClassStandard
+		if s.glaciermode {
+			// Check if there are any offline exents associated with this file.
+			// If so, we will return the InvalidObjectState error.
+			st, err := statMore(objPath)
+			if errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("stat more: %w", err)
+			}
+			if st.Offline_blocks != 0 {
+				sc = types.ObjectStorageClassGlacier
+			}
+		}
+
+		size := fi.Size()
+
+		return types.Object{
+			ETag:         &etag,
+			Key:          &path,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
+			Size:         &size,
+			StorageClass: sc,
+		}, nil
+	}
+}
+
+// RestoreObject will set stage request on file if offline and do nothing if
+// file is online
+func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
+	bucket := *input.Bucket
+	object := *input.Key
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	err = setStaging(filepath.Join(bucket, object))
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return fmt.Errorf("stage object: %w", err)
+	}
+
+	return nil
+}
+
+func setStaging(objname string) error {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return err
+	}
+
+	var oldflags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &oldflags)
+		if err != nil {
+			return err
+		}
+	}
+
+	newflags := oldflags | Staging
+
+	if newflags == oldflags {
+		// no flags change, just return
+		return nil
+	}
+
+	return fSetNewGlobalFlags(objname, newflags)
+}
+
+func isStaging(objname string) (bool, error) {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return false, err
+	}
+
+	var flags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &flags)
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return flags&Staging == Staging, nil
+}
+
+func fSetNewGlobalFlags(objname string, flags uint64) error {
+	b, err := json.Marshal(&flags)
+	if err != nil {
+		return err
+	}
+
+	return xattr.Set(objname, flagskey, b)
+}
+
+func isNoAttr(err error) bool {
+	if err == nil {
+		return false
+	}
+	xerr, ok := err.(*xattr.Error)
+	if ok && xerr.Err == xattr.ENOATTR {
+		return true
+	}
+	if err == syscall.ENODATA {
+		return true
+	}
+	return false
+}

backend/scoutfs/scoutfs_compat.go

@@ -0,0 +1,209 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build linux && amd64
+
+package scoutfs
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/versity/scoutfs-go"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+	p, err := posix.New(rootdir)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := os.Open(rootdir)
+	if err != nil {
+		return nil, fmt.Errorf("open %v: %w", rootdir, err)
+	}
+
+	s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
+	for _, opt := range opts {
+		opt(s)
+	}
+
+	return s, nil
+}
+
+const procfddir = "/proc/self/fd"
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
+	// This can help reduce contention within the namespace (parent directories),
+	// etc. And will auto cleanup the inode on close if we never link this
+	// file descriptor into the namespace.
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	if err != nil {
+		// O_TMPFILE not supported, try fallback
+		err := os.MkdirAll(dir, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("make temp dir: %w", err)
+		}
+		f, err := os.CreateTemp(dir,
+			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+		if err != nil {
+			return nil, err
+		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		// falloc is best effort, its fine if this fails
+		if size > 0 {
+			tmp.falloc()
+		}
+		return tmp, nil
+	}
+
+	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
+	// later to link file into namespace
+	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
+
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	// falloc is best effort, its fine if this fails
+	if size > 0 {
+		tmp.falloc()
+	}
+	return tmp, nil
+}
+
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
+func (tmp *tmpfile) link() error {
+	// We use Linkat/Rename as the atomic operation for object puts. The
+	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
+	// with any other simultaneous uploads. The final operation is to move the
+	// temp file into place for the object. This ensures the object semantics
+	// of last upload completed wins and is not some combination of writes
+	// from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
+	}
+
+	procdir, err := os.Open(procfddir)
+	if err != nil {
+		return fmt.Errorf("open proc dir: %w", err)
+	}
+	defer procdir.Close()
+
+	dir, err := os.Open(filepath.Dir(objPath))
+	if err != nil {
+		return fmt.Errorf("open parent dir: %w", err)
+	}
+	defer dir.Close()
+
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if err != nil {
+		return fmt.Errorf("link tmpfile: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}
+
+func moveData(from *os.File, to *os.File) error {
+	return scoutfs.MoveData(from, to)
+}
+
+func statMore(path string) (stat, error) {
+	st, err := scoutfs.StatMore(path)
+	if err != nil {
+		return stat{}, err
+	}
+	var s stat
+
+	s.Meta_seq = st.Meta_seq
+	s.Data_seq = st.Data_seq
+	s.Data_version = st.Data_version
+	s.Online_blocks = st.Online_blocks
+	s.Offline_blocks = st.Offline_blocks
+	s.Crtime_sec = st.Crtime_sec
+	s.Crtime_nsec = st.Crtime_nsec
+
+	return s, nil
+}

backend/scoutfs/scoutfs_incompat.go

@@ -0,0 +1,58 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !(linux && amd64)
+
+package scoutfs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+)
+
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+	return nil, fmt.Errorf("scoutfs only available on linux")
+}
+
+type tmpfile struct {
+	f *os.File
+}
+
+var (
+	errNotSupported = errors.New("not supported")
+)
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	return nil, errNotSupported
+}
+
+func (tmp *tmpfile) link() error {
+	return errNotSupported
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	return 0, errNotSupported
+}
+
+func (tmp *tmpfile) cleanup() {
+}
+
+func moveData(from *os.File, to *os.File) error {
+	return errNotSupported
+}
+
+func statMore(path string) (stat, error) {
+	return stat{}, errNotSupported
+}

backend/scoutfs/stat.go

@@ -0,0 +1,25 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package scoutfs
+
+type stat struct {
+	Meta_seq       uint64
+	Data_seq       uint64
+	Data_version   uint64
+	Online_blocks  uint64
+	Offline_blocks uint64
+	Crtime_sec     uint64
+	Crtime_nsec    uint32
+}

backend/walk.go

@@ -0,0 +1,222 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+type WalkResults struct {
+	CommonPrefixes []types.CommonPrefix
+	Objects        []types.Object
+	Truncated      bool
+	NextMarker     string
+}
+
+type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
+
+var ErrSkipObj = errors.New("skip this object")
+
+// Walk walks the supplied fs.FS and returns results compatible with list
+// objects responses
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+	cpmap := make(map[string]struct{})
+	var objects []types.Object
+
+	var pastMarker bool
+	if marker == "" {
+		pastMarker = true
+	}
+
+	pastMax := max == 0
+	var newMarker string
+	var truncated bool
+
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
+
+		if pastMax {
+			if len(objects) != 0 {
+				newMarker = *objects[len(objects)-1].Key
+				truncated = true
+			}
+			return fs.SkipAll
+		}
+
+		if d.IsDir() {
+			// If prefix is defined and the directory does not match prefix,
+			// do not descend into the directory because nothing will
+			// match this prefix. Make sure to append the / at the end of
+			// directories since this is implied as a directory path name.
+			// If path is a prefix of prefix, then path could still be
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
+			if prefix != "" &&
+				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
+				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
+				return fs.SkipDir
+			}
+
+			// TODO: can we do better here rather than a second readdir
+			// per directory?
+			ents, err := fs.ReadDir(fileSystem, path)
+			if err != nil {
+				return fmt.Errorf("readdir %q: %w", path, err)
+			}
+			if len(ents) == 0 {
+				dirobj, err := getObj(path, d)
+				if err == ErrSkipObj {
+					return nil
+				}
+				if err != nil {
+					return fmt.Errorf("directory to object %q: %w", path, err)
+				}
+				objects = append(objects, dirobj)
+			}
+
+			return nil
+		}
+
+		if !pastMarker {
+			if path == marker {
+				pastMarker = true
+				return nil
+			}
+			if path < marker {
+				return nil
+			}
+		}
+
+		// If object doesn't have prefix, don't include in results.
+		if prefix != "" && !strings.HasPrefix(path, prefix) {
+			return nil
+		}
+
+		if delimiter == "" {
+			// If no delimiter specified, then all files with matching
+			// prefix are included in results
+			obj, err := getObj(path, d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, obj)
+
+			if max > 0 && (len(objects)+len(cpmap)) == int(max) {
+				pastMax = true
+			}
+
+			return nil
+		}
+
+		// Since delimiter is specified, we only want results that
+		// do not contain the delimiter beyond the prefix.  If the
+		// delimiter exists past the prefix, then the substring
+		// between the prefix and delimiter is part of common prefixes.
+		//
+		// For example:
+		// prefix = A/
+		// delimiter = /
+		// and objects:
+		// A/file
+		// A/B/file
+		// B/C
+		// would return:
+		// objects: A/file
+		// common prefix: A/B/
+		//
+		// Note: No objects are included past the common prefix since
+		// these are all rolled up into the common prefix.
+		// Note: The delimiter can be anything, so we have to operate on
+		// the full path without any assumptions on posix directory hierarchy
+		// here.  Usually the delimiter will be "/", but thats not required.
+		suffix := strings.TrimPrefix(path, prefix)
+		before, _, found := strings.Cut(suffix, delimiter)
+		if !found {
+			obj, err := getObj(path, d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, obj)
+			if (len(objects) + len(cpmap)) == int(max) {
+				pastMax = true
+			}
+			return nil
+		}
+
+		// Common prefixes are a set, so should not have duplicates.
+		// These are abstractly a "directory", so need to include the
+		// delimiter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
+		if (len(objects) + len(cpmap)) == int(max) {
+			pastMax = true
+		}
+
+		return nil
+	})
+	if err != nil {
+		return WalkResults{}, err
+	}
+
+	var commonPrefixStrings []string
+	for k := range cpmap {
+		commonPrefixStrings = append(commonPrefixStrings, k)
+	}
+	sort.Strings(commonPrefixStrings)
+	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
+	for _, cp := range commonPrefixStrings {
+		pfx := cp
+		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
+			Prefix: &pfx,
+		})
+	}
+
+	return WalkResults{
+		CommonPrefixes: commonPrefixes,
+		Objects:        objects,
+		Truncated:      truncated,
+		NextMarker:     newMarker,
+	}, nil
+}
+
+func contains(a string, strs []string) bool {
+	for _, s := range strs {
+		if s == a {
+			return true
+		}
+	}
+	return false
+}

backend/walk_test.go

@@ -0,0 +1,206 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend_test
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/fs"
+	"testing"
+	"testing/fstest"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
+)
+
+type walkTest struct {
+	fsys     fs.FS
+	expected backend.WalkResults
+	getobj   backend.GetObjFunc
+}
+
+func getObj(path string, d fs.DirEntry) (types.Object, error) {
+	if d.IsDir() {
+		etag := getMD5(path)
+
+		fi, err := d.Info()
+		if err != nil {
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		}
+
+		return types.Object{
+			ETag:         &etag,
+			Key:          &path,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
+		}, nil
+	}
+
+	etag := getMD5(path)
+
+	fi, err := d.Info()
+	if err != nil {
+		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+	}
+
+	size := fi.Size()
+
+	return types.Object{
+		ETag:         &etag,
+		Key:          &path,
+		LastModified: backend.GetTimePtr(fi.ModTime()),
+		Size:         &size,
+	}, nil
+}
+
+func getMD5(text string) string {
+	hash := md5.Sum([]byte(text))
+	return hex.EncodeToString(hash[:])
+}
+
+func TestWalk(t *testing.T) {
+	tests := []walkTest{
+		{
+			// test case from
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
+			fsys: fstest.MapFS{
+				"sample.jpg":                       {},
+				"photos/2006/January/sample.jpg":   {},
+				"photos/2006/February/sample2.jpg": {},
+				"photos/2006/February/sample3.jpg": {},
+				"photos/2006/February/sample4.jpg": {},
+			},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("photos/"),
+				}},
+				Objects: []types.Object{{
+					Key: backend.GetStringPtr("sample.jpg"),
+				}},
+			},
+			getobj: getObj,
+		},
+		{
+			// test case single dir/single file
+			fsys: fstest.MapFS{
+				"test/file": {},
+			},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("test/"),
+				}},
+				Objects: []types.Object{},
+			},
+			getobj: getObj,
+		},
+	}
+
+	for _, tt := range tests {
+		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
+		if err != nil {
+			t.Fatalf("walk: %v", err)
+		}
+
+		compareResults(res, tt.expected, t)
+	}
+}
+
+func compareResults(got, wanted backend.WalkResults, t *testing.T) {
+	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
+		t.Errorf("unexpected common prefix, got %v wanted %v",
+			printCommonPrefixes(got.CommonPrefixes),
+			printCommonPrefixes(wanted.CommonPrefixes))
+	}
+
+	if !compareObjects(got.Objects, wanted.Objects) {
+		t.Errorf("unexpected object, got %v wanted %v",
+			printObjects(got.Objects),
+			printObjects(wanted.Objects))
+	}
+}
+
+func compareCommonPrefix(a, b []types.CommonPrefix) bool {
+	if len(a) == 0 && len(b) == 0 {
+		return true
+	}
+	if len(a) != len(b) {
+		return false
+	}
+
+	for _, cp := range a {
+		if containsCommonPrefix(cp, b) {
+			return true
+		}
+	}
+	return false
+}
+
+func containsCommonPrefix(c types.CommonPrefix, list []types.CommonPrefix) bool {
+	for _, cp := range list {
+		if *c.Prefix == *cp.Prefix {
+			return true
+		}
+	}
+	return false
+}
+
+func printCommonPrefixes(list []types.CommonPrefix) string {
+	res := "["
+	for _, cp := range list {
+		if res == "[" {
+			res = res + *cp.Prefix
+		} else {
+			res = res + ", " + *cp.Prefix
+		}
+	}
+	return res + "]"
+}
+
+func compareObjects(a, b []types.Object) bool {
+	if len(a) == 0 && len(b) == 0 {
+		return true
+	}
+	if len(a) != len(b) {
+		return false
+	}
+
+	for _, cp := range a {
+		if containsObject(cp, b) {
+			return true
+		}
+	}
+	return false
+}
+
+func containsObject(c types.Object, list []types.Object) bool {
+	for _, cp := range list {
+		if *c.Key == *cp.Key {
+			return true
+		}
+	}
+	return false
+}
+
+func printObjects(list []types.Object) string {
+	res := "["
+	for _, cp := range list {
+		if res == "[" {
+			res = res + *cp.Key
+		} else {
+			res = res + ", " + *cp.Key
+		}
+	}
+	return res + "]"
+}

certs/azurite-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDUaEoThzf6bWNU
+R7+V0sYGND0iArE8dx33jCqWBzUCeeAnf8VjsKTVKWK8Cernjot7jP/GlKHCXKVC
+Dpaw2SuekVn84gXr0UDD0clrPYYcytm8TWrfNmcPnmLARH6bes+5Glt21gug5fc7
+gvd/EFI/eNTAmU6ZPzVjKVP5plvkSZYXePgmgKzY8hs8j9IjjOZB+TzalRlQoMa4
+hPYTzNpvdS3aQCM/8PTefC8sxI0H9inx1utax22MPNCaGl+BvD9yqvFwQl7n0d9u
+3ZLJ8Br9UKRPX47NDerpsYsmJAAtjevei84v4liCKtxIzC6b/KuKpgEDhiza+TL/
+VoJ34xK9AgMBAAECggEBAISIPhBJQshjEKM50XTuetjMJ4jdHTGZMX2QW9IY1R6i
+ZRbARq2ZPoAyvoSNu6CX9Lg2ljGV9AiOPh8lcykNmIXsM7XyYbdubXbBo2fij5fP
+imRP+eskytGYBg3prwXcb1gT9hYEIGVYmBbt9Pe3e1pXToiOH9jG88zXsKoI/zVF
+ASfE73L9xlO9roaFWe+sQlNsHJ5SVhKVNq4FcjvIbZTi3JqMJvuGqgIjyOeKlRdH
+FszPjmn8tmlbT7BQIfeo4uIJwDrUxdp4Do9txScSOhV4xcdVjpOrbD+HaTIl18u8
+J32pf0n8E90ntKJlPyfSt68VIkIT1VtE0SH42UgxcAECgYEA2Ctw58hSKFSQtSDk
+HAOuxh9gSM4eIvWG8Ni4L+S6n66hCPYYj+1ZGBM1xIKqfug25STP2qV54zB34CMM
+RfPEJhKglgzN1VdTvTkdLfqSnrEDyF+67/X0IhomV54CLN52i7S6ra39OVClirQc
+5FbjX7Dwy57vanNW7E7ZKHyxRwECgYEA+4tiCw8+NeCIjLiOldB/R9P6CguFCIFY
+B+MUXc2K690ysRcVnHCVLGVrdYlmlAjiEEOo6e8nhl0pLJecOLUkLnEdZkC9x5je
+oy1itan7Y/WjdSPIpLE5sBnrWzzpy5YVoKi4rjIkPXYDHcgbvEcKUxN/qUpOpKk6
+rTeaAPOwp70CgYEAwPIFVNz4eAcDIqi48khXN3/J8TIItCtyxoap4BXIfb7g/Z6r
+TcwMOfDrjPsUMzIRzXWOERqiMKaSWPzvd4CdE16M92F2V3YayEqyQNfnBr35ImBP
++t8NiWLN1mayiloGdaxa86rY2s+g8qzRHP5w9Hh6dUTnbZyFeWbnbbvegAECgYAG
+T1zKQjuhRlymiwqon25R8vNWxSs1J2l56SxdngZaHFZlMtsL7ZcQYgrsC+JS3FYv
+akMWezVWnYem4ra8hW6+6399TSp1k1Qia8UKIZV40HSlP5yM5RU5Ya0RwNlsjftE
+6HaZiBB4qjkxyg9IDdAofVi6em62mrgqGDb4xyQrUQKBgHfzBE/pxKTZVCyjJgq+
+XKtvm7awq6tVByZzKVbFb+kqle7n0Kb+MjDSiH0gXqrntkhquEEpo8ViXenHeDq1
+I2Fgp6Tw8pKW7RANcWaxd2KLYDxUkqwNWCMYGQZoW9lhTy1/e5stXSTdqd+3CJkp
+qBbZOasECIEGL8XUpFEBmrTA
+-----END PRIVATE KEY-----

certs/azurite.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEMzCCApugAwIBAgIQbYDa23tko31nfiGrCJ4XfDANBgkqhkiG9w0BAQsFADBj
+MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExHDAaBgNVBAsME2JlbkBC
+eXRlTmV4dXMubG9jYWwxIzAhBgNVBAMMGm1rY2VydCBiZW5AQnl0ZU5leHVzLmxv
+Y2FsMB4XDTI0MDIyMTA2MDY1OFoXDTI2MDUyMTA1MDY1OFowRzEnMCUGA1UEChMe
+bWtjZXJ0IGRldmVsb3BtZW50IGNlcnRpZmljYXRlMRwwGgYDVQQLDBNiZW5AQnl0
+ZU5leHVzLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1GhK
+E4c3+m1jVEe/ldLGBjQ9IgKxPHcd94wqlgc1AnngJ3/FY7Ck1SlivAnq546Le4z/
+xpShwlylQg6WsNkrnpFZ/OIF69FAw9HJaz2GHMrZvE1q3zZnD55iwER+m3rPuRpb
+dtYLoOX3O4L3fxBSP3jUwJlOmT81YylT+aZb5EmWF3j4JoCs2PIbPI/SI4zmQfk8
+2pUZUKDGuIT2E8zab3Ut2kAjP/D03nwvLMSNB/Yp8dbrWsdtjDzQmhpfgbw/cqrx
+cEJe59Hfbt2SyfAa/VCkT1+OzQ3q6bGLJiQALY3r3ovOL+JYgircSMwum/yriqYB
+A4Ys2vky/1aCd+MSvQIDAQABo38wfTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAww
+CgYIKwYBBQUHAwEwHwYDVR0jBBgwFoAUHWzgJTYXxrKTuaJyaD2ddH4g43QwNQYD
+VR0RBC4wLIIJbG9jYWxob3N0ggdhenVyaXRlhwR/AAABhxAAAAAAAAAAAAAAAAAA
+AAABMA0GCSqGSIb3DQEBCwUAA4IBgQBANjSWFEE2XrQfCtqKjHRtYn+WnjfeUmJW
+4qfNl971spR+ui+WMNob53EKGV9O55Ai8zWUKhL14GSDo4bqygcqCSP7esUaqW5x
+LgqGkIDPiPN18OfWxZonlvcSJYaWJ9dL4Xix7OmPXQV88oNUYh5zASJRZ8mN3vZK
+v0BElS5w7X4pM8jWkIEVUjmgSQJAxapDnbDPCIntH13rjYPHZW3zF1wjWW/mfW59
++thLVpG0U9w8knDU9TJQWPX9rQB8lLBHYfBy14ebuMWmDl25Rr+wcUjFp76HP8f9
+D8wX7ZTJlMlaH60vbTi8ylsKMlu6eHxJ8TBNdRI5+Xa94Pi6A81a9EiagpU2uZs1
+P/v/3ug9AtCFoEBB2groj3waC6QTOrJArHhvioLKBz7yMsfGRQCSA+VK+KGuqE6f
+E/Knv1Phf8HIe6OpDNXqlJo9+5m0sz/y6F2OaeNqM643Y78OQRFijtDKu+TmB+t0
+4lo/rMVz6A0vIOBBQkiJz82xezvCiHQ=
+-----END CERTIFICATE-----

cmd/scoutgw/main.go

@@ -1,18 +0,0 @@
-package main
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/s3api"
-	"log"
-)
-
-func main() {
-	app := fiber.New(fiber.Config{})
-	back := backend.New()
-	if api, err := s3api.New(app, back, ":7070"); err != nil {
-		log.Fatalln(err)
-	} else if err = api.Serve(); err != nil {
-		log.Fatalln(err)
-	}
-}

cmd/versitygw/admin.go

@@ -0,0 +1,419 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	adminAccess   string
+	adminSecret   string
+	adminEndpoint string
+)
+
+func adminCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "admin",
+		Usage:       "admin CLI tool",
+		Description: `Admin CLI tool for interacting with admin APIs.`,
+		Subcommands: []*cli.Command{
+			{
+				Name:   "create-user",
+				Usage:  "Create a new user",
+				Action: createUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "access key id for the new user",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:     "secret",
+						Usage:    "secret access key for the new user",
+						Required: true,
+						Aliases:  []string{"s"},
+					},
+					&cli.StringFlag{
+						Name:     "role",
+						Usage:    "role for the new user",
+						Required: true,
+						Aliases:  []string{"r"},
+					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+					&cli.IntFlag{
+						Name:    "project-id",
+						Usage:   "projectID for the new user",
+						Aliases: []string{"pi"},
+					},
+				},
+			},
+			{
+				Name:   "delete-user",
+				Usage:  "Delete a user",
+				Action: deleteUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "access key id of the user to be deleted",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+				},
+			},
+			{
+				Name:   "list-users",
+				Usage:  "List all the gateway users",
+				Action: listUsers,
+			},
+			{
+				Name:  "change-bucket-owner",
+				Usage: "Changes the bucket owner",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "bucket",
+						Usage:    "the bucket name to change the owner",
+						Required: true,
+						Aliases:  []string{"b"},
+					},
+					&cli.StringFlag{
+						Name:     "owner",
+						Usage:    "the user access key id, who should be the bucket owner",
+						Required: true,
+						Aliases:  []string{"o"},
+					},
+				},
+				Action: changeBucketOwner,
+			},
+			{
+				Name:   "list-buckets",
+				Usage:  "Lists all the gateway buckets and owners.",
+				Action: listBuckets,
+			},
+		},
+		Flags: []cli.Flag{
+			// TODO: create a configuration file for this
+			&cli.StringFlag{
+				Name:        "access",
+				Usage:       "admin access key id",
+				EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
+				Aliases:     []string{"a"},
+				Required:    true,
+				Destination: &adminAccess,
+			},
+			&cli.StringFlag{
+				Name:        "secret",
+				Usage:       "admin secret access key",
+				EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
+				Aliases:     []string{"s"},
+				Required:    true,
+				Destination: &adminSecret,
+			},
+			&cli.StringFlag{
+				Name:        "endpoint-url",
+				Usage:       "admin apis endpoint url",
+				EnvVars:     []string{"ADMIN_ENDPOINT_URL"},
+				Aliases:     []string{"er"},
+				Required:    true,
+				Destination: &adminEndpoint,
+			},
+		},
+	}
+}
+
+func createUser(ctx *cli.Context) error {
+	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
+	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
+	if access == "" || secret == "" {
+		return fmt.Errorf("invalid input parameters for the new user")
+	}
+	if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
+		return fmt.Errorf("invalid input parameter for role: %v", role)
+	}
+
+	acc := auth.Account{
+		Access:    access,
+		Secret:    secret,
+		Role:      auth.Role(role),
+		UserID:    userID,
+		GroupID:   groupID,
+		ProjectID: projectID,
+	}
+
+	accJson, err := json.Marshal(acc)
+	if err != nil {
+		return fmt.Errorf("failed to parse user data: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(accJson)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
+func deleteUser(ctx *cli.Context) error {
+	access := ctx.String("access")
+	if access == "" {
+		return fmt.Errorf("invalid input parameter for the new user")
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
+func listUsers(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
+
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
+		return err
+	}
+
+	printAcctTable(accs)
+
+	return nil
+}
+
+const (
+	// account table formatting
+	minwidth int  = 2   // minimal cell width including any padding
+	tabwidth int  = 0   // width of tab characters (equivalent number of spaces)
+	padding  int  = 2   // padding added to a cell before computing its width
+	padchar  byte = ' ' // ASCII char used for padding
+	flags    uint = 0   // formatting control flags
+)
+
+func printAcctTable(accs []auth.Account) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
+	for _, acc := range accs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func changeBucketOwner(ctx *cli.Context) error {
+	bucket, owner := ctx.String("bucket"), ctx.String("owner")
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Println(string(body))
+
+	return nil
+}
+
+func printBuckets(buckets []s3response.Bucket) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Bucket\tOwner")
+	fmt.Fprintln(w, "-------\t----")
+	for _, acc := range buckets {
+		fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func listBuckets(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return err
+	}
+
+	printBuckets(buckets)
+
+	return nil
+}

cmd/versitygw/azure.go

@@ -0,0 +1,74 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/azure"
+)
+
+var (
+	azAccount, azKey, azServiceURL, azSASToken string
+)
+
+func azureCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "azure",
+		Usage:       "azure blob storage backend",
+		Description: `direct translation from s3 objects to azure blobs`,
+		Action:      runAzure,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "account",
+				Usage:       "azure account name",
+				EnvVars:     []string{"AZ_ACCOUNT_NAME"},
+				Aliases:     []string{"a"},
+				Destination: &azAccount,
+			},
+			&cli.StringFlag{
+				Name:        "access-key",
+				Usage:       "azure account key",
+				EnvVars:     []string{"AZ_ACCESS_KEY"},
+				Aliases:     []string{"k"},
+				Destination: &azKey,
+			},
+			&cli.StringFlag{
+				Name:        "sas-token",
+				Usage:       "azure blob storage SAS token",
+				EnvVars:     []string{"AZ_SAS_TOKEN"},
+				Aliases:     []string{"st"},
+				Destination: &azSASToken,
+			},
+			&cli.StringFlag{
+				Name:        "url",
+				Usage:       "azure service URL",
+				EnvVars:     []string{"AZ_ENDPOINT"},
+				Aliases:     []string{"u"},
+				Destination: &azServiceURL,
+			},
+		},
+	}
+}
+
+func runAzure(ctx *cli.Context) error {
+	be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
+	if err != nil {
+		return fmt.Errorf("init azure: %w", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/gateway_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/integration"
+)
+
+const (
+	tdir = "tempdir"
+)
+
+var (
+	wg sync.WaitGroup
+)
+
+func initEnv(dir string) {
+	// both
+	debug = true
+	region = "us-east-1"
+
+	// server
+	rootUserAccess = "user"
+	rootUserSecret = "pass"
+	iamDir = dir
+	port = "127.0.0.1:7070"
+
+	// client
+	awsID = "user"
+	awsSecret = "pass"
+	endpoint = "http://127.0.0.1:7070"
+}
+
+func initPosix(ctx context.Context) {
+	path, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("get current directory: %v", err)
+	}
+
+	tempdir := filepath.Join(path, tdir)
+	initEnv(tempdir)
+
+	err = os.RemoveAll(tempdir)
+	if err != nil {
+		log.Fatalf("remove temp directory: %v", err)
+	}
+
+	err = os.Mkdir(tempdir, 0755)
+	if err != nil {
+		log.Fatalf("make temp directory: %v", err)
+	}
+
+	be, err := posix.New(tempdir)
+	if err != nil {
+		log.Fatalf("init posix: %v", err)
+	}
+
+	wg.Add(1)
+	go func() {
+		err = runGateway(ctx, be)
+		if err != nil && err != context.Canceled {
+			log.Fatalf("run gateway: %v", err)
+		}
+
+		err := os.RemoveAll(tempdir)
+		if err != nil {
+			log.Fatalf("remove temp directory: %v", err)
+		}
+		wg.Done()
+	}()
+}
+
+func TestIntegration(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	initPosix(ctx)
+
+	opts := []integration.Option{
+		integration.WithAccess(awsID),
+		integration.WithSecret(awsSecret),
+		integration.WithRegion(region),
+		integration.WithEndpoint(endpoint),
+	}
+	if debug {
+		opts = append(opts, integration.WithDebug())
+	}
+
+	s := integration.NewS3Conf(opts...)
+
+	// replace below with desired test
+	err := integration.HeadBucket_non_existing_bucket(s)
+	if err != nil {
+		t.Error(err)
+	}
+
+	cancel()
+	wg.Wait()
+}

cmd/versitygw/main.go

@@ -0,0 +1,526 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api"
+	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3event"
+	"github.com/versity/versitygw/s3log"
+)
+
+var (
+	port, admPort                          string
+	rootUserAccess                         string
+	rootUserSecret                         string
+	region                                 string
+	admCertFile, admKeyFile                string
+	certFile, keyFile                      string
+	kafkaURL, kafkaTopic, kafkaKey         string
+	natsURL, natsTopic                     string
+	logWebhookURL                          string
+	accessLog                              string
+	healthPath                             string
+	debug                                  bool
+	quiet                                  bool
+	iamDir                                 string
+	ldapURL, ldapBindDN, ldapPassword      string
+	ldapQueryBase, ldapObjClasses          string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	s3IamAccess, s3IamSecret               string
+	s3IamRegion, s3IamBucket               string
+	s3IamEndpoint                          string
+	s3IamSslNoVerify, s3IamDebug           bool
+	iamCacheDisable                        bool
+	iamCacheTTL                            int
+	iamCachePrune                          int
+)
+
+var (
+	// Version is the latest tag (set within Makefile)
+	Version = "git"
+	// Build is the commit hash (set within Makefile)
+	Build = "norev"
+	// BuildTime is the date/time of build (set within Makefile)
+	BuildTime = "none"
+)
+
+func main() {
+	setupSignalHandler()
+
+	app := initApp()
+
+	app.Commands = []*cli.Command{
+		posixCommand(),
+		scoutfsCommand(),
+		s3Command(),
+		azureCommand(),
+		adminCommand(),
+		testCommand(),
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		<-sigDone
+		fmt.Fprintf(os.Stderr, "terminating signal caught, shutting down\n")
+		cancel()
+	}()
+
+	if err := app.RunContext(ctx, os.Args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func initApp() *cli.App {
+	return &cli.App{
+		Name:  "versitygw",
+		Usage: "Start S3 gateway service with specified backend storage.",
+		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.`,
+		Action: func(ctx *cli.Context) error {
+			return ctx.App.Command("help").Run(ctx)
+		},
+		Flags: initFlags(),
+	}
+}
+
+func initFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "version",
+			Usage:   "list versitygw version",
+			Aliases: []string{"v"},
+			Action: func(*cli.Context, bool) error {
+				fmt.Println("Version  :", Version)
+				fmt.Println("Build    :", Build)
+				fmt.Println("BuildTime:", BuildTime)
+				os.Exit(0)
+				return nil
+			},
+		},
+		&cli.StringFlag{
+			Name:        "port",
+			Usage:       "gateway listen address <ip>:<port> or :<port>",
+			EnvVars:     []string{"VGW_PORT"},
+			Value:       ":7070",
+			Destination: &port,
+			Aliases:     []string{"p"},
+		},
+		&cli.StringFlag{
+			Name:        "access",
+			Usage:       "root user access key",
+			EnvVars:     []string{"ROOT_ACCESS_KEY_ID", "ROOT_ACCESS_KEY"},
+			Aliases:     []string{"a"},
+			Destination: &rootUserAccess,
+		},
+		&cli.StringFlag{
+			Name:        "secret",
+			Usage:       "root user secret access key",
+			EnvVars:     []string{"ROOT_SECRET_ACCESS_KEY", "ROOT_SECRET_KEY"},
+			Aliases:     []string{"s"},
+			Destination: &rootUserSecret,
+		},
+		&cli.StringFlag{
+			Name:        "region",
+			Usage:       "s3 region string",
+			EnvVars:     []string{"VGW_REGION"},
+			Value:       "us-east-1",
+			Destination: &region,
+			Aliases:     []string{"r"},
+		},
+		&cli.StringFlag{
+			Name:        "cert",
+			Usage:       "TLS cert file",
+			EnvVars:     []string{"VGW_CERT"},
+			Destination: &certFile,
+		},
+		&cli.StringFlag{
+			Name:        "key",
+			Usage:       "TLS key file",
+			EnvVars:     []string{"VGW_KEY"},
+			Destination: &keyFile,
+		},
+		&cli.StringFlag{
+			Name:        "admin-port",
+			Usage:       "gateway admin server listen address <ip>:<port> or :<port>",
+			EnvVars:     []string{"VGW_ADMIN_PORT"},
+			Destination: &admPort,
+			Aliases:     []string{"ap"},
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert",
+			Usage:       "TLS cert file for admin server",
+			EnvVars:     []string{"VGW_ADMIN_CERT"},
+			Destination: &admCertFile,
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert-key",
+			Usage:       "TLS key file for admin server",
+			EnvVars:     []string{"VGW_ADMIN_CERT_KEY"},
+			Destination: &admKeyFile,
+		},
+		&cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "enable debug output",
+			EnvVars:     []string{"VGW_DEBUG"},
+			Destination: &debug,
+		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "silence stdout request logging output",
+			EnvVars:     []string{"VGW_QUIET"},
+			Destination: &quiet,
+			Aliases:     []string{"q"},
+		},
+		&cli.StringFlag{
+			Name:        "access-log",
+			Usage:       "enable server access logging to specified file",
+			EnvVars:     []string{"LOGFILE"},
+			Destination: &accessLog,
+		},
+		&cli.StringFlag{
+			Name:        "log-webhook-url",
+			Usage:       "webhook url to send the audit logs",
+			EnvVars:     []string{"WEBHOOK"},
+			Destination: &logWebhookURL,
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-url",
+			Usage:       "kafka server url to send the bucket notifications.",
+			EnvVars:     []string{"VGW_EVENT_KAFKA_URL"},
+			Destination: &kafkaURL,
+			Aliases:     []string{"eku"},
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-topic",
+			Usage:       "kafka server pub-sub topic to send the bucket notifications to",
+			EnvVars:     []string{"VGW_EVENT_KAFKA_TOPIC"},
+			Destination: &kafkaTopic,
+			Aliases:     []string{"ekt"},
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-key",
+			Usage:       "kafka server put-sub topic key to send the bucket notifications to",
+			EnvVars:     []string{"VGW_EVENT_KAFKA_KEY"},
+			Destination: &kafkaKey,
+			Aliases:     []string{"ekk"},
+		},
+		&cli.StringFlag{
+			Name:        "event-nats-url",
+			Usage:       "nats server url to send the bucket notifications",
+			EnvVars:     []string{"VGW_EVENT_NATS_URL"},
+			Destination: &natsURL,
+			Aliases:     []string{"enu"},
+		},
+		&cli.StringFlag{
+			Name:        "event-nats-topic",
+			Usage:       "nats server pub-sub topic to send the bucket notifications to",
+			EnvVars:     []string{"VGW_EVENT_NATS_TOPIC"},
+			Destination: &natsTopic,
+			Aliases:     []string{"ent"},
+		},
+		&cli.StringFlag{
+			Name:        "iam-dir",
+			Usage:       "if defined, run internal iam service within this directory",
+			EnvVars:     []string{"VGW_IAM_DIR"},
+			Destination: &iamDir,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-url",
+			Usage:       "ldap server url to store iam data",
+			EnvVars:     []string{"VGW_IAM_LDAP_URL"},
+			Destination: &ldapURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-dn",
+			Usage:       "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
+			EnvVars:     []string{"VGW_IAM_LDAP_BIND_DN"},
+			Destination: &ldapBindDN,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-pass",
+			Usage:       "ldap server user password",
+			EnvVars:     []string{"VGW_IAM_LDAP_BIND_PASS"},
+			Destination: &ldapPassword,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-query-base",
+			Usage:       "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
+			EnvVars:     []string{"VGW_IAM_LDAP_QUERY_BASE"},
+			Destination: &ldapQueryBase,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-object-classes",
+			Usage:       "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
+			EnvVars:     []string{"VGW_IAM_LDAP_OBJECT_CLASSES"},
+			Destination: &ldapObjClasses,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-access-atr",
+			Usage:       "ldap server user access key id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_ACCESS_ATR"},
+			Destination: &ldapAccessAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-secret-atr",
+			Usage:       "ldap server user secret access key attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_SECRET_ATR"},
+			Destination: &ldapSecAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-role-atr",
+			Usage:       "ldap server user role attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
+			Destination: &ldapRoleAtr,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-access",
+			Usage:       "s3 IAM access key",
+			EnvVars:     []string{"VGW_S3_IAM_ACCESS_KEY"},
+			Destination: &s3IamAccess,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-secret",
+			Usage:       "s3 IAM secret key",
+			EnvVars:     []string{"VGW_S3_IAM_SECRET_KEY"},
+			Destination: &s3IamSecret,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-region",
+			Usage:       "s3 IAM region",
+			EnvVars:     []string{"VGW_S3_IAM_REGION"},
+			Destination: &s3IamRegion,
+			Value:       "us-east-1",
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-bucket",
+			Usage:       "s3 IAM bucket",
+			EnvVars:     []string{"VGW_S3_IAM_BUCKET"},
+			Destination: &s3IamBucket,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-endpoint",
+			Usage:       "s3 IAM endpoint",
+			EnvVars:     []string{"VGW_S3_IAM_ENDPOINT"},
+			Destination: &s3IamEndpoint,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-noverify",
+			Usage:       "s3 IAM disable ssl verification",
+			EnvVars:     []string{"VGW_S3_IAM_NO_VERIFY"},
+			Destination: &s3IamSslNoVerify,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-debug",
+			Usage:       "s3 IAM debug output",
+			EnvVars:     []string{"VGW_S3_IAM_DEBUG"},
+			Destination: &s3IamDebug,
+		},
+		&cli.BoolFlag{
+			Name:        "iam-cache-disable",
+			Usage:       "disable local iam cache",
+			EnvVars:     []string{"VGW_IAM_CACHE_DISABLE"},
+			Destination: &iamCacheDisable,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-ttl",
+			Usage:       "local iam cache entry ttl (seconds)",
+			EnvVars:     []string{"VGW_IAM_CACHE_TTL"},
+			Value:       120,
+			Destination: &iamCacheTTL,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-prune",
+			Usage:       "local iam cache cleanup interval (seconds)",
+			EnvVars:     []string{"VGW_IAM_CACHE_PRUNE"},
+			Value:       3600,
+			Destination: &iamCachePrune,
+		},
+		&cli.StringFlag{
+			Name: "health",
+			Usage: `health check endpoint path. Health endpoint will be configured on GET http method: GET <health>
+					NOTICE: the path has to be specified with '/'. e.g /health`,
+			EnvVars:     []string{"VGW_HEALTH"},
+			Destination: &healthPath,
+		},
+	}
+}
+
+func runGateway(ctx context.Context, be backend.Backend) error {
+	app := fiber.New(fiber.Config{
+		AppName:           "versitygw",
+		ServerHeader:      "VERSITYGW",
+		StreamRequestBody: true,
+		DisableKeepalive:  true,
+	})
+
+	var opts []s3api.Option
+
+	if certFile != "" || keyFile != "" {
+		if certFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if keyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		opts = append(opts, s3api.WithTLS(cert))
+	}
+	if debug {
+		opts = append(opts, s3api.WithDebug())
+	}
+	if admPort == "" {
+		opts = append(opts, s3api.WithAdminServer())
+	}
+	if quiet {
+		opts = append(opts, s3api.WithQuiet())
+	}
+	if healthPath != "" {
+		opts = append(opts, s3api.WithHealth(healthPath))
+	}
+
+	admApp := fiber.New(fiber.Config{
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+	})
+
+	var admOpts []s3api.AdminOpt
+
+	if admCertFile != "" || admKeyFile != "" {
+		if admCertFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if admKeyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert))
+	}
+
+	iam, err := auth.New(&auth.Opts{
+		Dir:                iamDir,
+		LDAPServerURL:      ldapURL,
+		LDAPBindDN:         ldapBindDN,
+		LDAPPassword:       ldapPassword,
+		LDAPQueryBase:      ldapQueryBase,
+		LDAPObjClasses:     ldapObjClasses,
+		LDAPAccessAtr:      ldapAccessAtr,
+		LDAPSecretAtr:      ldapSecAtr,
+		LDAPRoleAtr:        ldapRoleAtr,
+		S3Access:           s3IamAccess,
+		S3Secret:           s3IamSecret,
+		S3Region:           s3IamRegion,
+		S3Bucket:           s3IamBucket,
+		S3Endpoint:         s3IamEndpoint,
+		S3DisableSSlVerfiy: s3IamSslNoVerify,
+		S3Debug:            s3IamDebug,
+		CacheDisable:       iamCacheDisable,
+		CacheTTL:           iamCacheTTL,
+		CachePrune:         iamCachePrune,
+	})
+	if err != nil {
+		return fmt.Errorf("setup iam: %w", err)
+	}
+
+	logger, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:    accessLog,
+		WebhookURL: logWebhookURL,
+	})
+	if err != nil {
+		return fmt.Errorf("setup logger: %w", err)
+	}
+
+	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
+		KafkaURL:      kafkaURL,
+		KafkaTopic:    kafkaTopic,
+		KafkaTopicKey: kafkaKey,
+		NatsURL:       natsURL,
+		NatsTopic:     natsTopic,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to connect to the message broker: %w", err)
+	}
+
+	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
+		Access: rootUserAccess,
+		Secret: rootUserSecret,
+	}, port, region, iam, logger, evSender, opts...)
+	if err != nil {
+		return fmt.Errorf("init gateway: %v", err)
+	}
+
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
+
+	c := make(chan error, 2)
+	go func() { c <- srv.Serve() }()
+	if admPort != "" {
+		go func() { c <- admSrv.Serve() }()
+	}
+
+	// for/select blocks until shutdown
+Loop:
+	for {
+		select {
+		case <-ctx.Done():
+			err = ctx.Err()
+			break Loop
+		case err = <-c:
+			break Loop
+		case <-sigHup:
+			if logger != nil {
+				err = logger.HangUp()
+				if err != nil {
+					err = fmt.Errorf("HUP logger: %w", err)
+					break Loop
+				}
+			}
+		}
+	}
+	saveErr := err
+
+	be.Shutdown()
+
+	err = iam.Shutdown()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
+	}
+
+	if logger != nil {
+		err := logger.Shutdown()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
+		}
+	}
+
+	return saveErr
+}

cmd/versitygw/posix.go

@@ -0,0 +1,53 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+func posixCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "posix",
+		Usage: "posix filesystem storage backend",
+		Description: `Any posix filesystem that supports extended attributes. The top level
+directory for the gateway must be provided. All sub directories of the
+top level directory are treated as buckets, and all files/directories
+below the "bucket directory" are treated as the objects. The object
+name is split on "/" separator to translate to posix storage.
+For example:
+top level: /mnt/fs/gwroot
+bucket: mybucket
+object: a/b/c/myobject
+will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
+		Action: runPosix,
+	}
+}
+
+func runPosix(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no directory provided for operation")
+	}
+
+	be, err := posix.New(ctx.Args().Get(0))
+	if err != nil {
+		return fmt.Errorf("init posix: %v", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}