Merge pull request #344 from versity/ben/s3proxy

Ben/s3proxy

.dockerignore

@@ -0,0 +1,46 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+cmd/versitygw/versitygw
+/versitygw
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# ignore IntelliJ directories
+.idea
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+/versitygw.spec.in
+*.tar
+*.tar.gz
+**/rand.data
+/profile.txt
+
+dist/
+
+# Release config files
+/.github
+
+# Docker configuration files
+*Dockerfile
+/docker-compose.yml
+
+# read files
+/LICENSE
+/NOTICE
+/CODE_OF_CONDUCT.md
+/README.md

.env.dev

@@ -0,0 +1,6 @@
+POSIX_PORT=
+PROXY_PORT=
+ACCESS_KEY_ID=
+SECRET_ACCESS_KEY=
+IAM_DIR=
+SETUP_DIR=

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Server Version**
+ output of
+```
+./versitygw -version
+uname -a
+```
+
+**Additional context**
+Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      dev-dependencies:
+        patterns:
+          - "*"

.github/workflows/functional.yml

@@ -0,0 +1,30 @@
+name: functional tests
+on: pull_request
+jobs:
+
+  build:
+    name: RunTests
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+
+    - name: Get Dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Build and Run
+      run: |
+        make testbin
+        ./runtests.sh
+
+    - name: Coverage Report
+      run: |
+        go tool covdata percent -i=/tmp/covdata

.github/workflows/go.yml

@@ -0,0 +1,38 @@
+name: general
+on: pull_request
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+
+    - name: Verify all files pass gofmt formatting
+      run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Build
+      run: make
+
+    - name: Test
+      run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
+
+    - name: Install govulncheck
+      run: go install golang.org/x/vuln/cmd/govulncheck@latest
+      shell: bash
+
+    - name: Run govulncheck
+      run: govulncheck ./...
+      shell: bash

.github/workflows/goreleaser.yml

@@ -0,0 +1,31 @@
+name: goreleaser
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+  # packages: write
+  # issues: write
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/static.yml

@@ -0,0 +1,22 @@
+name: staticcheck
+on: pull_request
+jobs: 
+
+  build: 
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+ 
+    - name: "staticcheck"
+      uses: dominikh/staticcheck-action@v1.3.0
+      with: 
+        install-go: false

.gitignore

@@ -7,6 +7,8 @@
 *.dll
 *.so
 *.dylib
+cmd/versitygw/versitygw
+/versitygw
 
 # Test binary, built with `go test -c`
 *.test
@@ -19,3 +21,21 @@
 
 # Go workspace file
 go.work
+
+# ignore IntelliJ directories
+.idea
+
+# ignore VS code directories
+.vscode
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+*.tar
+*.tar.gz
+**/rand.data
+/profile.txt
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,55 @@
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - goos:
+      - linux
+      - darwin
+      # windows is untested, we can start doing windows releases
+      # if someone is interested in taking on testing
+      # - windows
+    env:
+      # disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
+      # once we need to enable this, we will need to do per distro releases
+      - CGO_ENABLED=0
+    main: ./cmd/versitygw
+    binary: ./cmd/versitygw
+    id: versitygw
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge '
+
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+versitygw@versity.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

Dockerfile

@@ -0,0 +1,24 @@
+FROM golang:1.20-alpine
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY ./ ./
+
+WORKDIR /app/cmd/versitygw
+RUN go build -o versitygw
+
+FROM alpine:latest
+
+# These arguments can be overriden when building the image
+ARG IAM_DIR=/tmp/vgw
+ARG SETUP_DIR=/tmp/vgw
+
+RUN mkdir -p $IAM_DIR
+RUN mkdir -p $SETUP_DIR
+
+COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
+
+ENTRYPOINT [ "/app/versitygw" ]

Dockerfile.dev

@@ -0,0 +1,17 @@
+FROM golang:1.20
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY ./ ./
+
+ARG IAM_DIR=/tmp/vgw
+ARG SETUP_DIR=/tmp/vgw
+
+RUN mkdir -p $IAM_DIR
+RUN mkdir -p $SETUP_DIR
+
+RUN go get github.com/githubnemo/CompileDaemon
+RUN go install github.com/githubnemo/CompileDaemon

Makefile

@@ -0,0 +1,91 @@
+# Copyright 2023 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOCLEAN=$(GOCMD) clean
+GOTEST=$(GOCMD) test
+
+BIN=versitygw
+
+VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
+BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
+TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
+
+LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
+
+all: build
+
+build: $(BIN)
+
+.PHONY: $(BIN)
+$(BIN):
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
+
+testbin:
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
+
+.PHONY: test
+test: 
+	$(GOTEST) ./...
+
+.PHONY: check
+check:
+# note this requires staticcheck be in your PATH:
+# export PATH=$PATH:~/go/bin
+# go install honnef.co/go/tools/cmd/staticcheck@latest
+	staticcheck ./...
+	golint ./...
+	gofmt -s -l .
+
+.PHONY: clean
+clean: 
+	$(GOCLEAN)
+
+.PHONY: cleanall
+cleanall: clean
+	rm -f $(BIN)
+	rm -f versitygw-*.tar
+	rm -f versitygw-*.tar.gz
+	rm -f versitygw.spec
+
+%.spec: %.spec.in
+	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
+	mv $@+ $@
+
+TARFILE = $(BIN)-$(VERSION).tar
+
+dist: $(BIN).spec
+	echo $(VERSION) >VERSION
+	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
+	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
+	rm -f VERSION
+	rm -f $(BIN).spec
+	gzip -f $(TARFILE)
+
+# Creates and runs S3 gateway instance in a docker container
+.PHONY: up-posix
+up-posix:
+	docker compose --env-file .env.dev up posix
+
+# Creates and runs S3 gateway proxy instance in a docker container
+.PHONY: up-proxy
+up-proxy:
+	docker compose --env-file .env.dev up proxy
+
+# Creates and runs both S3 gateway and proxy server instances in docker containers
+.PHONY: up-app
+up-app:
+	docker compose --env-file .env.dev up

NOTICE

@@ -0,0 +1,2 @@
+versitygw - Versity S3 Gateway
+Copyright 2023 Versity Software

README.md

@@ -0,0 +1,67 @@
+# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
+  <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
+</picture>
+
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
+
+**Current status:** Beta: Most clients functional, work in progress for more test coverage. Issue reports welcome.
+
+See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
+
+* Share filesystem directory via S3 protocol
+* Simple to deploy S3 server with a single command
+* Protocol compatibility allows common access to files via posix or S3 
+
+Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
+
+The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
+
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
+
+The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
+
+The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework.  This framework is actively developed with a focus on performance.  S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3. 
+
+## Getting Started
+See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
+
+### Run the gateway with posix backend:
+
+```
+mkdir /tmp/vgw
+ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
+```
+This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
+
+To get the usage output, run the following:
+
+```
+./versitygw --help
+```
+
+The command format is
+
+```
+versitygw [global options] command [command options] [arguments...]
+```
+The global options are specified before the backend type and the backend options are specified after.
+
+***
+
+#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
+
+### Contact
+![versity logo](https://www.versity.com/wp-content/uploads/2022/12/cropped-android-chrome-512x512-1-32x32.png)
+info@versity.com <br />
++1 844 726 8826
+
+### @versitysoftware 
+[![linkedin](https://github.com/versity/versitygw/blob/assets/assets/linkedin.jpg)](https://www.linkedin.com/company/versity/) &nbsp; 
+[![twitter](https://github.com/versity/versitygw/blob/assets/assets/twitter.jpg)](https://twitter.com/VersitySoftware) &nbsp;
+[![facebook](https://github.com/versity/versitygw/blob/assets/assets/facebook.jpg)](https://www.facebook.com/versitysoftware) &nbsp;
+[![instagram](https://github.com/versity/versitygw/blob/assets/assets/instagram.jpg)](https://www.instagram.com/versitysoftware/) &nbsp;
+

assets/logo-white.svg

@@ -1,13 +0,0 @@
-<svg width="209" height="73" viewBox="0 0 209 73" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M47.427 30.0292C42.767 30.9134 40.4568 30.3528 40.2354 29.406C40.1625 29.0919 40.2887 28.6812 40.6122 28.1848C42.2518 25.6629 47.9677 22.1097 54.5119 19.5417C57.6668 18.3011 60.7686 17.3735 63.4818 16.8593C68.1376 15.976 70.4445 16.5363 70.6646 17.4827C70.7388 17.7976 70.6123 18.2078 70.2897 18.7048C68.6525 21.2235 62.9369 24.7783 56.3908 27.3497C53.2389 28.5879 50.1396 29.5145 47.4284 30.0287L47.427 30.0292Z" fill="white"/>
-<path d="M48.4866 39.9975C42.4383 40.4446 38.4817 39.0018 38.7133 37.4258C38.763 37.0777 39.0232 36.6989 39.4845 36.301C41.0086 34.9865 44.833 33.4548 49.9427 32.9982C50.1102 32.9836 50.2764 32.9697 50.4411 32.9574C56.4783 32.5107 60.4292 33.9536 60.1996 35.5285C60.1487 35.8758 59.8879 36.2549 59.4247 36.6544C57.8987 37.9699 54.0743 39.5016 48.9726 39.9577C48.8092 39.9718 48.648 39.9852 48.4871 39.997L48.4866 39.9975Z" fill="white"/>
-<path d="M44.5318 47.6969C44.5093 44.8455 42.0643 42.5515 39.0706 42.5727C37.1309 42.5866 35.4376 43.57 34.4912 45.0354L34.4904 45.0349L34.4755 45.0595C34.4417 45.1127 34.4089 45.1664 34.3773 45.2211L29.6958 52.93L10.1292 20.2803C8.6447 17.8035 5.33413 16.9425 2.73347 18.3555C0.192505 19.737 -0.72811 22.7797 0.616915 25.2285L0.615815 25.2295L0.643322 25.2748C0.666427 25.3162 0.688702 25.3579 0.712632 25.3987C0.736837 25.4396 0.763518 25.4786 0.788548 25.5179L28.8811 72.4019L28.8822 72.3956C29.031 72.6689 29.3281 72.8577 29.673 72.8577C29.9979 72.8577 30.2801 72.6909 30.4363 72.4433L30.4374 72.4467L43.8764 50.2044L43.8731 50.2028C44.2991 49.4586 44.5387 48.6041 44.5318 47.6969Z" fill="white"/>
-<path d="M69.6803 0.677278C68.2968 -1.32435 60.4824 1.26118 52.2266 6.45006C43.9703 11.641 38.3998 17.4709 39.7834 19.472C41.1669 21.4739 48.9818 18.8889 57.2376 13.6979C65.4929 8.50826 71.0644 2.67916 69.6803 0.677278Z" fill="white"/>
-<path d="M108.898 20.9902L101.884 39.4324H101.798L94.2404 20.9902H87.6453L99.6694 51.7645H103.261L115.498 20.9902H108.898Z" fill="white"/>
-<path d="M118.544 38.6442C118.626 38.1404 118.765 37.6502 118.962 37.1708C119.155 36.6966 119.415 36.2812 119.733 35.921C120.056 35.5652 120.444 35.2711 120.903 35.0454C121.363 34.8191 121.884 34.7061 122.468 34.7061C123.024 34.7061 123.523 34.8191 123.97 35.0454C124.415 35.2711 124.807 35.5704 125.141 35.9406C125.473 36.3138 125.741 36.7284 125.932 37.1912C126.127 37.6563 126.269 38.1404 126.35 38.6442H118.544ZM129.504 33.9109C128.711 32.956 127.701 32.2004 126.479 31.6458C125.252 31.0894 123.803 30.8105 122.135 30.8105C120.464 30.8105 119.009 31.0807 117.772 31.6236C116.534 32.1708 115.504 32.9168 114.681 33.8708C113.86 34.8261 113.242 35.9458 112.823 37.233C112.405 38.5185 112.197 39.9027 112.197 41.3869C112.197 42.8712 112.433 44.238 112.907 45.4838C113.38 46.7301 114.055 47.8124 114.933 48.726C115.807 49.6369 116.88 50.3485 118.147 50.8501C119.415 51.3556 120.838 51.6087 122.427 51.6087C124.6 51.6087 126.509 51.0785 128.166 50.0223C129.822 48.9665 131.014 47.4522 131.737 45.4838L126.35 44.6495C125.96 45.4181 125.454 46.0528 124.827 46.5552C124.2 47.0607 123.399 47.3139 122.427 47.3139C121.676 47.3139 121.036 47.1525 120.503 46.8397C119.975 46.5226 119.559 46.1185 119.256 45.6122C118.948 45.1158 118.722 44.5568 118.587 43.94C118.445 43.3197 118.376 42.7098 118.376 42.1055H131.863V41.4282C131.863 39.9705 131.668 38.5907 131.277 37.2935C130.887 35.9915 130.298 34.8648 129.504 33.9109Z" fill="white"/>
-<path d="M145.732 30.8105C144.645 30.8105 143.674 31.0424 142.81 31.5039C141.947 31.9694 141.223 32.6245 140.638 33.4759H140.557V31.4487H134.71V50.9689H140.557V42.1821C140.557 41.4413 140.597 40.684 140.68 39.9127C140.765 39.1471 140.959 38.4511 141.265 37.8294C141.572 37.2039 142.023 36.7028 142.621 36.3178C143.222 35.9345 144.021 35.7392 145.025 35.7392C145.915 35.7392 146.721 35.9806 147.448 36.4565L148.155 31.2864C147.765 31.1546 147.367 31.0424 146.966 30.9489C146.562 30.8562 146.152 30.8105 145.732 30.8105Z" fill="white"/>
-<path d="M162.394 40.9323C161.99 40.4929 161.537 40.1267 161.033 39.8387C160.538 39.5464 160.021 39.2923 159.492 39.0792C158.962 38.8669 158.447 38.679 157.948 38.505C157.444 38.3323 156.993 38.1483 156.591 37.9503C156.187 37.7498 155.868 37.5314 155.631 37.2935C155.395 37.0529 155.273 36.7497 155.273 36.3773C155.273 35.9001 155.477 35.5025 155.883 35.1837C156.283 34.8661 156.722 34.7056 157.195 34.7056C157.779 34.7056 158.339 34.8396 158.865 35.1063C159.396 35.3707 159.868 35.6883 160.286 36.0598L162.789 32.8403C161.955 32.1482 160.919 31.6401 159.681 31.3069C158.44 30.9754 157.28 30.8105 156.193 30.8105C155.219 30.8105 154.336 30.9706 153.539 31.2864C152.751 31.6062 152.067 32.0425 151.496 32.6002C150.925 33.1574 150.479 33.8269 150.16 34.6086C149.839 35.3929 149.68 36.2312 149.68 37.1321C149.68 37.9273 149.799 38.5985 150.034 39.1405C150.272 39.6869 150.577 40.1549 150.952 40.5525C151.327 40.9519 151.765 41.2816 152.268 41.5483C152.772 41.8141 153.269 42.0511 153.772 42.263C154.273 42.4761 154.775 42.6636 155.273 42.8381C155.776 43.0112 156.216 43.2044 156.591 43.4167C156.965 43.6281 157.271 43.8738 157.509 44.1522C157.743 44.432 157.861 44.7821 157.861 45.2032C157.861 45.8396 157.634 46.3547 157.175 46.7371C156.716 47.1194 156.167 47.3139 155.525 47.3139C154.661 47.3139 153.861 47.0868 153.124 46.6348C152.39 46.1863 151.739 45.6552 151.181 45.0479L148.511 48.3444C149.399 49.3802 150.479 50.1841 151.747 50.7531C153.011 51.3234 154.342 51.6087 155.735 51.6087C156.819 51.6087 157.841 51.4508 158.805 51.1293C159.76 50.81 160.605 50.3555 161.329 49.7591C162.051 49.1635 162.63 48.4388 163.06 47.591C163.494 46.7414 163.709 45.7735 163.709 44.6886C163.709 43.8408 163.59 43.1117 163.352 42.4992C163.118 41.8915 162.797 41.369 162.394 40.9323Z" fill="white"/>
-<path d="M169.093 22.4395C168.2 22.4395 167.45 22.7401 166.841 23.3317C166.233 23.9237 165.926 24.6332 165.926 25.458C165.926 26.3045 166.233 27.0171 166.841 27.5952C167.45 28.1777 168.2 28.4674 169.093 28.4674C169.98 28.4674 170.731 28.1777 171.339 27.5952C171.951 27.0171 172.257 26.3045 172.257 25.458C172.257 24.6332 171.951 23.9237 171.339 23.3317C170.731 22.7401 169.98 22.4395 169.093 22.4395Z" fill="white"/>
-<path d="M172.014 31.4492H166.167V50.9694H172.014V31.4492Z" fill="white"/>
-<path d="M202.482 31.4492L197.425 41.9607L192.336 31.4492H185.418H184.587H184.212V22.9521H178.364V31.4492H174.405V36.139H178.364V50.9693H184.212V36.139H188.688L194.422 47.9882L189.519 57.6672H195.949L209 31.4492H202.482Z" fill="white"/>
-</svg>

assets/logo.svg

@@ -1,13 +0,0 @@
-<svg width="150" height="53" viewBox="0 0 150 53" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M33.903 21.4665C30.5718 22.0985 28.9204 21.6977 28.7621 21.021C28.71 20.7965 28.8003 20.5028 29.0315 20.148C30.2036 18.3452 34.2896 15.8052 38.9676 13.9695C41.2229 13.0827 43.4402 12.4196 45.3797 12.052C48.7079 11.4206 50.357 11.8211 50.5143 12.4977C50.5673 12.7228 50.4769 13.016 50.2463 13.3713C49.076 15.1718 44.9902 17.7129 40.3107 19.551C38.0576 20.4362 35.8421 21.0985 33.904 21.4661L33.903 21.4665Z" fill="#191B2A"/>
-<path d="M34.6603 28.592C30.3367 28.9116 27.5083 27.8802 27.6739 26.7536C27.7095 26.5048 27.8955 26.234 28.2252 25.9495C29.3147 25.0099 32.0485 23.915 35.7012 23.5886C35.8209 23.5781 35.9397 23.5682 36.0575 23.5594C40.3731 23.2401 43.1974 24.2715 43.0332 25.3973C42.9969 25.6456 42.8105 25.9166 42.4794 26.2021C41.3885 27.1426 38.6547 28.2375 35.0077 28.5635C34.8909 28.5736 34.7757 28.5832 34.6607 28.5916L34.6603 28.592Z" fill="#191B2A"/>
-<path d="M31.8333 34.096C31.8172 32.0577 30.0694 30.4178 27.9294 30.433C26.5428 30.4429 25.3324 31.1459 24.6558 32.1934L24.6552 32.193L24.6446 32.2106C24.6204 32.2487 24.597 32.287 24.5744 32.3262L21.2279 37.8368L7.24078 14.4974C6.17961 12.7269 3.81307 12.1114 1.95401 13.1214C0.137611 14.109 -0.520485 16.2841 0.440998 18.0346L0.440212 18.0353L0.459875 18.0677C0.476391 18.0973 0.492314 18.1271 0.509421 18.1563C0.526723 18.1855 0.545796 18.2134 0.563688 18.2415L20.6455 51.7562L20.6463 51.7517C20.7527 51.947 20.965 52.082 21.2116 52.082C21.4438 52.082 21.6455 51.9628 21.7572 51.7858L21.758 51.7882L31.3648 35.8884L31.3624 35.8873C31.667 35.3553 31.8382 34.7444 31.8333 34.096Z" fill="#191B2A"/>
-<path d="M49.8105 0.484148C48.8215 -0.9467 43.2354 0.90155 37.3338 4.61078C31.4318 8.32151 27.4498 12.489 28.4388 13.9194C29.4278 15.3505 35.0143 13.5026 40.9159 9.79187C46.8171 6.08208 50.7999 1.91518 49.8105 0.484148Z" fill="#191B2A"/>
-<path d="M77.8452 15.0049L72.8313 28.1882H72.7699L67.3671 15.0049H62.6526L71.248 37.0037H73.8157L82.5626 15.0049H77.8452Z" fill="#191B2A"/>
-<path d="M84.7403 27.625C84.7988 27.2649 84.898 26.9144 85.0391 26.5717C85.1775 26.2328 85.363 25.9358 85.5906 25.6783C85.8211 25.424 86.0986 25.2138 86.4268 25.0524C86.7556 24.8907 87.1281 24.8098 87.5454 24.8098C87.9428 24.8098 88.2996 24.8907 88.619 25.0524C88.9373 25.2138 89.2178 25.4277 89.4565 25.6923C89.6938 25.9591 89.8848 26.2555 90.0216 26.5863C90.1614 26.9188 90.2623 27.2649 90.3207 27.625H84.7403ZM92.575 24.2414C92.0085 23.5588 91.2866 23.0187 90.4128 22.6222C89.5358 22.2245 88.5001 22.0251 87.3074 22.0251C86.113 22.0251 85.073 22.2183 84.1885 22.6063C83.3034 22.9975 82.5674 23.5308 81.9787 24.2128C81.3923 24.8956 80.9502 25.6961 80.6504 26.6162C80.352 27.5351 80.2031 28.5246 80.2031 29.5856C80.2031 30.6466 80.3723 31.6237 80.7108 32.5143C81.0488 33.4052 81.5314 34.1788 82.1593 34.8319C82.7839 35.483 83.5512 35.9918 84.4566 36.3503C85.363 36.7116 86.3801 36.8926 87.516 36.8926C89.0695 36.8926 90.434 36.5135 91.6183 35.7585C92.8022 35.0038 93.6544 33.9214 94.1716 32.5143L90.3207 31.9178C90.0419 32.4673 89.6798 32.921 89.2318 33.2801C88.7838 33.6415 88.2111 33.8225 87.516 33.8225C86.9795 33.8225 86.5218 33.7071 86.1411 33.4835C85.7636 33.2568 85.4658 32.9679 85.249 32.606C85.029 32.2512 84.8677 31.8516 84.7713 31.4106C84.6695 30.9672 84.6202 30.5312 84.6202 30.0993H94.2611V29.6151C94.2611 28.5731 94.1217 27.5867 93.8425 26.6594C93.5636 25.7287 93.1428 24.9233 92.575 24.2414Z" fill="#191B2A"/>
-<path d="M104.175 22.0251C103.398 22.0251 102.704 22.1909 102.087 22.5208C101.47 22.8536 100.952 23.3219 100.534 23.9304H100.476V22.4813H96.2964V36.4352H100.476V30.154C100.476 29.6245 100.505 29.0831 100.564 28.5318C100.625 27.9845 100.763 27.4869 100.982 27.0425C101.202 26.5954 101.524 26.2372 101.952 25.962C102.381 25.688 102.952 25.5484 103.67 25.5484C104.306 25.5484 104.883 25.721 105.402 26.0611L105.908 22.3653C105.629 22.2711 105.345 22.1909 105.058 22.124C104.769 22.0578 104.476 22.0251 104.175 22.0251Z" fill="#191B2A"/>
-<path d="M116.086 29.2606C115.797 28.9465 115.474 28.6847 115.114 28.4789C114.759 28.2699 114.39 28.0883 114.012 27.9359C113.633 27.7842 113.265 27.6498 112.908 27.5255C112.548 27.402 112.225 27.2705 111.938 27.129C111.649 26.9856 111.421 26.8295 111.252 26.6594C111.083 26.4875 110.996 26.2707 110.996 26.0045C110.996 25.6634 111.142 25.3792 111.432 25.1513C111.718 24.9242 112.031 24.8095 112.37 24.8095C112.787 24.8095 113.187 24.9053 113.563 25.0959C113.944 25.285 114.28 25.512 114.579 25.7775L116.369 23.4761C115.772 22.9813 115.032 22.6182 114.147 22.38C113.26 22.143 112.43 22.0251 111.653 22.0251C110.958 22.0251 110.326 22.1396 109.756 22.3653C109.193 22.5939 108.704 22.9058 108.296 23.3044C107.887 23.7028 107.569 24.1814 107.341 24.7402C107.111 25.3008 106.998 25.9001 106.998 26.5441C106.998 27.1125 107.083 27.5923 107.251 27.9798C107.421 28.3703 107.639 28.7049 107.907 28.9891C108.175 29.2746 108.489 29.5103 108.848 29.7009C109.208 29.8909 109.563 30.0604 109.923 30.2119C110.281 30.3642 110.64 30.4983 110.996 30.623C111.355 30.7467 111.67 30.8848 111.938 31.0365C112.206 31.1877 112.424 31.3634 112.594 31.5624C112.762 31.7623 112.846 32.0127 112.846 32.3137C112.846 32.7686 112.684 33.1368 112.355 33.4101C112.027 33.6835 111.635 33.8225 111.176 33.8225C110.559 33.8225 109.987 33.6601 109.46 33.337C108.935 33.0164 108.47 32.6368 108.071 32.2027L106.162 34.5591C106.797 35.2995 107.569 35.8742 108.475 36.2809C109.379 36.6886 110.331 36.8926 111.326 36.8926C112.101 36.8926 112.832 36.7797 113.521 36.5499C114.204 36.3217 114.807 35.9967 115.325 35.5704C115.841 35.1447 116.255 34.6266 116.562 34.0205C116.872 33.4132 117.026 32.7213 117.026 31.9458C117.026 31.3397 116.941 30.8186 116.771 30.3807C116.604 29.9463 116.375 29.5728 116.086 29.2606Z" fill="#191B2A"/>
-<path d="M120.875 16.041C120.237 16.041 119.701 16.2559 119.265 16.6788C118.83 17.102 118.611 17.6092 118.611 18.1988C118.611 18.8039 118.83 19.3133 119.265 19.7266C119.701 20.1429 120.237 20.35 120.875 20.35C121.509 20.35 122.046 20.1429 122.481 19.7266C122.918 19.3133 123.137 18.8039 123.137 18.1988C123.137 17.6092 122.918 17.102 122.481 16.6788C122.046 16.2559 121.509 16.041 120.875 16.041Z" fill="#191B2A"/>
-<path d="M122.963 22.4814H118.784V36.4353H122.963V22.4814Z" fill="#191B2A"/>
-<path d="M144.743 22.4813L141.128 29.9954L137.49 22.4813H132.545H131.951H131.683V16.4072H127.503V22.4813H124.673V25.8338H127.503V36.4351H131.683V25.8338H134.882L138.982 34.3041L135.477 41.223H140.073L149.402 22.4813H144.743Z" fill="#191B2A"/>
-</svg>

auth/acl.go

@@ -0,0 +1,252 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+)
+
+type ACL struct {
+	ACL      types.BucketCannedACL
+	Owner    string
+	Grantees []Grantee
+}
+
+type Grantee struct {
+	Permission types.Permission
+	Access     string
+}
+
+type GetBucketAclOutput struct {
+	Owner             *types.Owner
+	AccessControlList AccessControlList
+}
+
+type AccessControlList struct {
+	Grants []types.Grant `xml:"Grant"`
+}
+type AccessControlPolicy struct {
+	AccessControlList AccessControlList `xml:"AccessControlList"`
+	Owner             types.Owner
+}
+
+func ParseACL(data []byte) (ACL, error) {
+	if len(data) == 0 {
+		return ACL{}, nil
+	}
+
+	var acl ACL
+	if err := json.Unmarshal(data, &acl); err != nil {
+		return acl, fmt.Errorf("parse acl: %w", err)
+	}
+	return acl, nil
+}
+
+func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
+	var acl ACL
+	if err := json.Unmarshal(data, &acl); err != nil {
+		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
+	}
+
+	grants := []types.Grant{}
+
+	for _, elem := range acl.Grantees {
+		acs := elem.Access
+		grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
+	}
+
+	return GetBucketAclOutput{
+		Owner: &types.Owner{
+			ID: &acl.Owner,
+		},
+		AccessControlList: AccessControlList{
+			Grants: grants,
+		},
+	}, nil
+}
+
+func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
+	if input == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	// if the ACL is specified, set the ACL, else replace the grantees
+	if input.ACL != "" {
+		acl.ACL = input.ACL
+		acl.Grantees = []Grantee{}
+	} else {
+		grantees := []Grantee{}
+		accs := []string{}
+
+		if input.GrantRead != nil {
+			fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
+
+			if *input.GrantFullControl != "" {
+				fullControlList = splitUnique(*input.GrantFullControl, ",")
+				for _, str := range fullControlList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+				}
+			}
+			if *input.GrantRead != "" {
+				readList = splitUnique(*input.GrantRead, ",")
+				for _, str := range readList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+				}
+			}
+			if *input.GrantReadACP != "" {
+				readACPList = splitUnique(*input.GrantReadACP, ",")
+				for _, str := range readACPList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+				}
+			}
+			if *input.GrantWrite != "" {
+				writeList = splitUnique(*input.GrantWrite, ",")
+				for _, str := range writeList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+				}
+			}
+			if *input.GrantWriteACP != "" {
+				writeACPList = splitUnique(*input.GrantWriteACP, ",")
+				for _, str := range writeACPList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+				}
+			}
+
+			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
+		} else {
+			cache := make(map[string]bool)
+			for _, grt := range input.AccessControlPolicy.Grants {
+				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
+				if _, ok := cache[*grt.Grantee.ID]; !ok {
+					cache[*grt.Grantee.ID] = true
+					accs = append(accs, *grt.Grantee.ID)
+				}
+			}
+		}
+
+		// Check if the specified accounts exist
+		accList, err := CheckIfAccountsExist(accs, iam)
+		if err != nil {
+			return nil, err
+		}
+		if len(accList) > 0 {
+			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
+		}
+
+		acl.Grantees = grantees
+		acl.ACL = ""
+	}
+
+	result, err := json.Marshal(acl)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
+	result := []string{}
+
+	for _, acc := range accs {
+		_, err := iam.GetUserAccount(acc)
+		if err != nil {
+			if err == ErrNoSuchUser {
+				result = append(result, acc)
+				continue
+			}
+			if err == ErrNotSupported {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			}
+			return nil, fmt.Errorf("check user account: %w", err)
+		}
+	}
+	return result, nil
+}
+
+func splitUnique(s, divider string) []string {
+	elements := strings.Split(s, divider)
+	uniqueElements := make(map[string]bool)
+	result := make([]string, 0, len(elements))
+
+	for _, element := range elements {
+		if _, ok := uniqueElements[element]; !ok {
+			result = append(result, element)
+			uniqueElements[element] = true
+		}
+	}
+
+	return result
+}
+
+func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
+	if isRoot {
+		return nil
+	}
+
+	if acl.Owner == access {
+		return nil
+	}
+
+	if acl.ACL != "" {
+		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+
+		return nil
+	} else {
+		grantee := Grantee{Access: access, Permission: permission}
+		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
+
+		isFound := false
+
+		for _, grt := range acl.Grantees {
+			if grt == grantee || grt == granteeFullCtrl {
+				isFound = true
+				break
+			}
+		}
+
+		if isFound {
+			return nil
+		}
+	}
+
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}
+
+func IsAdmin(acct Account, isRoot bool) error {
+	if isRoot {
+		return nil
+	}
+
+	if acct.Role == "admin" {
+		return nil
+	}
+
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
+}

auth/iam.go

@@ -0,0 +1,103 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+	"fmt"
+	"time"
+)
+
+// Account is a gateway IAM account
+type Account struct {
+	Access    string `json:"access"`
+	Secret    string `json:"secret"`
+	Role      string `json:"role"`
+	UserID    int    `json:"userID"`
+	GroupID   int    `json:"groupID"`
+	ProjectID int    `json:"projectID"`
+}
+
+// IAMService is the interface for all IAM service implementations
+//
+//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
+type IAMService interface {
+	CreateAccount(account Account) error
+	GetUserAccount(access string) (Account, error)
+	DeleteUserAccount(access string) error
+	ListUserAccounts() ([]Account, error)
+	Shutdown() error
+}
+
+var ErrNoSuchUser = errors.New("user not found")
+
+type Opts struct {
+	Dir                string
+	LDAPServerURL      string
+	LDAPBindDN         string
+	LDAPPassword       string
+	LDAPQueryBase      string
+	LDAPObjClasses     string
+	LDAPAccessAtr      string
+	LDAPSecretAtr      string
+	LDAPRoleAtr        string
+	S3Access           string
+	S3Secret           string
+	S3Region           string
+	S3Bucket           string
+	S3Endpoint         string
+	S3DisableSSlVerfiy bool
+	S3Debug            bool
+	CacheDisable       bool
+	CacheTTL           int
+	CachePrune         int
+}
+
+func New(o *Opts) (IAMService, error) {
+	var svc IAMService
+	var err error
+
+	switch {
+	case o.Dir != "":
+		svc, err = NewInternal(o.Dir)
+		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
+	case o.LDAPServerURL != "":
+		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
+			o.LDAPObjClasses)
+		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
+	case o.S3Endpoint != "":
+		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
+		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
+			o.S3Endpoint, o.S3Bucket)
+	default:
+		// if no iam options selected, default to the single user mode
+		fmt.Println("No IAM service configured, enabling single account mode")
+		return IAMServiceSingle{}, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if o.CacheDisable {
+		return svc, nil
+	}
+
+	return NewCache(svc,
+		time.Duration(o.CacheTTL)*time.Second,
+		time.Duration(o.CachePrune)*time.Second), nil
+}

auth/iam_cache.go

@@ -0,0 +1,179 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+)
+
+// IAMCache is an in memory cache of the IAM accounts
+// with expiration. This helps to alleviate the load on
+// the real IAM service if the gateway is handling
+// many requests. This forwards account updates to the
+// underlying service, and returns cached results while
+// the in memory account is not expired.
+type IAMCache struct {
+	service  IAMService
+	iamcache *icache
+	cancel   context.CancelFunc
+}
+
+var _ IAMService = &IAMCache{}
+
+type item struct {
+	value Account
+	exp   time.Time
+}
+
+type icache struct {
+	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
+}
+
+func (i *icache) set(k string, v Account) {
+	cpy := v
+	i.Lock()
+	i.items[k] = item{
+		exp:   time.Now().Add(i.expire),
+		value: cpy,
+	}
+	i.Unlock()
+}
+
+func (i *icache) get(k string) (Account, bool) {
+	i.RLock()
+	v, ok := i.items[k]
+	i.RUnlock()
+	if !ok || !v.exp.After(time.Now()) {
+		return Account{}, false
+	}
+	return v.value, true
+}
+
+func (i *icache) Delete(k string) {
+	i.Lock()
+	delete(i.items, k)
+	i.Unlock()
+}
+
+func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+
+		now := time.Now()
+
+		i.Lock()
+		// prune expired entries
+		for k, v := range i.items {
+			if now.After(v.exp) {
+				delete(i.items, k)
+			}
+		}
+		i.Unlock()
+
+		// sleep for the clean interval or context cancelation,
+		// whichever comes first
+		select {
+		case <-ctx.Done():
+		case <-time.After(interval):
+		}
+	}
+}
+
+// NewCache initializes an IAM cache for the provided service. The expireTime
+// is the duration a cache entry can be valid, and the cleanupInterval is
+// how often to scan cache and cleanup expired entries.
+func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
+	i := &IAMCache{
+		service: service,
+		iamcache: &icache{
+			items:  make(map[string]item),
+			expire: expireTime,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go i.iamcache.gcCache(ctx, cleanupInterval)
+	i.cancel = cancel
+
+	return i
+}
+
+// CreateAccount send create to IAM service and creates an account cache entry
+func (c *IAMCache) CreateAccount(account Account) error {
+	err := c.service.CreateAccount(account)
+	if err != nil {
+		return err
+	}
+
+	// we need a copy of account to be able to store beyond the
+	// lifetime of the request, otherwise Fiber will reuse and corrupt
+	// these entries
+	acct := Account{
+		Access: strings.Clone(account.Access),
+		Secret: strings.Clone(account.Secret),
+		Role:   strings.Clone(account.Role),
+	}
+
+	c.iamcache.set(acct.Access, acct)
+	return nil
+}
+
+// GetUserAccount retrieves the cache account if it is in the cache and not
+// expired. Otherwise retrieves from underlying IAM service and caches
+// result for the expire duration.
+func (c *IAMCache) GetUserAccount(access string) (Account, error) {
+	acct, found := c.iamcache.get(access)
+	if found {
+		return acct, nil
+	}
+
+	a, err := c.service.GetUserAccount(access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	c.iamcache.set(access, a)
+	return a, nil
+}
+
+// DeleteUserAccount deletes account from IAM service and cache
+func (c *IAMCache) DeleteUserAccount(access string) error {
+	err := c.service.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.Delete(access)
+	return nil
+}
+
+// ListUserAccounts is a passthrough to the underlying service and
+// does not make use of the cache
+func (c *IAMCache) ListUserAccounts() ([]Account, error) {
+	return c.service.ListUserAccounts()
+}
+
+// Shutdown graceful termination of service
+func (c *IAMCache) Shutdown() error {
+	c.cancel()
+	return nil
+}

auth/iam_internal.go

@@ -0,0 +1,335 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+const (
+	iamFile       = "users.json"
+	iamBackupFile = "users.json.backup"
+)
+
+// IAMServiceInternal manages the internal IAM service
+type IAMServiceInternal struct {
+	dir string
+}
+
+// UpdateAcctFunc accepts the current data and returns the new data to be stored
+type UpdateAcctFunc func([]byte) ([]byte, error)
+
+// iAMConfig stores all internal IAM accounts
+type iAMConfig struct {
+	AccessAccounts map[string]Account `json:"accessAccounts"`
+}
+
+var _ IAMService = &IAMServiceInternal{}
+
+// NewInternal creates a new instance for the Internal IAM service
+func NewInternal(dir string) (*IAMServiceInternal, error) {
+	i := &IAMServiceInternal{
+		dir: dir,
+	}
+
+	err := i.initIAM()
+	if err != nil {
+		return nil, fmt.Errorf("init iam: %w", err)
+	}
+
+	return i, nil
+}
+
+// CreateAccount creates a new IAM account. Returns an error if the account
+// already exists.
+func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		_, ok := conf.AccessAccounts[account.Access]
+		if ok {
+			return nil, fmt.Errorf("account already exists")
+		}
+		conf.AccessAccounts[account.Access] = account
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
+// GetUserAccount retrieves account info for the requested user. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	conf, err := s.getIAM()
+	if err != nil {
+		return Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+// DeleteUserAccount deletes the specified user account. Does not check if
+// account exists.
+func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		delete(conf.AccessAccounts, access)
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
+// ListUserAccounts lists all the user accounts stored.
+func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getIAM()
+	if err != nil {
+		return []Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// Shutdown graceful termination of service
+func (s *IAMServiceInternal) Shutdown() error {
+	return nil
+}
+
+const (
+	iamMode = 0600
+)
+
+func (s *IAMServiceInternal) initIAM() error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	_, err := os.ReadFile(fname)
+	if errors.Is(err, fs.ErrNotExist) {
+		b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
+		if err != nil {
+			return fmt.Errorf("marshal default iam: %w", err)
+		}
+		err = os.WriteFile(fname, b, iamMode)
+		if err != nil {
+			return fmt.Errorf("write default iam: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
+	b, err := s.readIAMData()
+	if err != nil {
+		return iAMConfig{}, err
+	}
+
+	return parseIAM(b)
+}
+
+func parseIAM(b []byte) (iAMConfig, error) {
+	var conf iAMConfig
+	if err := json.Unmarshal(b, &conf); err != nil {
+		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
+	}
+
+	return conf, nil
+}
+
+const (
+	backoff  = 100 * time.Millisecond
+	maxretry = 300
+)
+
+func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
+	// We are going to be racing with other running gateways without any
+	// coordination. So we might find the file does not exist at times.
+	// For this case we need to retry for a while assuming the other gateway
+	// will eventually write the file. If it doesn't after the max retries,
+	// then we will return the error.
+
+	retries := 0
+
+	for {
+		b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+			return nil, fmt.Errorf("read iam file: %w", err)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		return b, nil
+	}
+}
+
+func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
+	// We are going to be racing with other running gateways without any
+	// coordination. So the strategy here is to read the current file data.
+	// If the file doesn't exist, then we assume someone else is currently
+	// updating the file. So we just need to keep retrying. We also need
+	// to make sure the data is consistent within a single update. So racing
+	// writes to a file would possibly leave this in some invalid state.
+	// We can get atomic updates with rename. If we read the data, update
+	// the data, write to a temp file, then rename the tempfile back to the
+	// data file. This should always result in a complete data image.
+
+	// There is at least one unsolved failure mode here.
+	// If a gateway removes the data file and then crashes, all other
+	// gateways will retry forever thinking that the original will eventually
+	// write the file.
+
+	retries := 0
+	fname := filepath.Join(s.dir, iamFile)
+
+	for {
+		b, err := os.ReadFile(fname)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+
+			// we have been unsuccessful trying to read the iam file
+			// so this must be the case where something happened and
+			// the file did not get updated successfully, and probably
+			// isn't going to be. The recovery procedure would be to
+			// copy the backup file into place of the original.
+			return fmt.Errorf("no iam file, needs backup recovery")
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("read iam file: %w", err)
+		}
+
+		// reset retries on successful read
+		retries = 0
+
+		err = os.Remove(iamFile)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			time.Sleep(backoff)
+			continue
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("remove old iam file: %w", err)
+		}
+
+		// save copy of data
+		datacopy := make([]byte, len(b))
+		copy(datacopy, b)
+
+		// make a backup copy in case we crash before update
+		// this is after remove, so there is a small window something
+		// can go wrong, but the remove should barrier other gateways
+		// from trying to write backup at the same time. Only one
+		// gateway will successfully remove the file.
+		os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
+
+		b, err = update(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return fmt.Errorf("update iam data: %w", err)
+		}
+
+		err = s.writeTempFile(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return err
+		}
+
+		break
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) writeTempFile(b []byte) error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	f, err := os.CreateTemp(s.dir, iamFile)
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	_, err = f.Write(b)
+	if err != nil {
+		return fmt.Errorf("write temp file: %w", err)
+	}
+
+	err = os.Rename(f.Name(), fname)
+	if err != nil {
+		return fmt.Errorf("rename temp file: %w", err)
+	}
+
+	return nil
+}

auth/iam_ldap.go

@@ -0,0 +1,133 @@
+package auth
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-ldap/ldap/v3"
+)
+
+type LdapIAMService struct {
+	conn       *ldap.Conn
+	queryBase  string
+	objClasses []string
+	accessAtr  string
+	secretAtr  string
+	roleAtr    string
+}
+
+var _ IAMService = &LdapIAMService{}
+
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+		return nil, fmt.Errorf("required parameters list not fully provided")
+	}
+	conn, err := ldap.Dial("tcp", url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
+	}
+
+	err = conn.Bind(bindDN, pass)
+	if err != nil {
+		return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
+	}
+	return &LdapIAMService{
+		conn:       conn,
+		queryBase:  queryBase,
+		objClasses: strings.Split(objClasses, ","),
+		accessAtr:  accAtr,
+		secretAtr:  secAtr,
+		roleAtr:    roleAtr,
+	}, nil
+}
+
+func (ld *LdapIAMService) CreateAccount(account Account) error {
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry.Attribute("objectClass", ld.objClasses)
+	userEntry.Attribute(ld.accessAtr, []string{account.Access})
+	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
+	userEntry.Attribute(ld.roleAtr, []string{account.Role})
+
+	err := ld.conn.Add(userEntry)
+	if err != nil {
+		return fmt.Errorf("error adding an entry: %w", err)
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	result, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return Account{}, err
+	}
+
+	entry := result.Entries[0]
+	return Account{
+		Access: entry.GetAttributeValue(ld.accessAtr),
+		Secret: entry.GetAttributeValue(ld.secretAtr),
+		Role:   entry.GetAttributeValue(ld.roleAtr),
+	}, nil
+}
+
+func (ld *LdapIAMService) DeleteUserAccount(access string) error {
+	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+
+	err := ld.conn.Del(delReq)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
+	searchFilter := ""
+	for _, el := range ld.objClasses {
+		searchFilter += fmt.Sprintf("(objectClass=%v)", el)
+	}
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(&%v)", searchFilter),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	resp, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	result := []Account{}
+	for _, el := range resp.Entries {
+		result = append(result, Account{
+			Access: el.GetAttributeValue(ld.accessAtr),
+			Secret: el.GetAttributeValue(ld.secretAtr),
+			Role:   el.GetAttributeValue(ld.roleAtr),
+		})
+	}
+
+	return result, nil
+}
+
+// Shutdown graceful termination of service
+func (ld *LdapIAMService) Shutdown() error {
+	return ld.conn.Close()
+}

auth/iam_s3_object.go

@@ -0,0 +1,263 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+)
+
+// IAMServiceS3 stores user accounts in an S3 object
+// The endpoint, credentials, bucket, and region are provided
+// from cli configuration.
+// The object format and name is the same as the internal IAM service:
+// coming from iAMConfig and iamFile in iam_internal.
+
+type IAMServiceS3 struct {
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
+	sslSkipVerify bool
+	debug         bool
+	client        *s3.Client
+}
+
+var _ IAMService = &IAMServiceS3{}
+
+func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+	if access == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service access key")
+	}
+	if secret == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service secret key")
+	}
+	if region == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service region")
+	}
+	if bucket == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service bucket")
+	}
+	if endpoint == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service endpoint")
+	}
+
+	i := &IAMServiceS3{
+		access:        access,
+		secret:        secret,
+		region:        region,
+		bucket:        bucket,
+		endpoint:      endpoint,
+		sslSkipVerify: sslSkipVerify,
+		debug:         debug,
+	}
+
+	cfg, err := i.getConfig()
+	if err != nil {
+		return nil, fmt.Errorf("init s3 IAM: %v", err)
+	}
+
+	i.client = s3.NewFromConfig(cfg)
+	return i, nil
+}
+
+func (s *IAMServiceS3) CreateAccount(account Account) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[account.Access]
+	if ok {
+		return fmt.Errorf("account already exists")
+	}
+	conf.AccessAccounts[account.Access] = account
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[access]
+	if !ok {
+		return fmt.Errorf("account does not exist")
+	}
+	delete(conf.AccessAccounts, access)
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.region,
+		HostnameImmutable: true,
+	}, nil
+}
+
+func (s *IAMServiceS3) Shutdown() error {
+	return nil
+}
+
+func (s *IAMServiceS3) getConfig() (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.region),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(context.Background(), opts...)
+}
+
+func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
+	obj := iamFile
+
+	out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
+		Bucket: &s.bucket,
+		Key:    &obj,
+	})
+	if err != nil {
+		// if the error is object not exists,
+		// init empty accounts stuct and return that
+		var nsk *types.NoSuchKey
+		if errors.As(err, &nsk) {
+			return iAMConfig{}, nil
+		}
+		var apiErr smithy.APIError
+		if errors.As(err, &apiErr) {
+			if apiErr.ErrorCode() == "NotFound" {
+				return iAMConfig{}, nil
+			}
+		}
+
+		// all other errors, return the error
+		return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
+	}
+
+	defer out.Body.Close()
+
+	b, err := io.ReadAll(out.Body)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
+	}
+
+	conf, err := parseIAM(b)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
+	}
+
+	return conf, nil
+}
+
+func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
+	b, err := json.Marshal(conf)
+	if err != nil {
+		return fmt.Errorf("failed to serialize iam: %w", err)
+	}
+
+	obj := iamFile
+	uploader := manager.NewUploader(s.client)
+	upinfo := &s3.PutObjectInput{
+		Body:   bytes.NewReader(b),
+		Bucket: &s.bucket,
+		Key:    &obj,
+	}
+	_, err = uploader.Upload(context.Background(), upinfo)
+	if err != nil {
+		return fmt.Errorf("store accounts in %v: %w", iamFile, err)
+	}
+
+	return nil
+}

auth/iam_single.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+)
+
+// IAMServiceSingle manages the single tenant (root-only) IAM service
+type IAMServiceSingle struct{}
+
+var _ IAMService = &IAMServiceSingle{}
+
+var ErrNotSupported = errors.New("method is not supported")
+
+// CreateAccount not valid in single tenant mode
+func (IAMServiceSingle) CreateAccount(account Account) error {
+	return ErrNotSupported
+}
+
+// GetUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	return Account{}, ErrNotSupported
+}
+
+// DeleteUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) DeleteUserAccount(access string) error {
+	return ErrNotSupported
+}
+
+// ListUserAccounts no accounts in single tenant mode
+func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
+	return []Account{}, nil
+}
+
+// Shutdown graceful termination of service
+func (IAMServiceSingle) Shutdown() error {
+	return nil
+}

backend/backend.go

@@ -0,0 +1,197 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+	"github.com/versity/versitygw/s3select"
+)
+
+//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
+type Backend interface {
+	fmt.Stringer
+	Shutdown()
+
+	// bucket operations
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
+	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
+	CreateBucket(context.Context, *s3.CreateBucketInput) error
+	PutBucketAcl(_ context.Context, bucket string, data []byte) error
+	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
+
+	// multipart operations
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
+	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
+	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+
+	// standard object operations
+	PutObject(context.Context, *s3.PutObjectInput) (string, error)
+	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
+
+	// special case object operations
+	RestoreObject(context.Context, *s3.RestoreObjectInput) error
+	SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
+
+	// object tags operations
+	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
+	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
+	DeleteObjectTagging(_ context.Context, bucket, object string) error
+
+	// non AWS actions
+	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
+}
+
+type BackendUnsupported struct{}
+
+var _ Backend = &BackendUnsupported{}
+
+func New() Backend {
+	return &BackendUnsupported{}
+}
+func (BackendUnsupported) Shutdown() {}
+func (BackendUnsupported) String() string {
+	return "Unsupported"
+}
+func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
+	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
+	return func(w *bufio.Writer) {
+		var getProgress s3select.GetProgress
+		progress := input.RequestProgress
+		if progress != nil && *progress.Enabled {
+			getProgress = func() (bytesScanned int64, bytesProcessed int64) {
+				return -1, -1
+			}
+		}
+		mh := s3select.NewMessageHandler(ctx, w, getProgress)
+		apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
+		mh.FinishWithError(apiErr.Code, apiErr.Description)
+	}
+}
+
+func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
+	return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}

backend/common.go

@@ -0,0 +1,122 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/fs"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	// RFC3339TimeFormat RFC3339 time format
+	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+)
+
+func IsValidBucketName(name string) bool { return true }
+
+type ByBucketName []s3response.ListAllMyBucketsEntry
+
+func (d ByBucketName) Len() int           { return len(d) }
+func (d ByBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d ByBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
+
+type ByObjectName []types.Object
+
+func (d ByObjectName) Len() int           { return len(d) }
+func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
+
+func GetStringPtr(s string) *string {
+	return &s
+}
+
+func GetTimePtr(t time.Time) *time.Time {
+	return &t
+}
+
+var (
+	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
+)
+
+// ParseRange parses input range header and returns startoffset, length, and
+// error. If no endoffset specified, then length is set to -1.
+func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
+	if acceptRange == "" {
+		return 0, fi.Size(), nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) < 2 {
+		return 0, 0, errInvalidRange
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) < 1 || len(bRange) > 2 {
+		return 0, 0, errInvalidRange
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, 0, errInvalidRange
+	}
+
+	endOffset := int64(-1)
+	if len(bRange) == 1 || bRange[1] == "" {
+		return startOffset, endOffset, nil
+	}
+
+	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, 0, errInvalidRange
+	}
+
+	if endOffset < startOffset {
+		return 0, 0, errInvalidRange
+	}
+
+	return startOffset, endOffset - startOffset + 1, nil
+}
+
+func GetMultipartMD5(parts []types.CompletedPart) string {
+	var partsEtagBytes []byte
+	for _, part := range parts {
+		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
+	}
+	s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
+	return s3MD5
+}
+
+func getEtagBytes(etag string) []byte {
+	decode, err := hex.DecodeString(strings.ReplaceAll(etag, string('"'), ""))
+	if err != nil {
+		return []byte(etag)
+	}
+	return decode
+}
+
+func md5String(data []byte) string {
+	sum := md5.Sum(data)
+	return hex.EncodeToString(sum[:])
+}

backend/posix/posix_darwin.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/posix/posix_linux.go

@@ -0,0 +1,164 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+const procfddir = "/proc/self/fd"
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
+	// This can help reduce contention within the namespace (parent directories),
+	// etc. And will auto cleanup the inode on close if we never link this
+	// file descriptor into the namespace.
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	if err != nil {
+		// O_TMPFILE not supported, try fallback
+		err := os.MkdirAll(dir, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("make temp dir: %w", err)
+		}
+		f, err := os.CreateTemp(dir,
+			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+		if err != nil {
+			return nil, err
+		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		// falloc is best effort, its fine if this fails
+		if size > 0 {
+			tmp.falloc()
+		}
+		return tmp, nil
+	}
+
+	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
+	// later to link file into namespace
+	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
+
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	// falloc is best effort, its fine if this fails
+	if size > 0 {
+		tmp.falloc()
+	}
+	return tmp, nil
+}
+
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
+func (tmp *tmpfile) link() error {
+	// We use Linkat/Rename as the atomic operation for object puts. The
+	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
+	// with any other simultaneous uploads. The final operation is to move the
+	// temp file into place for the object. This ensures the object semantics
+	// of last upload completed wins and is not some combination of writes
+	// from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
+	}
+
+	procdir, err := os.Open(procfddir)
+	if err != nil {
+		return fmt.Errorf("open proc dir: %w", err)
+	}
+	defer procdir.Close()
+
+	dir, err := os.Open(filepath.Dir(objPath))
+	if err != nil {
+		return fmt.Errorf("open parent dir: %w", err)
+	}
+	defer dir.Close()
+
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if err != nil {
+		return fmt.Errorf("link tmpfile (%q in %q): %w",
+			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/posix/posix_windows.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length")
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/s3proxy/client.go

@@ -0,0 +1,79 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+)
+
+func (s *S3Proxy) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
+	cfg, err := s.getConfig(ctx, s.access, s.secret)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg), nil
+}
+
+func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.disableChecksum {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(ctx, opts...)
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}

backend/s3proxy/s3.go

@@ -0,0 +1,651 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+type S3Proxy struct {
+	backend.BackendUnsupported
+
+	access          string
+	secret          string
+	endpoint        string
+	awsRegion       string
+	disableChecksum bool
+	sslSkipVerify   bool
+	debug           bool
+}
+
+func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3Proxy {
+	return &S3Proxy{
+		access:          access,
+		secret:          secret,
+		endpoint:        endpoint,
+		awsRegion:       region,
+		disableChecksum: disableChecksum,
+		sslSkipVerify:   sslSkipVerify,
+		debug:           debug,
+	}
+}
+
+func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, err
+	}
+
+	output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
+	err = handleError(err)
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, err
+	}
+
+	var buckets []s3response.ListAllMyBucketsEntry
+	for _, b := range output.Buckets {
+		buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+			Name:         *b.Name,
+			CreationDate: *b.CreationDate,
+		})
+	}
+
+	return s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID: *output.Owner.ID,
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: buckets,
+		},
+	}, nil
+}
+
+func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.HeadBucket(ctx, input)
+	err = handleError(err)
+
+	return out, err
+}
+
+func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.CreateBucket(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteBucket(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.CreateMultipartUpload(ctx, input)
+	err = handleError(err)
+	return out, err
+}
+
+func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.CompleteMultipartUpload(ctx, input)
+	err = handleError(err)
+	return out, err
+}
+
+func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.AbortMultipartUpload(ctx, input)
+	err = handleError(err)
+	return err
+}
+
+func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, err
+	}
+
+	output, err := client.ListMultipartUploads(ctx, input)
+	err = handleError(err)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, err
+	}
+
+	var uploads []s3response.Upload
+	for _, u := range output.Uploads {
+		uploads = append(uploads, s3response.Upload{
+			Key:      *u.Key,
+			UploadID: *u.UploadId,
+			Initiator: s3response.Initiator{
+				ID:          *u.Initiator.ID,
+				DisplayName: *u.Initiator.DisplayName,
+			},
+			Owner: s3response.Owner{
+				ID:          *u.Owner.ID,
+				DisplayName: *u.Owner.DisplayName,
+			},
+			StorageClass: string(u.StorageClass),
+			Initiated:    u.Initiated.Format(backend.RFC3339TimeFormat),
+		})
+	}
+
+	var cps []s3response.CommonPrefix
+	for _, c := range output.CommonPrefixes {
+		cps = append(cps, s3response.CommonPrefix{
+			Prefix: *c.Prefix,
+		})
+	}
+
+	return s3response.ListMultipartUploadsResult{
+		Bucket:             *output.Bucket,
+		KeyMarker:          *output.KeyMarker,
+		UploadIDMarker:     *output.UploadIdMarker,
+		NextKeyMarker:      *output.NextKeyMarker,
+		NextUploadIDMarker: *output.NextUploadIdMarker,
+		Delimiter:          *output.Delimiter,
+		Prefix:             *output.Prefix,
+		EncodingType:       string(output.EncodingType),
+		MaxUploads:         int(*output.MaxUploads),
+		IsTruncated:        *output.IsTruncated,
+		Uploads:            uploads,
+		CommonPrefixes:     cps,
+	}, nil
+}
+
+func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListPartsResult{}, err
+	}
+
+	output, err := client.ListParts(ctx, input)
+	err = handleError(err)
+	if err != nil {
+		return s3response.ListPartsResult{}, err
+	}
+
+	var parts []s3response.Part
+	for _, p := range output.Parts {
+		parts = append(parts, s3response.Part{
+			PartNumber:   int(*p.PartNumber),
+			LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
+			ETag:         *p.ETag,
+			Size:         *p.Size,
+		})
+	}
+	pnm, err := strconv.Atoi(*output.PartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse part number marker: %w", err)
+	}
+
+	npmn, err := strconv.Atoi(*output.NextPartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse next part number marker: %w", err)
+	}
+
+	return s3response.ListPartsResult{
+		Bucket:   *output.Bucket,
+		Key:      *output.Key,
+		UploadID: *output.UploadId,
+		Initiator: s3response.Initiator{
+			ID:          *output.Initiator.ID,
+			DisplayName: *output.Initiator.DisplayName,
+		},
+		Owner: s3response.Owner{
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
+		},
+		StorageClass:         string(output.StorageClass),
+		PartNumberMarker:     pnm,
+		NextPartNumberMarker: npmn,
+		MaxParts:             int(*output.MaxParts),
+		IsTruncated:          *output.IsTruncated,
+		Parts:                parts,
+	}, nil
+}
+
+func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := client.UploadPart(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
+	err = handleError(err)
+	if err != nil {
+		return "", err
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.CopyObjectResult{}, err
+	}
+
+	output, err := client.UploadPartCopy(ctx, input)
+	err = handleError(err)
+	if err != nil {
+		return s3response.CopyObjectResult{}, err
+	}
+
+	return s3response.CopyObjectResult{
+		LastModified: *output.CopyPartResult.LastModified,
+		ETag:         *output.CopyPartResult.ETag,
+	}, nil
+}
+
+func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := client.PutObject(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
+	err = handleError(err)
+	if err != nil {
+		return "", err
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.HeadObject(ctx, input)
+	err = handleError(err)
+
+	return out, err
+}
+
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetObject(ctx, input)
+	err = handleError(err)
+	if err != nil {
+		return nil, err
+	}
+	defer output.Body.Close()
+
+	_, err = io.Copy(w, output.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}
+
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.GetObjectAttributes(ctx, input)
+	err = handleError(err)
+
+	return out, err
+}
+
+func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.CopyObject(ctx, input)
+	err = handleError(err)
+
+	return out, err
+}
+
+func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.ListObjects(ctx, input)
+	err = handleError(err)
+
+	return out, err
+}
+
+func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := client.ListObjectsV2(ctx, input)
+	err = handleError(err)
+
+	return out, err
+}
+
+func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteObject(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, err
+	}
+
+	if len(input.Delete.Objects) == 0 {
+		input.Delete.Objects = []types.ObjectIdentifier{}
+	}
+
+	output, err := client.DeleteObjects(ctx, input)
+	err = handleError(err)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, err
+	}
+
+	return s3response.DeleteObjectsResult{
+		Deleted: output.Deleted,
+		Error:   output.Errors,
+	}, nil
+}
+
+func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetBucketAcl(ctx, input)
+	err = handleError(err)
+	if err != nil {
+		return nil, err
+	}
+
+	var acl auth.ACL
+
+	acl.Owner = *output.Owner.ID
+	for _, el := range output.Grants {
+		acl.Grantees = append(acl.Grantees, auth.Grantee{
+			Permission: el.Permission,
+			Access:     *el.Grantee.ID,
+		})
+	}
+
+	return json.Marshal(acl)
+}
+
+func (s S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	acl, err := auth.ParseACL(data)
+	if err != nil {
+		return err
+	}
+
+	input := &s3.PutBucketAclInput{
+		Bucket: &bucket,
+		ACL:    acl.ACL,
+		AccessControlPolicy: &types.AccessControlPolicy{
+			Owner: &types.Owner{
+				ID: &acl.Owner,
+			},
+		},
+	}
+
+	for _, el := range acl.Grantees {
+		acc := el.Access
+		input.AccessControlPolicy.Grants = append(input.AccessControlPolicy.Grants, types.Grant{
+			Permission: el.Permission,
+			Grantee: &types.Grantee{
+				ID:   &acc,
+				Type: types.TypeCanonicalUser,
+			},
+		})
+	}
+
+	_, err = client.PutBucketAcl(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	tagging := &types.Tagging{
+		TagSet: []types.Tag{},
+	}
+	for key, val := range tags {
+		tagging.TagSet = append(tagging.TagSet, types.Tag{
+			Key:   &key,
+			Value: &val,
+		})
+	}
+
+	_, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
+		Bucket:  &bucket,
+		Key:     &object,
+		Tagging: tagging,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	err = handleError(err)
+	if err != nil {
+		return nil, err
+	}
+
+	tags := make(map[string]string)
+	for _, el := range output.TagSet {
+		tags[*el.Key] = *el.Value
+	}
+
+	return tags, nil
+}
+
+func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	if resp.StatusCode > 300 {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		return fmt.Errorf(string(body))
+	}
+
+	return nil
+}
+
+func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return []s3response.Bucket{}, err
+	}
+	defer resp.Body.Close()
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return []s3response.Bucket{}, err
+	}
+
+	return buckets, nil
+}
+
+func handleError(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		apiErr := s3err.APIError{
+			Code:        ae.ErrorCode(),
+			Description: ae.ErrorMessage(),
+		}
+		var re *awshttp.ResponseError
+		if errors.As(err, &re) {
+			apiErr.HTTPStatusCode = re.Response.StatusCode
+		}
+		return apiErr
+	}
+	return err
+}

backend/scoutfs/scoutfs.go

@@ -0,0 +1,813 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package scoutfs
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/s3err"
+)
+
+type ScoutFS struct {
+	*posix.Posix
+	rootfd  *os.File
+	rootdir string
+
+	// glaciermode enables the following behavior:
+	// GET object:  if file offline, return invalid object state
+	// HEAD object: if file offline, set obj storage class to GLACIER
+	//              if file offline and staging, x-amz-restore: ongoing-request="true"
+	//              if file offline and not staging, x-amz-restore: ongoing-request="false"
+	//              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
+	//              note: this expiry-date is not used but provided for client glacier compatibility
+	// ListObjects: if file offline, set obj storage class to GLACIER
+	// RestoreObject: add batch stage request to file
+	glaciermode bool
+}
+
+var _ backend.Backend = &ScoutFS{}
+
+const (
+	metaTmpDir          = ".sgwtmp"
+	metaTmpMultipartDir = metaTmpDir + "/multipart"
+	tagHdr              = "X-Amz-Tagging"
+	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
+	etagkey             = "user.etag"
+)
+
+var (
+	stageComplete      = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\""
+	stageInProgress    = "true"
+	stageNotInProgress = "false"
+)
+
+const (
+	// ScoutFS special xattr types
+
+	systemPrefix = "scoutfs.hide."
+	onameAttr    = systemPrefix + "objname"
+	flagskey     = systemPrefix + "sam_flags"
+	stagecopykey = systemPrefix + "sam_stagereq"
+)
+
+const (
+	// ScoutAM Flags
+
+	// Staging - file requested stage
+	Staging uint64 = 1 << iota
+	// StageFail - all copies failed to stage
+	StageFail
+	// NoArchive - no archive copies of file should be made
+	NoArchive
+	// ExtCacheRequested means file policy requests Ext Cache
+	ExtCacheRequested
+	// ExtCacheDone means this file ext cache copy has been
+	// created already (and possibly pruned, so may not exist)
+	ExtCacheDone
+)
+
+// Option sets various options for scoutfs
+type Option func(s *ScoutFS)
+
+// WithGlacierEmulation sets glacier mode emulation
+func WithGlacierEmulation() Option {
+	return func(s *ScoutFS) { s.glaciermode = true }
+}
+
+func (s *ScoutFS) Shutdown() {
+	s.Posix.Shutdown()
+	s.rootfd.Close()
+	_ = s.rootdir
+}
+
+func (*ScoutFS) String() string {
+	return "ScoutFS Gateway"
+}
+
+// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
+// ioctl to not have to read and copy the part data to the final object. This
+// saves a read and write cycle for all mutlipart uploads.
+func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+	uploadID := *input.UploadId
+	parts := input.MultipartUpload.Parts
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	sum, err := s.checkUploadIDExists(bucket, object, uploadID)
+	if err != nil {
+		return nil, err
+	}
+
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	// check all parts ok
+	last := len(parts) - 1
+	partsize := int64(0)
+	var totalsize int64
+	for i, p := range parts {
+		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
+		fi, err := os.Lstat(partPath)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		if i == 0 {
+			partsize = fi.Size()
+		}
+		totalsize += fi.Size()
+		// all parts except the last need to be the same size
+		if i < last && partsize != fi.Size() {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		// non-last part sizes need to be multiples of 4k for move blocks
+		// TODO: fallback to no move blocks if not 4k aligned?
+		if i == 0 && i < last && fi.Size()%4096 != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+
+		b, err := xattr.Get(partPath, "user.etag")
+		etag := string(b)
+		if err != nil {
+			etag = ""
+		}
+		if etag != *parts[i].ETag {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+	}
+
+	// use totalsize=0 because we wont be writing to the file, only moving
+	// extents around.  so we dont want to fallocate this.
+	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
+	if err != nil {
+		return nil, fmt.Errorf("open temp file: %w", err)
+	}
+	defer f.cleanup()
+
+	for _, p := range parts {
+		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
+		if err != nil {
+			return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
+		}
+
+		// scoutfs move data is a metadata only operation that moves the data
+		// extent references from the source, appeding to the destination.
+		// this needs to be 4k aligned.
+		err = moveData(pf, f.f)
+		pf.Close()
+		if err != nil {
+			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
+		}
+	}
+
+	userMetaData := make(map[string]string)
+	upiddir := filepath.Join(objdir, uploadID)
+	loadUserMetaData(upiddir, userMetaData)
+
+	objname := filepath.Join(bucket, object)
+	dir := filepath.Dir(objname)
+	if dir != "" {
+		if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
+			if err != nil {
+				return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			}
+		}
+	}
+	err = f.link()
+	if err != nil {
+		return nil, fmt.Errorf("link object in namespace: %w", err)
+	}
+
+	for k, v := range userMetaData {
+		err = xattr.Set(objname, "user."+k, []byte(v))
+		if err != nil {
+			// cleanup object if returning error
+			os.Remove(objname)
+			return nil, fmt.Errorf("set user attr %q: %w", k, err)
+		}
+	}
+
+	// Calculate s3 compatible md5sum for complete multipart.
+	s3MD5 := backend.GetMultipartMD5(parts)
+
+	err = xattr.Set(objname, "user.etag", []byte(s3MD5))
+	if err != nil {
+		// cleanup object if returning error
+		os.Remove(objname)
+		return nil, fmt.Errorf("set etag attr: %w", err)
+	}
+
+	// cleanup tmp dirs
+	os.RemoveAll(upiddir)
+	// use Remove for objdir in case there are still other uploads
+	// for same object name outstanding
+	os.Remove(objdir)
+
+	return &s3.CompleteMultipartUploadOutput{
+		Bucket: &bucket,
+		ETag:   &s3MD5,
+		Key:    &object,
+	}, nil
+}
+
+func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) {
+	sum := sha256.Sum256([]byte(object))
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+
+	_, err := os.Stat(filepath.Join(objdir, uploadID))
+	if errors.Is(err, fs.ErrNotExist) {
+		return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
+	}
+	if err != nil {
+		return [32]byte{}, fmt.Errorf("stat upload: %w", err)
+	}
+	return sum, nil
+}
+
+func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
+	ents, err := xattr.List(path)
+	if err != nil || len(ents) == 0 {
+		return
+	}
+	for _, e := range ents {
+		if !isValidMeta(e) {
+			continue
+		}
+		b, err := xattr.Get(path, e)
+		if err == syscall.ENODATA {
+			m[strings.TrimPrefix(e, "user.")] = ""
+			continue
+		}
+		if err != nil {
+			continue
+		}
+		m[strings.TrimPrefix(e, "user.")] = string(b)
+	}
+
+	b, err := xattr.Get(path, "user.content-type")
+	contentType = string(b)
+	if err != nil {
+		contentType = ""
+	}
+	if contentType != "" {
+		m["content-type"] = contentType
+	}
+
+	b, err = xattr.Get(path, "user.content-encoding")
+	contentEncoding = string(b)
+	if err != nil {
+		contentEncoding = ""
+	}
+	if contentEncoding != "" {
+		m["content-encoding"] = contentEncoding
+	}
+
+	return
+}
+
+func isValidMeta(val string) bool {
+	if strings.HasPrefix(val, "user.X-Amz-Meta") {
+		return true
+	}
+	if strings.EqualFold(val, "user.Expires") {
+		return true
+	}
+	return false
+}
+
+// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
+// when appropriate
+func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = mkdirAll(path[:j-1], perm, bucket, object)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, perm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+	return nil
+}
+
+func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	objPath := filepath.Join(bucket, object)
+	fi, err := os.Stat(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat object: %w", err)
+	}
+
+	userMetaData := make(map[string]string)
+	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+
+	b, err := xattr.Get(objPath, etagkey)
+	etag := string(b)
+	if err != nil {
+		etag = ""
+	}
+
+	stclass := types.StorageClassStandard
+	requestOngoing := ""
+	if s.glaciermode {
+		requestOngoing = stageComplete
+
+		// Check if there are any offline exents associated with this file.
+		// If so, we will set storage class to glacier.
+		st, err := statMore(objPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat more: %w", err)
+		}
+		if st.Offline_blocks != 0 {
+			stclass = types.StorageClassGlacier
+			requestOngoing = stageNotInProgress
+
+			ok, err := isStaging(objPath)
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+			}
+			if err != nil {
+				return nil, fmt.Errorf("check stage status: %w", err)
+			}
+			if ok {
+				requestOngoing = stageInProgress
+			}
+		}
+	}
+
+	contentLength := fi.Size()
+
+	return &s3.HeadObjectOutput{
+		ContentLength:   &contentLength,
+		ContentType:     &contentType,
+		ContentEncoding: &contentEncoding,
+		ETag:            &etag,
+		LastModified:    backend.GetTimePtr(fi.ModTime()),
+		Metadata:        userMetaData,
+		StorageClass:    stclass,
+		Restore:         &requestOngoing,
+	}, nil
+}
+
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+	acceptRange := *input.Range
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	objPath := filepath.Join(bucket, object)
+	fi, err := os.Stat(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat object: %w", err)
+	}
+
+	startOffset, length, err := backend.ParseRange(fi, acceptRange)
+	if err != nil {
+		return nil, err
+	}
+
+	if length == -1 {
+		length = fi.Size() - startOffset + 1
+	}
+
+	if startOffset+length > fi.Size() {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
+	if s.glaciermode {
+		// Check if there are any offline exents associated with this file.
+		// If so, we will return the InvalidObjectState error.
+		st, err := statMore(objPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat more: %w", err)
+		}
+		if st.Offline_blocks != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidObjectState)
+		}
+	}
+
+	f, err := os.Open(objPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("open object: %w", err)
+	}
+	defer f.Close()
+
+	rdr := io.NewSectionReader(f, startOffset, length)
+	_, err = io.Copy(writer, rdr)
+	if err != nil {
+		return nil, fmt.Errorf("copy data: %w", err)
+	}
+
+	userMetaData := make(map[string]string)
+
+	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+
+	b, err := xattr.Get(objPath, etagkey)
+	etag := string(b)
+	if err != nil {
+		etag = ""
+	}
+
+	tags, err := s.getXattrTags(bucket, object)
+	if err != nil {
+		return nil, fmt.Errorf("get object tags: %w", err)
+	}
+
+	tagCount := int32(len(tags))
+
+	return &s3.GetObjectOutput{
+		AcceptRanges:    &acceptRange,
+		ContentLength:   &length,
+		ContentEncoding: &contentEncoding,
+		ContentType:     &contentType,
+		ETag:            &etag,
+		LastModified:    backend.GetTimePtr(fi.ModTime()),
+		Metadata:        userMetaData,
+		TagCount:        &tagCount,
+		StorageClass:    types.StorageClassStandard,
+	}, nil
+}
+
+func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) {
+	tags := make(map[string]string)
+	b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if isNoAttr(err) {
+		return tags, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get tags: %w", err)
+	}
+
+	err = json.Unmarshal(b, &tags)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal tags: %w", err)
+	}
+
+	return tags, nil
+}
+
+func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	bucket := *input.Bucket
+	prefix := ""
+	if input.Prefix != nil {
+		prefix = *input.Prefix
+	}
+	marker := ""
+	if input.Marker != nil {
+		marker = *input.Marker
+	}
+	delim := ""
+	if input.Delimiter != nil {
+		delim = *input.Delimiter
+	}
+	maxkeys := int32(0)
+	if input.MaxKeys != nil {
+		maxkeys = *input.MaxKeys
+	}
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	fileSystem := os.DirFS(bucket)
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
+		s.fileToObj(bucket), []string{metaTmpDir})
+	if err != nil {
+		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+	}
+
+	return &s3.ListObjectsOutput{
+		CommonPrefixes: results.CommonPrefixes,
+		Contents:       results.Objects,
+		Delimiter:      &delim,
+		IsTruncated:    &results.Truncated,
+		Marker:         &marker,
+		MaxKeys:        &maxkeys,
+		Name:           &bucket,
+		NextMarker:     &results.NextMarker,
+		Prefix:         &prefix,
+	}, nil
+}
+
+func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	bucket := *input.Bucket
+	prefix := ""
+	if input.Prefix != nil {
+		prefix = *input.Prefix
+	}
+	marker := ""
+	if input.ContinuationToken != nil {
+		marker = *input.ContinuationToken
+	}
+	delim := ""
+	if input.Delimiter != nil {
+		delim = *input.Delimiter
+	}
+	maxkeys := int32(0)
+	if input.MaxKeys != nil {
+		maxkeys = *input.MaxKeys
+	}
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	fileSystem := os.DirFS(bucket)
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
+		s.fileToObj(bucket), []string{metaTmpDir})
+	if err != nil {
+		return nil, fmt.Errorf("walk %v: %w", bucket, err)
+	}
+
+	return &s3.ListObjectsV2Output{
+		CommonPrefixes:        results.CommonPrefixes,
+		Contents:              results.Objects,
+		Delimiter:             &delim,
+		IsTruncated:           &results.Truncated,
+		ContinuationToken:     &marker,
+		MaxKeys:               &maxkeys,
+		Name:                  &bucket,
+		NextContinuationToken: &results.NextMarker,
+		Prefix:                &prefix,
+	}, nil
+}
+
+func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
+	return func(path string, d fs.DirEntry) (types.Object, error) {
+		objPath := filepath.Join(bucket, path)
+		if d.IsDir() {
+			// directory object only happens if directory empty
+			// check to see if this is a directory object by checking etag
+			etagBytes, err := xattr.Get(objPath, etagkey)
+			if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("get etag: %w", err)
+			}
+			etag := string(etagBytes)
+
+			fi, err := d.Info()
+			if errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			}
+
+			key := path + "/"
+
+			return types.Object{
+				ETag:         &etag,
+				Key:          &key,
+				LastModified: backend.GetTimePtr(fi.ModTime()),
+			}, nil
+		}
+
+		// file object, get object info and fill out object data
+		etagBytes, err := xattr.Get(objPath, etagkey)
+		if errors.Is(err, fs.ErrNotExist) {
+			return types.Object{}, backend.ErrSkipObj
+		}
+		if err != nil && !isNoAttr(err) {
+			return types.Object{}, fmt.Errorf("get etag: %w", err)
+		}
+		etag := string(etagBytes)
+
+		fi, err := d.Info()
+		if errors.Is(err, fs.ErrNotExist) {
+			return types.Object{}, backend.ErrSkipObj
+		}
+		if err != nil {
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		}
+
+		sc := types.ObjectStorageClassStandard
+		if s.glaciermode {
+			// Check if there are any offline exents associated with this file.
+			// If so, we will return the InvalidObjectState error.
+			st, err := statMore(objPath)
+			if errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("stat more: %w", err)
+			}
+			if st.Offline_blocks != 0 {
+				sc = types.ObjectStorageClassGlacier
+			}
+		}
+
+		size := fi.Size()
+
+		return types.Object{
+			ETag:         &etag,
+			Key:          &path,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
+			Size:         &size,
+			StorageClass: sc,
+		}, nil
+	}
+}
+
+// RestoreObject will set stage request on file if offline and do nothing if
+// file is online
+func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
+	bucket := *input.Bucket
+	object := *input.Key
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	err = setStaging(filepath.Join(bucket, object))
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if err != nil {
+		return fmt.Errorf("stage object: %w", err)
+	}
+
+	return nil
+}
+
+func setStaging(objname string) error {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return err
+	}
+
+	var oldflags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &oldflags)
+		if err != nil {
+			return err
+		}
+	}
+
+	newflags := oldflags | Staging
+
+	if newflags == oldflags {
+		// no flags change, just return
+		return nil
+	}
+
+	return fSetNewGlobalFlags(objname, newflags)
+}
+
+func isStaging(objname string) (bool, error) {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return false, err
+	}
+
+	var flags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &flags)
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return flags&Staging == Staging, nil
+}
+
+func fSetNewGlobalFlags(objname string, flags uint64) error {
+	b, err := json.Marshal(&flags)
+	if err != nil {
+		return err
+	}
+
+	return xattr.Set(objname, flagskey, b)
+}
+
+func isNoAttr(err error) bool {
+	if err == nil {
+		return false
+	}
+	xerr, ok := err.(*xattr.Error)
+	if ok && xerr.Err == xattr.ENOATTR {
+		return true
+	}
+	if err == syscall.ENODATA {
+		return true
+	}
+	return false
+}

backend/scoutfs/scoutfs_compat.go

@@ -0,0 +1,209 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build linux && amd64
+
+package scoutfs
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/versity/scoutfs-go"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+	p, err := posix.New(rootdir)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := os.Open(rootdir)
+	if err != nil {
+		return nil, fmt.Errorf("open %v: %w", rootdir, err)
+	}
+
+	s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
+	for _, opt := range opts {
+		opt(s)
+	}
+
+	return s, nil
+}
+
+const procfddir = "/proc/self/fd"
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
+	// This can help reduce contention within the namespace (parent directories),
+	// etc. And will auto cleanup the inode on close if we never link this
+	// file descriptor into the namespace.
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	if err != nil {
+		// O_TMPFILE not supported, try fallback
+		err := os.MkdirAll(dir, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("make temp dir: %w", err)
+		}
+		f, err := os.CreateTemp(dir,
+			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+		if err != nil {
+			return nil, err
+		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		// falloc is best effort, its fine if this fails
+		if size > 0 {
+			tmp.falloc()
+		}
+		return tmp, nil
+	}
+
+	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
+	// later to link file into namespace
+	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
+
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	// falloc is best effort, its fine if this fails
+	if size > 0 {
+		tmp.falloc()
+	}
+	return tmp, nil
+}
+
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
+func (tmp *tmpfile) link() error {
+	// We use Linkat/Rename as the atomic operation for object puts. The
+	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
+	// with any other simultaneous uploads. The final operation is to move the
+	// temp file into place for the object. This ensures the object semantics
+	// of last upload completed wins and is not some combination of writes
+	// from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
+	}
+
+	procdir, err := os.Open(procfddir)
+	if err != nil {
+		return fmt.Errorf("open proc dir: %w", err)
+	}
+	defer procdir.Close()
+
+	dir, err := os.Open(filepath.Dir(objPath))
+	if err != nil {
+		return fmt.Errorf("open parent dir: %w", err)
+	}
+	defer dir.Close()
+
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if err != nil {
+		return fmt.Errorf("link tmpfile: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}
+
+func moveData(from *os.File, to *os.File) error {
+	return scoutfs.MoveData(from, to)
+}
+
+func statMore(path string) (stat, error) {
+	st, err := scoutfs.StatMore(path)
+	if err != nil {
+		return stat{}, err
+	}
+	var s stat
+
+	s.Meta_seq = st.Meta_seq
+	s.Data_seq = st.Data_seq
+	s.Data_version = st.Data_version
+	s.Online_blocks = st.Online_blocks
+	s.Offline_blocks = st.Offline_blocks
+	s.Crtime_sec = st.Crtime_sec
+	s.Crtime_nsec = st.Crtime_nsec
+
+	return s, nil
+}

backend/scoutfs/scoutfs_incompat.go

@@ -0,0 +1,58 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !(linux && amd64)
+
+package scoutfs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+)
+
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+	return nil, fmt.Errorf("scoutfs only available on linux")
+}
+
+type tmpfile struct {
+	f *os.File
+}
+
+var (
+	errNotSupported = errors.New("not supported")
+)
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	return nil, errNotSupported
+}
+
+func (tmp *tmpfile) link() error {
+	return errNotSupported
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	return 0, errNotSupported
+}
+
+func (tmp *tmpfile) cleanup() {
+}
+
+func moveData(from *os.File, to *os.File) error {
+	return errNotSupported
+}
+
+func statMore(path string) (stat, error) {
+	return stat{}, errNotSupported
+}

backend/scoutfs/stat.go

@@ -0,0 +1,25 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package scoutfs
+
+type stat struct {
+	Meta_seq       uint64
+	Data_seq       uint64
+	Data_version   uint64
+	Online_blocks  uint64
+	Offline_blocks uint64
+	Crtime_sec     uint64
+	Crtime_nsec    uint32
+}

backend/walk.go

@@ -0,0 +1,222 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+type WalkResults struct {
+	CommonPrefixes []types.CommonPrefix
+	Objects        []types.Object
+	Truncated      bool
+	NextMarker     string
+}
+
+type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
+
+var ErrSkipObj = errors.New("skip this object")
+
+// Walk walks the supplied fs.FS and returns results compatible with list
+// objects responses
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+	cpmap := make(map[string]struct{})
+	var objects []types.Object
+
+	var pastMarker bool
+	if marker == "" {
+		pastMarker = true
+	}
+
+	pastMax := max == 0
+	var newMarker string
+	var truncated bool
+
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
+
+		if pastMax {
+			if len(objects) != 0 {
+				newMarker = *objects[len(objects)-1].Key
+				truncated = true
+			}
+			return fs.SkipAll
+		}
+
+		if d.IsDir() {
+			// If prefix is defined and the directory does not match prefix,
+			// do not descend into the directory because nothing will
+			// match this prefix. Make sure to append the / at the end of
+			// directories since this is implied as a directory path name.
+			// If path is a prefix of prefix, then path could still be
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
+			if prefix != "" &&
+				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
+				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
+				return fs.SkipDir
+			}
+
+			// TODO: can we do better here rather than a second readdir
+			// per directory?
+			ents, err := fs.ReadDir(fileSystem, path)
+			if err != nil {
+				return fmt.Errorf("readdir %q: %w", path, err)
+			}
+			if len(ents) == 0 {
+				dirobj, err := getObj(path, d)
+				if err == ErrSkipObj {
+					return nil
+				}
+				if err != nil {
+					return fmt.Errorf("directory to object %q: %w", path, err)
+				}
+				objects = append(objects, dirobj)
+			}
+
+			return nil
+		}
+
+		if !pastMarker {
+			if path == marker {
+				pastMarker = true
+				return nil
+			}
+			if path < marker {
+				return nil
+			}
+		}
+
+		// If object doesn't have prefix, don't include in results.
+		if prefix != "" && !strings.HasPrefix(path, prefix) {
+			return nil
+		}
+
+		if delimiter == "" {
+			// If no delimiter specified, then all files with matching
+			// prefix are included in results
+			obj, err := getObj(path, d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, obj)
+
+			if max > 0 && (len(objects)+len(cpmap)) == int(max) {
+				pastMax = true
+			}
+
+			return nil
+		}
+
+		// Since delimiter is specified, we only want results that
+		// do not contain the delimiter beyond the prefix.  If the
+		// delimiter exists past the prefix, then the substring
+		// between the prefix and delimiter is part of common prefixes.
+		//
+		// For example:
+		// prefix = A/
+		// delimiter = /
+		// and objects:
+		// A/file
+		// A/B/file
+		// B/C
+		// would return:
+		// objects: A/file
+		// common prefix: A/B/
+		//
+		// Note: No objects are included past the common prefix since
+		// these are all rolled up into the common prefix.
+		// Note: The delimiter can be anything, so we have to operate on
+		// the full path without any assumptions on posix directory hierarchy
+		// here.  Usually the delimiter will be "/", but thats not required.
+		suffix := strings.TrimPrefix(path, prefix)
+		before, _, found := strings.Cut(suffix, delimiter)
+		if !found {
+			obj, err := getObj(path, d)
+			if err == ErrSkipObj {
+				return nil
+			}
+			if err != nil {
+				return fmt.Errorf("file to object %q: %w", path, err)
+			}
+			objects = append(objects, obj)
+			if (len(objects) + len(cpmap)) == int(max) {
+				pastMax = true
+			}
+			return nil
+		}
+
+		// Common prefixes are a set, so should not have duplicates.
+		// These are abstractly a "directory", so need to include the
+		// delimiter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
+		if (len(objects) + len(cpmap)) == int(max) {
+			pastMax = true
+		}
+
+		return nil
+	})
+	if err != nil {
+		return WalkResults{}, err
+	}
+
+	var commonPrefixStrings []string
+	for k := range cpmap {
+		commonPrefixStrings = append(commonPrefixStrings, k)
+	}
+	sort.Strings(commonPrefixStrings)
+	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
+	for _, cp := range commonPrefixStrings {
+		pfx := cp
+		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
+			Prefix: &pfx,
+		})
+	}
+
+	return WalkResults{
+		CommonPrefixes: commonPrefixes,
+		Objects:        objects,
+		Truncated:      truncated,
+		NextMarker:     newMarker,
+	}, nil
+}
+
+func contains(a string, strs []string) bool {
+	for _, s := range strs {
+		if s == a {
+			return true
+		}
+	}
+	return false
+}

backend/walk_test.go

@@ -0,0 +1,206 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend_test
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/fs"
+	"testing"
+	"testing/fstest"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
+)
+
+type walkTest struct {
+	fsys     fs.FS
+	expected backend.WalkResults
+	getobj   backend.GetObjFunc
+}
+
+func getObj(path string, d fs.DirEntry) (types.Object, error) {
+	if d.IsDir() {
+		etag := getMD5(path)
+
+		fi, err := d.Info()
+		if err != nil {
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		}
+
+		return types.Object{
+			ETag:         &etag,
+			Key:          &path,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
+		}, nil
+	}
+
+	etag := getMD5(path)
+
+	fi, err := d.Info()
+	if err != nil {
+		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+	}
+
+	size := fi.Size()
+
+	return types.Object{
+		ETag:         &etag,
+		Key:          &path,
+		LastModified: backend.GetTimePtr(fi.ModTime()),
+		Size:         &size,
+	}, nil
+}
+
+func getMD5(text string) string {
+	hash := md5.Sum([]byte(text))
+	return hex.EncodeToString(hash[:])
+}
+
+func TestWalk(t *testing.T) {
+	tests := []walkTest{
+		{
+			// test case from
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
+			fsys: fstest.MapFS{
+				"sample.jpg":                       {},
+				"photos/2006/January/sample.jpg":   {},
+				"photos/2006/February/sample2.jpg": {},
+				"photos/2006/February/sample3.jpg": {},
+				"photos/2006/February/sample4.jpg": {},
+			},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("photos/"),
+				}},
+				Objects: []types.Object{{
+					Key: backend.GetStringPtr("sample.jpg"),
+				}},
+			},
+			getobj: getObj,
+		},
+		{
+			// test case single dir/single file
+			fsys: fstest.MapFS{
+				"test/file": {},
+			},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("test/"),
+				}},
+				Objects: []types.Object{},
+			},
+			getobj: getObj,
+		},
+	}
+
+	for _, tt := range tests {
+		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
+		if err != nil {
+			t.Fatalf("walk: %v", err)
+		}
+
+		compareResults(res, tt.expected, t)
+	}
+}
+
+func compareResults(got, wanted backend.WalkResults, t *testing.T) {
+	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
+		t.Errorf("unexpected common prefix, got %v wanted %v",
+			printCommonPrefixes(got.CommonPrefixes),
+			printCommonPrefixes(wanted.CommonPrefixes))
+	}
+
+	if !compareObjects(got.Objects, wanted.Objects) {
+		t.Errorf("unexpected object, got %v wanted %v",
+			printObjects(got.Objects),
+			printObjects(wanted.Objects))
+	}
+}
+
+func compareCommonPrefix(a, b []types.CommonPrefix) bool {
+	if len(a) == 0 && len(b) == 0 {
+		return true
+	}
+	if len(a) != len(b) {
+		return false
+	}
+
+	for _, cp := range a {
+		if containsCommonPrefix(cp, b) {
+			return true
+		}
+	}
+	return false
+}
+
+func containsCommonPrefix(c types.CommonPrefix, list []types.CommonPrefix) bool {
+	for _, cp := range list {
+		if *c.Prefix == *cp.Prefix {
+			return true
+		}
+	}
+	return false
+}
+
+func printCommonPrefixes(list []types.CommonPrefix) string {
+	res := "["
+	for _, cp := range list {
+		if res == "[" {
+			res = res + *cp.Prefix
+		} else {
+			res = res + ", " + *cp.Prefix
+		}
+	}
+	return res + "]"
+}
+
+func compareObjects(a, b []types.Object) bool {
+	if len(a) == 0 && len(b) == 0 {
+		return true
+	}
+	if len(a) != len(b) {
+		return false
+	}
+
+	for _, cp := range a {
+		if containsObject(cp, b) {
+			return true
+		}
+	}
+	return false
+}
+
+func containsObject(c types.Object, list []types.Object) bool {
+	for _, cp := range list {
+		if *c.Key == *cp.Key {
+			return true
+		}
+	}
+	return false
+}
+
+func printObjects(list []types.Object) string {
+	res := "["
+	for _, cp := range list {
+		if res == "[" {
+			res = res + *cp.Key
+		} else {
+			res = res + ", " + *cp.Key
+		}
+	}
+	return res + "]"
+}

cmd/versitygw/admin.go

@@ -0,0 +1,416 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	adminAccess   string
+	adminSecret   string
+	adminEndpoint string
+)
+
+func adminCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "admin",
+		Usage:       "admin CLI tool",
+		Description: `Admin CLI tool for interacting with admin APIs.`,
+		Subcommands: []*cli.Command{
+			{
+				Name:   "create-user",
+				Usage:  "Create a new user",
+				Action: createUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "access key id for the new user",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:     "secret",
+						Usage:    "secret access key for the new user",
+						Required: true,
+						Aliases:  []string{"s"},
+					},
+					&cli.StringFlag{
+						Name:     "role",
+						Usage:    "role for the new user",
+						Required: true,
+						Aliases:  []string{"r"},
+					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+					&cli.IntFlag{
+						Name:    "project-id",
+						Usage:   "projectID for the new user",
+						Aliases: []string{"pi"},
+					},
+				},
+			},
+			{
+				Name:   "delete-user",
+				Usage:  "Delete a user",
+				Action: deleteUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "access key id of the user to be deleted",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+				},
+			},
+			{
+				Name:   "list-users",
+				Usage:  "List all the gateway users",
+				Action: listUsers,
+			},
+			{
+				Name:  "change-bucket-owner",
+				Usage: "Changes the bucket owner",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "bucket",
+						Usage:    "the bucket name to change the owner",
+						Required: true,
+						Aliases:  []string{"b"},
+					},
+					&cli.StringFlag{
+						Name:     "owner",
+						Usage:    "the user access key id, who should be the bucket owner",
+						Required: true,
+						Aliases:  []string{"o"},
+					},
+				},
+				Action: changeBucketOwner,
+			},
+			{
+				Name:   "list-buckets",
+				Usage:  "Lists all the gateway buckets and owners.",
+				Action: listBuckets,
+			},
+		},
+		Flags: []cli.Flag{
+			// TODO: create a configuration file for this
+			&cli.StringFlag{
+				Name:        "access",
+				Usage:       "admin access key id",
+				EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
+				Aliases:     []string{"a"},
+				Required:    true,
+				Destination: &adminAccess,
+			},
+			&cli.StringFlag{
+				Name:        "secret",
+				Usage:       "admin secret access key",
+				EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
+				Aliases:     []string{"s"},
+				Required:    true,
+				Destination: &adminSecret,
+			},
+			&cli.StringFlag{
+				Name:        "endpoint-url",
+				Usage:       "admin apis endpoint url",
+				EnvVars:     []string{"ADMIN_ENDPOINT_URL"},
+				Aliases:     []string{"er"},
+				Required:    true,
+				Destination: &adminEndpoint,
+			},
+		},
+	}
+}
+
+func createUser(ctx *cli.Context) error {
+	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
+	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
+	if access == "" || secret == "" {
+		return fmt.Errorf("invalid input parameters for the new user")
+	}
+	if role != "admin" && role != "user" {
+		return fmt.Errorf("invalid input parameter for role")
+	}
+
+	acc := auth.Account{
+		Access:    access,
+		Secret:    secret,
+		Role:      role,
+		UserID:    userID,
+		GroupID:   groupID,
+		ProjectID: projectID,
+	}
+
+	accJson, err := json.Marshal(acc)
+	if err != nil {
+		return fmt.Errorf("failed to parse user data: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(accJson)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
+func deleteUser(ctx *cli.Context) error {
+	access := ctx.String("access")
+	if access == "" {
+		return fmt.Errorf("invalid input parameter for the new user")
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
+func listUsers(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
+		return err
+	}
+	fmt.Println(accs)
+
+	printAcctTable(accs)
+
+	return nil
+}
+
+const (
+	// account table formatting
+	minwidth int  = 2   // minimal cell width including any padding
+	tabwidth int  = 0   // width of tab characters (equivalent number of spaces)
+	padding  int  = 2   // padding added to a cell before computing its width
+	padchar  byte = ' ' // ASCII char used for padding
+	flags    uint = 0   // formatting control flags
+)
+
+func printAcctTable(accs []auth.Account) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
+	for _, acc := range accs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func changeBucketOwner(ctx *cli.Context) error {
+	bucket, owner := ctx.String("bucket"), ctx.String("owner")
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Println(string(body))
+
+	return nil
+}
+
+func printBuckets(buckets []s3response.Bucket) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Bucket\tOwner")
+	fmt.Fprintln(w, "-------\t----")
+	for _, acc := range buckets {
+		fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func listBuckets(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf(string(body))
+	}
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return err
+	}
+
+	printBuckets(buckets)
+
+	return nil
+}

cmd/versitygw/gateway_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/integration"
+)
+
+const (
+	tdir = "tempdir"
+)
+
+var (
+	wg sync.WaitGroup
+)
+
+func initEnv(dir string) {
+	// both
+	debug = true
+	region = "us-east-1"
+
+	// server
+	rootUserAccess = "user"
+	rootUserSecret = "pass"
+	iamDir = dir
+	port = "127.0.0.1:7070"
+
+	// client
+	awsID = "user"
+	awsSecret = "pass"
+	endpoint = "http://127.0.0.1:7070"
+}
+
+func initPosix(ctx context.Context) {
+	path, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("get current directory: %v", err)
+	}
+
+	tempdir := filepath.Join(path, tdir)
+	initEnv(tempdir)
+
+	err = os.RemoveAll(tempdir)
+	if err != nil {
+		log.Fatalf("remove temp directory: %v", err)
+	}
+
+	err = os.Mkdir(tempdir, 0755)
+	if err != nil {
+		log.Fatalf("make temp directory: %v", err)
+	}
+
+	be, err := posix.New(tempdir)
+	if err != nil {
+		log.Fatalf("init posix: %v", err)
+	}
+
+	wg.Add(1)
+	go func() {
+		err = runGateway(ctx, be)
+		if err != nil && err != context.Canceled {
+			log.Fatalf("run gateway: %v", err)
+		}
+
+		err := os.RemoveAll(tempdir)
+		if err != nil {
+			log.Fatalf("remove temp directory: %v", err)
+		}
+		wg.Done()
+	}()
+}
+
+func TestIntegration(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	initPosix(ctx)
+
+	opts := []integration.Option{
+		integration.WithAccess(awsID),
+		integration.WithSecret(awsSecret),
+		integration.WithRegion(region),
+		integration.WithEndpoint(endpoint),
+	}
+	if debug {
+		opts = append(opts, integration.WithDebug())
+	}
+
+	s := integration.NewS3Conf(opts...)
+
+	// replace below with desired test
+	err := integration.HeadBucket_non_existing_bucket(s)
+	if err != nil {
+		t.Error(err)
+	}
+
+	cancel()
+	wg.Wait()
+}

cmd/versitygw/main.go

@@ -0,0 +1,479 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api"
+	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3event"
+	"github.com/versity/versitygw/s3log"
+)
+
+var (
+	port, admPort                          string
+	rootUserAccess                         string
+	rootUserSecret                         string
+	region                                 string
+	admCertFile, admKeyFile                string
+	certFile, keyFile                      string
+	kafkaURL, kafkaTopic, kafkaKey         string
+	natsURL, natsTopic                     string
+	logWebhookURL                          string
+	accessLog                              string
+	debug                                  bool
+	iamDir                                 string
+	ldapURL, ldapBindDN, ldapPassword      string
+	ldapQueryBase, ldapObjClasses          string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	s3IamAccess, s3IamSecret               string
+	s3IamRegion, s3IamBucket               string
+	s3IamEndpoint                          string
+	s3IamSslNoVerify, s3IamDebug           bool
+	iamCacheDisable                        bool
+	iamCacheTTL                            int
+	iamCachePrune                          int
+)
+
+var (
+	// Version is the latest tag (set within Makefile)
+	Version = "git"
+	// Build is the commit hash (set within Makefile)
+	Build = "norev"
+	// BuildTime is the date/time of build (set within Makefile)
+	BuildTime = "none"
+)
+
+func main() {
+	setupSignalHandler()
+
+	app := initApp()
+
+	app.Commands = []*cli.Command{
+		posixCommand(),
+		scoutfsCommand(),
+		s3Command(),
+		adminCommand(),
+		testCommand(),
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		<-sigDone
+		fmt.Fprintf(os.Stderr, "terminating signal caught, shutting down\n")
+		cancel()
+	}()
+
+	if err := app.RunContext(ctx, os.Args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func initApp() *cli.App {
+	return &cli.App{
+		Name:  "versitygw",
+		Usage: "Start S3 gateway service with specified backend storage.",
+		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.`,
+		Action: func(ctx *cli.Context) error {
+			return ctx.App.Command("help").Run(ctx)
+		},
+		Flags: initFlags(),
+	}
+}
+
+func initFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "version",
+			Usage:   "list versitygw version",
+			Aliases: []string{"v"},
+			Action: func(*cli.Context, bool) error {
+				fmt.Println("Version  :", Version)
+				fmt.Println("Build    :", Build)
+				fmt.Println("BuildTime:", BuildTime)
+				os.Exit(0)
+				return nil
+			},
+		},
+		&cli.StringFlag{
+			Name:        "port",
+			Usage:       "gateway listen address <ip>:<port> or :<port>",
+			Value:       ":7070",
+			Destination: &port,
+			Aliases:     []string{"p"},
+		},
+		&cli.StringFlag{
+			Name:        "access",
+			Usage:       "root user access key",
+			EnvVars:     []string{"ROOT_ACCESS_KEY_ID", "ROOT_ACCESS_KEY"},
+			Aliases:     []string{"a"},
+			Destination: &rootUserAccess,
+		},
+		&cli.StringFlag{
+			Name:        "secret",
+			Usage:       "root user secret access key",
+			EnvVars:     []string{"ROOT_SECRET_ACCESS_KEY", "ROOT_SECRET_KEY"},
+			Aliases:     []string{"s"},
+			Destination: &rootUserSecret,
+		},
+		&cli.StringFlag{
+			Name:        "region",
+			Usage:       "s3 region string",
+			Value:       "us-east-1",
+			Destination: &region,
+			Aliases:     []string{"r"},
+		},
+		&cli.StringFlag{
+			Name:        "cert",
+			Usage:       "TLS cert file",
+			Destination: &certFile,
+		},
+		&cli.StringFlag{
+			Name:        "key",
+			Usage:       "TLS key file",
+			Destination: &keyFile,
+		},
+		&cli.StringFlag{
+			Name:        "admin-port",
+			Usage:       "gateway admin server listen address <ip>:<port> or :<port>",
+			Destination: &admPort,
+			Aliases:     []string{"ap"},
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert",
+			Usage:       "TLS cert file for admin server",
+			Destination: &admCertFile,
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert-key",
+			Usage:       "TLS key file for admin server",
+			Destination: &admKeyFile,
+		},
+		&cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "enable debug output",
+			Destination: &debug,
+		},
+		&cli.StringFlag{
+			Name:        "access-log",
+			Usage:       "enable server access logging to specified file",
+			EnvVars:     []string{"LOGFILE"},
+			Destination: &accessLog,
+		},
+		&cli.StringFlag{
+			Name:        "log-webhook-url",
+			Usage:       "webhook url to send the audit logs",
+			EnvVars:     []string{"WEBHOOK"},
+			Destination: &logWebhookURL,
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-url",
+			Usage:       "kafka server url to send the bucket notifications.",
+			Destination: &kafkaURL,
+			Aliases:     []string{"eku"},
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-topic",
+			Usage:       "kafka server pub-sub topic to send the bucket notifications to",
+			Destination: &kafkaTopic,
+			Aliases:     []string{"ekt"},
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-key",
+			Usage:       "kafka server put-sub topic key to send the bucket notifications to",
+			Destination: &kafkaKey,
+			Aliases:     []string{"ekk"},
+		},
+		&cli.StringFlag{
+			Name:        "event-nats-url",
+			Usage:       "nats server url to send the bucket notifications",
+			Destination: &natsURL,
+			Aliases:     []string{"enu"},
+		},
+		&cli.StringFlag{
+			Name:        "event-nats-topic",
+			Usage:       "nats server pub-sub topic to send the bucket notifications to",
+			Destination: &natsTopic,
+			Aliases:     []string{"ent"},
+		},
+		&cli.StringFlag{
+			Name:        "iam-dir",
+			Usage:       "if defined, run internal iam service within this directory",
+			Destination: &iamDir,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-url",
+			Usage:       "ldap server url to store iam data",
+			Destination: &ldapURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-dn",
+			Usage:       "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
+			Destination: &ldapBindDN,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-pass",
+			Usage:       "ldap server user password",
+			Destination: &ldapPassword,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-query-base",
+			Usage:       "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
+			Destination: &ldapQueryBase,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-object-classes",
+			Usage:       "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
+			Destination: &ldapObjClasses,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-access-atr",
+			Usage:       "ldap server user access key id attribute name",
+			Destination: &ldapAccessAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-secret-atr",
+			Usage:       "ldap server user secret access key attribute name",
+			Destination: &ldapSecAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-role-atr",
+			Usage:       "ldap server user role attribute name",
+			Destination: &ldapRoleAtr,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-access",
+			Usage:       "s3 IAM access key",
+			Destination: &s3IamAccess,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-secret",
+			Usage:       "s3 IAM secret key",
+			Destination: &s3IamSecret,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-region",
+			Usage:       "s3 IAM region",
+			Destination: &s3IamRegion,
+			Value:       "us-east-1",
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-bucket",
+			Usage:       "s3 IAM bucket",
+			Destination: &s3IamBucket,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-endpoint",
+			Usage:       "s3 IAM endpoint",
+			Destination: &s3IamEndpoint,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-noverify",
+			Usage:       "s3 IAM disable ssl verification",
+			Destination: &s3IamSslNoVerify,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-debug",
+			Usage:       "s3 IAM debug output",
+			Destination: &s3IamDebug,
+		},
+		&cli.BoolFlag{
+			Name:        "iam-cache-disable",
+			Usage:       "disable local iam cache",
+			Destination: &iamCacheDisable,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-ttl",
+			Usage:       "local iam cache entry ttl (seconds)",
+			Value:       120,
+			Destination: &iamCacheTTL,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-prune",
+			Usage:       "local iam cache cleanup interval (seconds)",
+			Value:       3600,
+			Destination: &iamCachePrune,
+		},
+	}
+}
+
+func runGateway(ctx context.Context, be backend.Backend) error {
+	// int32 max for 32 bit arch
+	blimit := int64(2*1024*1024*1024 - 1)
+	if strconv.IntSize > 32 {
+		// 5GB max for 64 bit arch
+		blimit = int64(5 * 1024 * 1024 * 1024)
+	}
+
+	app := fiber.New(fiber.Config{
+		AppName:           "versitygw",
+		ServerHeader:      "VERSITYGW",
+		BodyLimit:         int(blimit),
+		StreamRequestBody: true,
+	})
+
+	var opts []s3api.Option
+
+	if certFile != "" || keyFile != "" {
+		if certFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if keyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		opts = append(opts, s3api.WithTLS(cert))
+	}
+	if debug {
+		opts = append(opts, s3api.WithDebug())
+	}
+	if admPort == "" {
+		opts = append(opts, s3api.WithAdminServer())
+	}
+
+	admApp := fiber.New(fiber.Config{
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+	})
+
+	var admOpts []s3api.AdminOpt
+
+	if admCertFile != "" || admKeyFile != "" {
+		if admCertFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if admKeyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert))
+	}
+
+	iam, err := auth.New(&auth.Opts{
+		Dir:                iamDir,
+		LDAPServerURL:      ldapURL,
+		LDAPBindDN:         ldapBindDN,
+		LDAPPassword:       ldapPassword,
+		LDAPQueryBase:      ldapQueryBase,
+		LDAPObjClasses:     ldapObjClasses,
+		LDAPAccessAtr:      ldapAccessAtr,
+		LDAPSecretAtr:      ldapSecAtr,
+		LDAPRoleAtr:        ldapRoleAtr,
+		S3Access:           s3IamAccess,
+		S3Secret:           s3IamSecret,
+		S3Region:           s3IamRegion,
+		S3Bucket:           s3IamBucket,
+		S3Endpoint:         s3IamEndpoint,
+		S3DisableSSlVerfiy: s3IamSslNoVerify,
+		S3Debug:            s3IamDebug,
+		CacheDisable:       iamCacheDisable,
+		CacheTTL:           iamCacheTTL,
+		CachePrune:         iamCachePrune,
+	})
+	if err != nil {
+		return fmt.Errorf("setup iam: %w", err)
+	}
+
+	logger, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:    accessLog,
+		WebhookURL: logWebhookURL,
+	})
+	if err != nil {
+		return fmt.Errorf("setup logger: %w", err)
+	}
+
+	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
+		KafkaURL:      kafkaURL,
+		KafkaTopic:    kafkaTopic,
+		KafkaTopicKey: kafkaKey,
+		NatsURL:       natsURL,
+		NatsTopic:     natsTopic,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to connect to the message broker: %w", err)
+	}
+
+	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
+		Access: rootUserAccess,
+		Secret: rootUserSecret,
+	}, port, region, iam, logger, evSender, opts...)
+	if err != nil {
+		return fmt.Errorf("init gateway: %v", err)
+	}
+
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
+
+	c := make(chan error, 2)
+	go func() { c <- srv.Serve() }()
+	if admPort != "" {
+		go func() { c <- admSrv.Serve() }()
+	}
+
+	// for/select blocks until shutdown
+Loop:
+	for {
+		select {
+		case <-ctx.Done():
+			err = ctx.Err()
+			break Loop
+		case err = <-c:
+			break Loop
+		case <-sigHup:
+			if logger != nil {
+				err = logger.HangUp()
+				if err != nil {
+					err = fmt.Errorf("HUP logger: %w", err)
+					break Loop
+				}
+			}
+		}
+	}
+	saveErr := err
+
+	be.Shutdown()
+
+	err = iam.Shutdown()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
+	}
+
+	if logger != nil {
+		err := logger.Shutdown()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
+		}
+	}
+
+	return saveErr
+}

cmd/versitygw/posix.go

@@ -0,0 +1,53 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+func posixCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "posix",
+		Usage: "posix filesystem storage backend",
+		Description: `Any posix filesystem that supports extended attributes. The top level
+directory for the gateway must be provided. All sub directories of the
+top level directory are treated as buckets, and all files/directories
+below the "bucket directory" are treated as the objects. The object
+name is split on "/" separator to translate to posix storage.
+For example:
+top level: /mnt/fs/gwroot
+bucket: mybucket
+object: a/b/c/myobject
+will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
+		Action: runPosix,
+	}
+}
+
+func runPosix(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no directory provided for operation")
+	}
+
+	be, err := posix.New(ctx.Args().Get(0))
+	if err != nil {
+		return fmt.Errorf("init posix: %v", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/s3.go

@@ -0,0 +1,94 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/s3proxy"
+)
+
+var (
+	s3proxyAccess          string
+	s3proxySecret          string
+	s3proxyEndpoint        string
+	s3proxyRegion          string
+	s3proxyDisableChecksum bool
+	s3proxySslSkipVerify   bool
+	s3proxyDebug           bool
+)
+
+func s3Command() *cli.Command {
+	return &cli.Command{
+		Name:  "s3",
+		Usage: "s3 storage backend",
+		Description: `This runs the gateway like an s3 proxy redirecting requests
+to an s3 storage backend service.`,
+		Action: runS3,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "access",
+				Usage:       "s3 proxy server access key id",
+				Value:       "",
+				Required:    true,
+				Destination: &s3proxyAccess,
+				Aliases:     []string{"a"},
+			},
+			&cli.StringFlag{
+				Name:        "secret",
+				Usage:       "s3 proxy server secret access key",
+				Value:       "",
+				Required:    true,
+				Destination: &s3proxySecret,
+				Aliases:     []string{"s"},
+			},
+			&cli.StringFlag{
+				Name:        "endpoint",
+				Usage:       "s3 service endpoint, default AWS if not specified",
+				Value:       "",
+				Destination: &s3proxyEndpoint,
+			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "s3 service region, default 'us-east-1' if not specified",
+				Value:       "us-east-1",
+				Destination: &s3proxyRegion,
+			},
+			&cli.BoolFlag{
+				Name:        "disable-checksum",
+				Usage:       "disable gateway to server object checksums",
+				Value:       false,
+				Destination: &s3proxyDisableChecksum,
+			},
+			&cli.BoolFlag{
+				Name:        "ssl-skip-verify",
+				Usage:       "skip ssl cert verification for s3 service",
+				Value:       false,
+				Destination: &s3proxySslSkipVerify,
+			},
+			&cli.BoolFlag{
+				Name:        "debug",
+				Usage:       "output extra debug tracing",
+				Value:       false,
+				Destination: &s3proxyDebug,
+			},
+		},
+	}
+}
+
+func runS3(ctx *cli.Context) error {
+	be := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/scoutfs.go

@@ -0,0 +1,73 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/scoutfs"
+)
+
+var (
+	glacier bool
+)
+
+func scoutfsCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "scoutfs",
+		Usage: "scoutfs filesystem storage backend",
+		Description: `Support for ScoutFS.
+The top level directory for the gateway must be provided. All sub directories
+of the top level directory are treated as buckets, and all files/directories
+below the "bucket directory" are treated as the objects. The object name is
+split on "/" separator to translate to posix storage.
+For example:
+top level: /mnt/fs/gwroot
+bucket: mybucket
+object: a/b/c/myobject
+will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+
+ScoutFS contains optimizations for multipart uploads using extent
+move interfaces as well as support for tiered filesystems.`,
+		Action: runScoutfs,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "glacier",
+				Usage:       "enable glacier emulation mode",
+				Aliases:     []string{"g"},
+				Destination: &glacier,
+			},
+		},
+	}
+}
+
+func runScoutfs(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no directory provided for operation")
+	}
+
+	var opts []scoutfs.Option
+	if glacier {
+		opts = append(opts, scoutfs.WithGlacierEmulation())
+	}
+
+	be, err := scoutfs.New(ctx.Args().Get(0), opts...)
+	if err != nil {
+		return fmt.Errorf("init scoutfs: %v", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/signal.go

@@ -0,0 +1,44 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+var (
+	sigDone = make(chan bool, 1)
+	sigHup  = make(chan bool, 1)
+)
+
+func setupSignalHandler() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
+
+	go func() {
+		for sig := range sigs {
+			fmt.Fprintf(os.Stderr, "caught signal %v\n", sig)
+			switch sig {
+			case syscall.SIGINT, syscall.SIGTERM:
+				sigDone <- true
+			case syscall.SIGHUP:
+				sigHup <- true
+			}
+		}
+	}()
+}

cmd/versitygw/test.go

@@ -0,0 +1,294 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/integration"
+)
+
+var (
+	awsID           string
+	awsSecret       string
+	endpoint        string
+	prefix          string
+	dstBucket       string
+	partSize        int64
+	objSize         int64
+	concurrency     int
+	files           int
+	totalReqs       int
+	upload          bool
+	download        bool
+	pathStyle       bool
+	checksumDisable bool
+)
+
+func testCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "test",
+		Usage: "Client side testing command for the gateway",
+		Description: `The testing CLI is used to test group of versitygw actions.
+		It also includes some performance and stress testing`,
+		Subcommands: initTestCommands(),
+		Flags:       initTestFlags(),
+	}
+}
+
+func initTestFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "access",
+			Usage:       "aws user access key",
+			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"},
+			Aliases:     []string{"a"},
+			Destination: &awsID,
+		},
+		&cli.StringFlag{
+			Name:        "secret",
+			Usage:       "aws user secret access key",
+			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"},
+			Aliases:     []string{"s"},
+			Destination: &awsSecret,
+		},
+		&cli.StringFlag{
+			Name:        "endpoint",
+			Usage:       "s3 server endpoint",
+			Destination: &endpoint,
+			Aliases:     []string{"e"},
+		},
+		&cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "enable debug mode",
+			Aliases:     []string{"d"},
+			Destination: &debug,
+		},
+	}
+}
+
+func initTestCommands() []*cli.Command {
+	return append([]*cli.Command{
+		{
+			Name:        "full-flow",
+			Usage:       "Tests the full flow of gateway.",
+			Description: `Runs all the available tests to test the full flow of the gateway.`,
+			Action:      getAction(integration.TestFullFlow),
+		},
+		{
+			Name:   "posix",
+			Usage:  "Tests posix specific features",
+			Action: getAction(integration.TestPosix),
+		},
+		{
+			Name:  "bench",
+			Usage: "Runs download/upload performance test on the gateway",
+			Description: `Uploads/downloads some number(specified by flags) of files with some capacity(bytes).
+			Logs the results to the console`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "files",
+					Usage:       "Number of objects to read/write",
+					Value:       1,
+					Destination: &files,
+				},
+				&cli.Int64Flag{
+					Name:        "objsize",
+					Usage:       "Uploading object size",
+					Value:       0,
+					Destination: &objSize,
+				},
+				&cli.StringFlag{
+					Name:        "prefix",
+					Usage:       "Object name prefix",
+					Destination: &prefix,
+				},
+				&cli.BoolFlag{
+					Name:        "upload",
+					Usage:       "Upload data to the gateway",
+					Value:       false,
+					Destination: &upload,
+				},
+				&cli.BoolFlag{
+					Name:        "download",
+					Usage:       "Download data to the gateway",
+					Value:       false,
+					Destination: &download,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to read/write data",
+					Destination: &dstBucket,
+				},
+				&cli.Int64Flag{
+					Name:        "partSize",
+					Usage:       "Upload/download size per thread",
+					Value:       64 * 1024 * 1024,
+					Destination: &partSize,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "Upload/download threads per object",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "pathStyle",
+					Usage:       "Use Pathstyle bucket addressing",
+					Value:       false,
+					Destination: &pathStyle,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				if upload && download {
+					return fmt.Errorf("must only specify one of upload or download")
+				}
+				if !upload && !download {
+					return fmt.Errorf("must specify one of upload or download")
+				}
+
+				if dstBucket == "" {
+					return fmt.Errorf("must specify bucket")
+				}
+
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+					integration.WithPartSize(partSize),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if pathStyle {
+					opts = append(opts, integration.WithPathStyle())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				if upload {
+					return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+				} else {
+					return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+				}
+			},
+		},
+		{
+			Name:        "throughput",
+			Usage:       "Runs throughput performance test on the gateway",
+			Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "reqs",
+					Usage:       "Total number of requests to send.",
+					Value:       1000,
+					Destination: &totalReqs,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to make the requests",
+					Destination: &dstBucket,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "threads per request",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				if dstBucket == "" {
+					return fmt.Errorf("must specify the destination bucket")
+				}
+
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+			},
+		},
+	}, extractIntTests()...)
+}
+
+type testFunc func(*integration.S3Conf)
+
+func getAction(tf testFunc) func(*cli.Context) error {
+	return func(ctx *cli.Context) error {
+		opts := []integration.Option{
+			integration.WithAccess(awsID),
+			integration.WithSecret(awsSecret),
+			integration.WithRegion(region),
+			integration.WithEndpoint(endpoint),
+		}
+		if debug {
+			opts = append(opts, integration.WithDebug())
+		}
+
+		s := integration.NewS3Conf(opts...)
+		tf(s)
+
+		fmt.Println()
+		fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount)
+		if integration.FailCount > 0 {
+			return fmt.Errorf("test failed with %v errors", integration.FailCount)
+		}
+		return nil
+	}
+}
+
+func extractIntTests() (commands []*cli.Command) {
+	tests := integration.GetIntTests()
+	for key, val := range tests {
+		testKey := key
+		testFunc := val
+		commands = append(commands, &cli.Command{
+			Name:  testKey,
+			Usage: fmt.Sprintf("Runs %v integration test", testKey),
+			Action: func(ctx *cli.Context) error {
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+
+				s := integration.NewS3Conf(opts...)
+				err := testFunc(s)
+				return err
+			},
+		})
+	}
+	return
+}

docker-compose.yml

@@ -0,0 +1,23 @@
+version: "3"
+services:
+  posix:
+    build: 
+      context: .
+      dockerfile: ./Dockerfile.dev
+      args:
+        - IAM_DIR=${IAM_DIR}
+        - SETUP_DIR=${SETUP_DIR}
+    volumes:
+      - ./:/app
+    ports:
+      - "${POSIX_PORT}:${POSIX_PORT}"
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$POSIX_PORT -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR posix $SETUP_DIR"]
+  proxy:
+    build: 
+      context: .
+      dockerfile: ./Dockerfile.dev
+    volumes:
+      - ./:/app
+    ports:
+      - "${PROXY_PORT}:${PROXY_PORT}"
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$PROXY_PORT s3 -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --endpoint http://posix:$POSIX_PORT"]

go.mod

@@ -0,0 +1,60 @@
+module github.com/versity/versitygw
+
+go 1.20
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.24.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.6
+	github.com/aws/smithy-go v1.19.0
+	github.com/go-ldap/ldap/v3 v3.4.6
+	github.com/gofiber/fiber/v2 v2.51.0
+	github.com/google/uuid v1.5.0
+	github.com/nats-io/nats.go v1.31.0
+	github.com/pkg/xattr v0.4.9
+	github.com/segmentio/kafka-go v0.4.47
+	github.com/urfave/cli/v2 v2.26.0
+	github.com/valyala/fasthttp v1.51.0
+	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
+	golang.org/x/sys v0.15.0
+)
+
+require (
+	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.26.1
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.12
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.8
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+)

go.sum

@@ -0,0 +1,176 @@
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
+github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
+github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o=
+github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.8 h1:7wCngExMTAW2Bjf0Y92uWap6ZUcenLLWI5T3VJiQneU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.8/go.mod h1:XVrAWYYM4ZRwOCOuLoUiao5hbLqNutEdqwCR3ZvkXgc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.6 h1:bkmlzokzTJyrFNA0J+EPlsF8x4/wp+9D45HTHO/ZUiY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.6/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
+github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
+github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
+github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ=
+github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
+github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
+github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
+github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
+github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
+github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI=
+github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
+github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration/bench.go

@@ -0,0 +1,142 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type prefResult struct {
+	elapsed time.Duration
+	size    int64
+	err     error
+}
+
+func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+	if objSize == 0 {
+		return fmt.Errorf("must specify object size for upload")
+	}
+
+	if objSize > (int64(10000) * s.PartSize) {
+		return fmt.Errorf("object size can not exceed 10000 * chunksize")
+	}
+
+	runF("performance test: upload objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			var r io.Reader = NewDataReader(int(objSize), int(s.PartSize))
+
+			start := time.Now()
+			err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = objSize
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed := time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			break
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run upload: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return nil
+}
+
+func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+
+	runF("performance test: download objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			nw := NewNullWriter()
+			start := time.Now()
+			n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = n
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed := time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			break
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run download: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return nil
+}
+
+func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
+	client := s3.NewFromConfig(s.Config())
+	var wg sync.WaitGroup
+	var resErr error
+
+	// Record the start time
+	startTime := time.Now()
+	runF("performance test: measuring request per second")
+
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < totalReqs/s.Concurrency; i++ {
+				_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket})
+				if err != nil && resErr != nil {
+					resErr = err
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	if resErr != nil {
+		failF("performance test failed with error: %w", resErr)
+		return nil
+	}
+	elapsedTime := time.Since(startTime)
+	rps := int(float64(totalReqs) / elapsedTime.Seconds())
+
+	passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
+	return nil
+}

integration/data-io.go

@@ -0,0 +1,84 @@
+package integration
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"hash"
+	"io"
+)
+
+type RReader struct {
+	buf      []byte
+	dataleft int
+	hash     hash.Hash
+}
+
+func NewDataReader(totalsize, bufsize int) *RReader {
+	b := make([]byte, bufsize)
+	rand.Read(b)
+	return &RReader{
+		buf:      b,
+		dataleft: totalsize,
+		hash:     sha256.New(),
+	}
+}
+
+func (r *RReader) Read(p []byte) (int, error) {
+	n := min(len(p), len(r.buf), r.dataleft)
+	r.dataleft -= n
+	err := error(nil)
+	if n == 0 {
+		err = io.EOF
+	}
+	r.hash.Write(r.buf[:n])
+	return copy(p, r.buf[:n]), err
+}
+
+func (r *RReader) Sum() []byte {
+	return r.hash.Sum(nil)
+}
+
+type ZReader struct {
+	buf      []byte
+	dataleft int
+}
+
+func NewZeroReader(totalsize, bufsize int) *ZReader {
+	b := make([]byte, bufsize)
+	return &ZReader{buf: b, dataleft: totalsize}
+}
+
+func (r *ZReader) Read(p []byte) (int, error) {
+	n := min(len(p), len(r.buf), r.dataleft)
+	r.dataleft -= n
+	err := error(nil)
+	if n == 0 {
+		err = io.EOF
+	}
+	return copy(p, r.buf[:n]), err
+}
+
+func min(values ...int) int {
+	if len(values) == 0 {
+		return 0
+	}
+
+	min := values[0]
+	for _, v := range values {
+		if v < min {
+			min = v
+		}
+	}
+
+	return min
+}
+
+type NW struct{}
+
+func NewNullWriter() NW {
+	return NW{}
+}
+
+func (NW) WriteAt(p []byte, off int64) (n int, err error) {
+	return len(p), nil
+}

integration/group-tests.go

@@ -0,0 +1,355 @@
+package integration
+
+func TestAuthentication(s *S3Conf) {
+	Authentication_empty_auth_header(s)
+	Authentication_invalid_auth_header(s)
+	Authentication_unsupported_signature_version(s)
+	Authentication_malformed_credentials(s)
+	Authentication_malformed_credentials_invalid_parts(s)
+	Authentication_credentials_terminated_string(s)
+	Authentication_credentials_incorrect_service(s)
+	Authentication_credentials_incorrect_region(s)
+	Authentication_credentials_invalid_date(s)
+	Authentication_credentials_future_date(s)
+	Authentication_credentials_past_date(s)
+	Authentication_credentials_non_existing_access_key(s)
+	Authentication_invalid_signed_headers(s)
+	Authentication_missing_date_header(s)
+	Authentication_invalid_date_header(s)
+	Authentication_date_mismatch(s)
+	Authentication_incorrect_payload_hash(s)
+	Authentication_incorrect_md5(s)
+	Authentication_signature_error_incorrect_secret_key(s)
+}
+
+func TestCreateBucket(s *S3Conf) {
+	CreateBucket_invalid_bucket_name(s)
+	CreateBucket_existing_bucket(s)
+	CreateBucket_as_user(s)
+	CreateDeleteBucket_success(s)
+}
+
+func TestHeadBucket(s *S3Conf) {
+	HeadBucket_non_existing_bucket(s)
+	HeadBucket_success(s)
+}
+
+func TestListBuckets(s *S3Conf) {
+	ListBuckets_as_user(s)
+	ListBuckets_as_admin(s)
+	ListBuckets_success(s)
+}
+
+func TestDeleteBucket(s *S3Conf) {
+	DeleteBucket_non_existing_bucket(s)
+	DeleteBucket_non_empty_bucket(s)
+	DeleteBucket_success_status_code(s)
+}
+
+func TestPutObject(s *S3Conf) {
+	PutObject_non_existing_bucket(s)
+	PutObject_special_chars(s)
+	PutObject_invalid_long_tags(s)
+	PutObject_success(s)
+	PutObject_invalid_credentials(s)
+}
+
+func TestHeadObject(s *S3Conf) {
+	HeadObject_non_existing_object(s)
+	HeadObject_success(s)
+}
+
+func TestGetObject(s *S3Conf) {
+	GetObject_non_existing_key(s)
+	GetObject_invalid_ranges(s)
+	GetObject_with_meta(s)
+	GetObject_success(s)
+	GetObject_by_range_success(s)
+}
+
+func TestListObjects(s *S3Conf) {
+	ListObjects_non_existing_bucket(s)
+	ListObjects_with_prefix(s)
+	ListObject_truncated(s)
+	ListObjects_invalid_max_keys(s)
+	ListObjects_max_keys_0(s)
+	ListObjects_delimiter(s)
+	ListObjects_max_keys_none(s)
+	ListObjects_marker_not_from_obj_list(s)
+}
+
+func TestDeleteObject(s *S3Conf) {
+	DeleteObject_non_existing_object(s)
+	DeleteObject_success(s)
+	DeleteObject_success_status_code(s)
+}
+
+func TestDeleteObjects(s *S3Conf) {
+	DeleteObjects_empty_input(s)
+	DeleteObjects_non_existing_objects(s)
+	DeleteObjects_success(s)
+}
+
+func TestCopyObject(s *S3Conf) {
+	CopyObject_non_existing_dst_bucket(s)
+	CopyObject_not_owned_source_bucket(s)
+	CopyObject_copy_to_itself(s)
+	CopyObject_to_itself_with_new_metadata(s)
+	CopyObject_success(s)
+}
+
+func TestPutObjectTagging(s *S3Conf) {
+	PutObjectTagging_non_existing_object(s)
+	PutObjectTagging_long_tags(s)
+	PutObjectTagging_success(s)
+}
+
+func TestGetObjectTagging(s *S3Conf) {
+	GetObjectTagging_non_existing_object(s)
+	GetObjectTagging_success(s)
+}
+
+func TestDeleteObjectTagging(s *S3Conf) {
+	DeleteObjectTagging_non_existing_object(s)
+	DeleteObjectTagging_success_status(s)
+	DeleteObjectTagging_success(s)
+}
+
+func TestCreateMultipartUpload(s *S3Conf) {
+	CreateMultipartUpload_non_existing_bucket(s)
+	CreateMultipartUpload_success(s)
+}
+
+func TestUploadPart(s *S3Conf) {
+	UploadPart_non_existing_bucket(s)
+	UploadPart_invalid_part_number(s)
+	UploadPart_non_existing_key(s)
+	UploadPart_non_existing_mp_upload(s)
+	UploadPart_success(s)
+}
+
+func TestUploadPartCopy(s *S3Conf) {
+	UploadPartCopy_non_existing_bucket(s)
+	UploadPartCopy_incorrect_uploadId(s)
+	UploadPartCopy_incorrect_object_key(s)
+	UploadPartCopy_invalid_part_number(s)
+	UploadPartCopy_invalid_copy_source(s)
+	UploadPartCopy_non_existing_source_bucket(s)
+	UploadPartCopy_non_existing_source_object_key(s)
+	UploadPartCopy_success(s)
+	UploadPartCopy_by_range_invalid_range(s)
+	UploadPartCopy_greater_range_than_obj_size(s)
+	UploadPartCopy_by_range_success(s)
+}
+
+func TestListParts(s *S3Conf) {
+	ListParts_incorrect_uploadId(s)
+	ListParts_incorrect_object_key(s)
+	ListParts_success(s)
+}
+
+func TestListMultipartUploads(s *S3Conf) {
+	ListMultipartUploads_non_existing_bucket(s)
+	ListMultipartUploads_empty_result(s)
+	ListMultipartUploads_invalid_max_uploads(s)
+	ListMultipartUploads_max_uploads(s)
+	ListMultipartUploads_incorrect_next_key_marker(s)
+	ListMultipartUploads_ignore_upload_id_marker(s)
+	ListMultipartUploads_success(s)
+}
+
+func TestAbortMultipartUpload(s *S3Conf) {
+	AbortMultipartUpload_non_existing_bucket(s)
+	AbortMultipartUpload_incorrect_uploadId(s)
+	AbortMultipartUpload_incorrect_object_key(s)
+	AbortMultipartUpload_success(s)
+	AbortMultipartUpload_success_status_code(s)
+}
+
+func TestCompleteMultipartUpload(s *S3Conf) {
+	CompletedMultipartUpload_non_existing_bucket(s)
+	CompleteMultipartUpload_invalid_part_number(s)
+	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_success(s)
+}
+
+func TestPutBucketAcl(s *S3Conf) {
+	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_invalid_acl_canned_and_acp(s)
+	PutBucketAcl_invalid_acl_canned_and_grants(s)
+	PutBucketAcl_invalid_acl_acp_and_grants(s)
+	PutBucketAcl_invalid_owner(s)
+	PutBucketAcl_success_access_denied(s)
+	PutBucketAcl_success_grants(s)
+	PutBucketAcl_success_canned_acl(s)
+	PutBucketAcl_success_acp(s)
+}
+
+func TestGetBucketAcl(s *S3Conf) {
+	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_access_denied(s)
+	GetBucketAcl_success(s)
+}
+
+func TestFullFlow(s *S3Conf) {
+	TestAuthentication(s)
+	TestCreateBucket(s)
+	TestHeadBucket(s)
+	TestListBuckets(s)
+	TestDeleteBucket(s)
+	TestPutObject(s)
+	TestHeadObject(s)
+	TestGetObject(s)
+	TestListObjects(s)
+	TestDeleteObject(s)
+	TestDeleteObjects(s)
+	TestCopyObject(s)
+	TestPutObjectTagging(s)
+	TestDeleteObjectTagging(s)
+	TestCreateMultipartUpload(s)
+	TestUploadPart(s)
+	TestUploadPartCopy(s)
+	TestListParts(s)
+	TestListMultipartUploads(s)
+	TestAbortMultipartUpload(s)
+	TestCompleteMultipartUpload(s)
+	TestPutBucketAcl(s)
+	TestGetBucketAcl(s)
+}
+
+func TestPosix(s *S3Conf) {
+	PutObject_overwrite_dir_obj(s)
+	PutObject_overwrite_file_obj(s)
+	PutObject_dir_obj_with_data(s)
+	CreateMultipartUpload_dir_obj(s)
+}
+
+type IntTests map[string]func(s *S3Conf) error
+
+func GetIntTests() IntTests {
+	return IntTests{
+		"Authentication_empty_auth_header":                    Authentication_empty_auth_header,
+		"Authentication_invalid_auth_header":                  Authentication_invalid_auth_header,
+		"Authentication_unsupported_signature_version":        Authentication_unsupported_signature_version,
+		"Authentication_malformed_credentials":                Authentication_malformed_credentials,
+		"Authentication_malformed_credentials_invalid_parts":  Authentication_malformed_credentials_invalid_parts,
+		"Authentication_credentials_terminated_string":        Authentication_credentials_terminated_string,
+		"Authentication_credentials_incorrect_service":        Authentication_credentials_incorrect_service,
+		"Authentication_credentials_incorrect_region":         Authentication_credentials_incorrect_region,
+		"Authentication_credentials_invalid_date":             Authentication_credentials_invalid_date,
+		"Authentication_credentials_future_date":              Authentication_credentials_future_date,
+		"Authentication_credentials_past_date":                Authentication_credentials_past_date,
+		"Authentication_credentials_non_existing_access_key":  Authentication_credentials_non_existing_access_key,
+		"Authentication_invalid_signed_headers":               Authentication_invalid_signed_headers,
+		"Authentication_missing_date_header":                  Authentication_missing_date_header,
+		"Authentication_invalid_date_header":                  Authentication_invalid_date_header,
+		"Authentication_date_mismatch":                        Authentication_date_mismatch,
+		"Authentication_incorrect_payload_hash":               Authentication_incorrect_payload_hash,
+		"Authentication_incorrect_md5":                        Authentication_incorrect_md5,
+		"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
+		"CreateBucket_invalid_bucket_name":                    CreateBucket_invalid_bucket_name,
+		"CreateBucket_existing_bucket":                        CreateBucket_existing_bucket,
+		"CreateBucket_as_user":                                CreateBucket_as_user,
+		"CreateDeleteBucket_success":                          CreateDeleteBucket_success,
+		"HeadBucket_non_existing_bucket":                      HeadBucket_non_existing_bucket,
+		"HeadBucket_success":                                  HeadBucket_success,
+		"ListBuckets_as_user":                                 ListBuckets_as_user,
+		"ListBuckets_as_admin":                                ListBuckets_as_admin,
+		"ListBuckets_success":                                 ListBuckets_success,
+		"DeleteBucket_non_existing_bucket":                    DeleteBucket_non_existing_bucket,
+		"DeleteBucket_non_empty_bucket":                       DeleteBucket_non_empty_bucket,
+		"DeleteBucket_success_status_code":                    DeleteBucket_success_status_code,
+		"PutObject_non_existing_bucket":                       PutObject_non_existing_bucket,
+		"PutObject_special_chars":                             PutObject_special_chars,
+		"PutObject_invalid_long_tags":                         PutObject_invalid_long_tags,
+		"PutObject_success":                                   PutObject_success,
+		"PutObject_invalid_credentials":                       PutObject_invalid_credentials,
+		"HeadObject_non_existing_object":                      HeadObject_non_existing_object,
+		"HeadObject_success":                                  HeadObject_success,
+		"GetObject_non_existing_key":                          GetObject_non_existing_key,
+		"GetObject_invalid_ranges":                            GetObject_invalid_ranges,
+		"GetObject_with_meta":                                 GetObject_with_meta,
+		"GetObject_success":                                   GetObject_success,
+		"GetObject_by_range_success":                          GetObject_by_range_success,
+		"ListObjects_non_existing_bucket":                     ListObjects_non_existing_bucket,
+		"ListObjects_with_prefix":                             ListObjects_with_prefix,
+		"ListObject_truncated":                                ListObject_truncated,
+		"ListObjects_invalid_max_keys":                        ListObjects_invalid_max_keys,
+		"ListObjects_max_keys_0":                              ListObjects_max_keys_0,
+		"ListObjects_delimiter":                               ListObjects_delimiter,
+		"ListObjects_max_keys_none":                           ListObjects_max_keys_none,
+		"ListObjects_marker_not_from_obj_list":                ListObjects_marker_not_from_obj_list,
+		"DeleteObject_non_existing_object":                    DeleteObject_non_existing_object,
+		"DeleteObject_success":                                DeleteObject_success,
+		"DeleteObject_success_status_code":                    DeleteObject_success_status_code,
+		"DeleteObjects_empty_input":                           DeleteObjects_empty_input,
+		"DeleteObjects_non_existing_objects":                  DeleteObjects_non_existing_objects,
+		"DeleteObjects_success":                               DeleteObjects_success,
+		"CopyObject_non_existing_dst_bucket":                  CopyObject_non_existing_dst_bucket,
+		"CopyObject_not_owned_source_bucket":                  CopyObject_not_owned_source_bucket,
+		"CopyObject_copy_to_itself":                           CopyObject_copy_to_itself,
+		"CopyObject_to_itself_with_new_metadata":              CopyObject_to_itself_with_new_metadata,
+		"CopyObject_success":                                  CopyObject_success,
+		"PutObjectTagging_non_existing_object":                PutObjectTagging_non_existing_object,
+		"PutObjectTagging_long_tags":                          PutObjectTagging_long_tags,
+		"PutObjectTagging_success":                            PutObjectTagging_success,
+		"GetObjectTagging_non_existing_object":                GetObjectTagging_non_existing_object,
+		"GetObjectTagging_success":                            GetObjectTagging_success,
+		"DeleteObjectTagging_non_existing_object":             DeleteObjectTagging_non_existing_object,
+		"DeleteObjectTagging_success_status":                  DeleteObjectTagging_success_status,
+		"DeleteObjectTagging_success":                         DeleteObjectTagging_success,
+		"CreateMultipartUpload_non_existing_bucket":           CreateMultipartUpload_non_existing_bucket,
+		"CreateMultipartUpload_success":                       CreateMultipartUpload_success,
+		"UploadPart_non_existing_bucket":                      UploadPart_non_existing_bucket,
+		"UploadPart_invalid_part_number":                      UploadPart_invalid_part_number,
+		"UploadPart_non_existing_key":                         UploadPart_non_existing_key,
+		"UploadPart_non_existing_mp_upload":                   UploadPart_non_existing_mp_upload,
+		"UploadPart_success":                                  UploadPart_success,
+		"UploadPartCopy_non_existing_bucket":                  UploadPartCopy_non_existing_bucket,
+		"UploadPartCopy_incorrect_uploadId":                   UploadPartCopy_incorrect_uploadId,
+		"UploadPartCopy_incorrect_object_key":                 UploadPartCopy_incorrect_object_key,
+		"UploadPartCopy_invalid_part_number":                  UploadPartCopy_invalid_part_number,
+		"UploadPartCopy_invalid_copy_source":                  UploadPartCopy_invalid_copy_source,
+		"UploadPartCopy_non_existing_source_bucket":           UploadPartCopy_non_existing_source_bucket,
+		"UploadPartCopy_non_existing_source_object_key":       UploadPartCopy_non_existing_source_object_key,
+		"UploadPartCopy_success":                              UploadPartCopy_success,
+		"UploadPartCopy_by_range_invalid_range":               UploadPartCopy_by_range_invalid_range,
+		"UploadPartCopy_greater_range_than_obj_size":          UploadPartCopy_greater_range_than_obj_size,
+		"UploadPartCopy_by_range_success":                     UploadPartCopy_by_range_success,
+		"ListParts_incorrect_uploadId":                        ListParts_incorrect_uploadId,
+		"ListParts_incorrect_object_key":                      ListParts_incorrect_object_key,
+		"ListParts_success":                                   ListParts_success,
+		"ListMultipartUploads_non_existing_bucket":            ListMultipartUploads_non_existing_bucket,
+		"ListMultipartUploads_empty_result":                   ListMultipartUploads_empty_result,
+		"ListMultipartUploads_invalid_max_uploads":            ListMultipartUploads_invalid_max_uploads,
+		"ListMultipartUploads_max_uploads":                    ListMultipartUploads_max_uploads,
+		"ListMultipartUploads_incorrect_next_key_marker":      ListMultipartUploads_incorrect_next_key_marker,
+		"ListMultipartUploads_ignore_upload_id_marker":        ListMultipartUploads_ignore_upload_id_marker,
+		"ListMultipartUploads_success":                        ListMultipartUploads_success,
+		"AbortMultipartUpload_non_existing_bucket":            AbortMultipartUpload_non_existing_bucket,
+		"AbortMultipartUpload_incorrect_uploadId":             AbortMultipartUpload_incorrect_uploadId,
+		"AbortMultipartUpload_incorrect_object_key":           AbortMultipartUpload_incorrect_object_key,
+		"AbortMultipartUpload_success":                        AbortMultipartUpload_success,
+		"AbortMultipartUpload_success_status_code":            AbortMultipartUpload_success_status_code,
+		"CompletedMultipartUpload_non_existing_bucket":        CompletedMultipartUpload_non_existing_bucket,
+		"CompleteMultipartUpload_invalid_part_number":         CompleteMultipartUpload_invalid_part_number,
+		"CompleteMultipartUpload_invalid_ETag":                CompleteMultipartUpload_invalid_ETag,
+		"CompleteMultipartUpload_success":                     CompleteMultipartUpload_success,
+		"PutBucketAcl_non_existing_bucket":                    PutBucketAcl_non_existing_bucket,
+		"PutBucketAcl_invalid_acl_canned_and_acp":             PutBucketAcl_invalid_acl_canned_and_acp,
+		"PutBucketAcl_invalid_acl_canned_and_grants":          PutBucketAcl_invalid_acl_canned_and_grants,
+		"PutBucketAcl_invalid_acl_acp_and_grants":             PutBucketAcl_invalid_acl_acp_and_grants,
+		"PutBucketAcl_invalid_owner":                          PutBucketAcl_invalid_owner,
+		"PutBucketAcl_success_access_denied":                  PutBucketAcl_success_access_denied,
+		"PutBucketAcl_success_grants":                         PutBucketAcl_success_grants,
+		"PutBucketAcl_success_canned_acl":                     PutBucketAcl_success_canned_acl,
+		"PutBucketAcl_success_acp":                            PutBucketAcl_success_acp,
+		"GetBucketAcl_non_existing_bucket":                    GetBucketAcl_non_existing_bucket,
+		"GetBucketAcl_access_denied":                          GetBucketAcl_access_denied,
+		"GetBucketAcl_success":                                GetBucketAcl_success,
+		"PutObject_overwrite_dir_obj":                         PutObject_overwrite_dir_obj,
+		"PutObject_overwrite_file_obj":                        PutObject_overwrite_file_obj,
+		"PutObject_dir_obj_with_data":                         PutObject_dir_obj_with_data,
+		"CreateMultipartUpload_dir_obj":                       CreateMultipartUpload_dir_obj,
+	}
+}

integration/output.go

@@ -0,0 +1,31 @@
+package integration
+
+import "fmt"
+
+var (
+	colorReset = "\033[0m"
+	colorRed   = "\033[31m"
+	colorGreen = "\033[32m"
+	colorCyan  = "\033[36m"
+)
+
+var (
+	RunCount  = 0
+	PassCount = 0
+	FailCount = 0
+)
+
+func runF(format string, a ...interface{}) {
+	RunCount++
+	fmt.Printf(colorCyan+"RUN  "+colorReset+format+"\n", a...)
+}
+
+func failF(format string, a ...interface{}) {
+	FailCount++
+	fmt.Printf(colorRed+"FAIL "+colorReset+format+"\n", a...)
+}
+
+func passF(format string, a ...interface{}) {
+	PassCount++
+	fmt.Printf(colorGreen+"PASS "+colorReset+format+"\n", a...)
+}

integration/s3conf.go

@@ -0,0 +1,153 @@
+package integration
+
+import (
+	"context"
+	"io"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+)
+
+type S3Conf struct {
+	awsID           string
+	awsSecret       string
+	awsRegion       string
+	endpoint        string
+	checksumDisable bool
+	pathStyle       bool
+	PartSize        int64
+	Concurrency     int
+	debug           bool
+}
+
+func NewS3Conf(opts ...Option) *S3Conf {
+	s := &S3Conf{}
+
+	for _, opt := range opts {
+		opt(s)
+	}
+	return s
+}
+
+type Option func(*S3Conf)
+
+func WithAccess(ak string) Option {
+	return func(s *S3Conf) { s.awsID = ak }
+}
+func WithSecret(sk string) Option {
+	return func(s *S3Conf) { s.awsSecret = sk }
+}
+func WithRegion(r string) Option {
+	return func(s *S3Conf) { s.awsRegion = r }
+}
+func WithEndpoint(e string) Option {
+	return func(s *S3Conf) { s.endpoint = e }
+}
+func WithDisableChecksum() Option {
+	return func(s *S3Conf) { s.checksumDisable = true }
+}
+func WithPathStyle() Option {
+	return func(s *S3Conf) { s.pathStyle = true }
+}
+func WithPartSize(p int64) Option {
+	return func(s *S3Conf) { s.PartSize = p }
+}
+func WithConcurrency(c int) Option {
+	return func(s *S3Conf) { s.Concurrency = c }
+}
+func WithDebug() Option {
+	return func(s *S3Conf) { s.debug = true }
+}
+
+func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
+	// TODO support token/IAM
+	if c.awsSecret == "" {
+		c.awsSecret = os.Getenv("AWS_SECRET_ACCESS_KEY")
+	}
+	if c.awsSecret == "" {
+		log.Fatal("no AWS_SECRET_ACCESS_KEY found")
+	}
+
+	return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "")
+}
+
+func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               c.endpoint,
+		SigningRegion:     c.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}
+
+func (c *S3Conf) Config() aws.Config {
+	creds := c.getCreds()
+
+	tr := &http.Transport{}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(c.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if c.endpoint != "" && c.endpoint != "aws" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(c))
+	}
+
+	if c.checksumDisable {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if c.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	cfg, err := config.LoadDefaultConfig(
+		context.TODO(), opts...)
+	if err != nil {
+		log.Fatalln("error:", err)
+	}
+
+	return cfg
+}
+
+func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
+	uploader := manager.NewUploader(s3.NewFromConfig(c.Config()))
+	uploader.PartSize = c.PartSize
+	uploader.Concurrency = c.Concurrency
+
+	upinfo := &s3.PutObjectInput{
+		Body:   r,
+		Bucket: &bucket,
+		Key:    &object,
+	}
+
+	_, err := uploader.Upload(context.Background(), upinfo)
+	return err
+}
+
+func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
+	downloader := manager.NewDownloader(s3.NewFromConfig(c.Config()))
+	downloader.PartSize = c.PartSize
+	downloader.Concurrency = c.Concurrency
+
+	downinfo := &s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &object,
+	}
+
+	return downloader.Download(context.Background(), w, downinfo)
+}

integration/utils.go

@@ -0,0 +1,553 @@
+package integration
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	rnd "math/rand"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	bcktCount  = 0
+	succUsrCrt = "The user has been created successfully"
+	failUsrCrt = "failed to create a user: update iam data: account already exists"
+)
+
+func getBucketName() string {
+	bcktCount++
+	return fmt.Sprintf("test-bucket-%v", bcktCount)
+}
+
+func setup(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func teardown(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	deleteObject := func(bucket, key, versionId *string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
+			Bucket:    bucket,
+			Key:       key,
+			VersionId: versionId,
+		})
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to delete object %v: %w", *key, err)
+		}
+		return nil
+	}
+
+	in := &s3.ListObjectsV2Input{Bucket: &bucket}
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListObjectsV2(ctx, in)
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to list objects: %w", err)
+		}
+
+		for _, item := range out.Contents {
+			err = deleteObject(&bucket, item.Key, nil)
+			if err != nil {
+				return err
+			}
+		}
+
+		if *out.IsTruncated {
+			in.ContinuationToken = out.ContinuationToken
+		} else {
+			break
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) error {
+	runF(testName)
+	bucketName := getBucketName()
+	err := setup(s, bucketName)
+	if err != nil {
+		failF("%v: failed to create a bucket: %v", testName, err)
+		return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
+	}
+	client := s3.NewFromConfig(s.Config())
+	handlerErr := handler(client, bucketName)
+	if handlerErr != nil {
+		failF("%v: %v", testName, handlerErr)
+	}
+
+	err = teardown(s, bucketName)
+	if err != nil {
+		fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
+		if handlerErr == nil {
+			return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
+		}
+	}
+	if handlerErr == nil {
+		passF(testName)
+	}
+
+	return handlerErr
+}
+
+type authConfig struct {
+	testName string
+	path     string
+	method   string
+	body     []byte
+	service  string
+	date     time.Time
+}
+
+func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
+	runF(cfg.testName)
+	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err)
+		return fmt.Errorf("%v: %w", cfg.testName, err)
+	}
+
+	err = handler(req)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err)
+		return fmt.Errorf("%v: %w", cfg.testName, err)
+	}
+	passF(cfg.testName)
+	return nil
+}
+
+func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
+	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(body)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date)
+	if signErr != nil {
+		return nil, fmt.Errorf("failed to sign the request: %w", signErr)
+	}
+
+	return req, nil
+}
+
+func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var errResp s3err.APIErrorResponse
+	err = xml.Unmarshal(body, &errResp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != apiErr.HTTPStatusCode {
+		return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
+	}
+	if errResp.Code != apiErr.Code {
+		return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
+	}
+	if errResp.Message != apiErr.Description {
+		return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
+	}
+
+	return nil
+}
+
+func checkApiErr(err error, apiErr s3err.APIError) error {
+	if err == nil {
+		return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
+			return nil
+		}
+
+		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+	}
+	return fmt.Errorf("expected aws api error, instead got: %w", err)
+}
+
+func checkSdkApiErr(err error, code string) error {
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() != code {
+			return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
+		}
+		return nil
+	}
+	return err
+}
+
+func putObjects(client *s3.Client, objs []string, bucket string) error {
+	for _, key := range objs {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Key:    &key,
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
+	data = make([]byte, lgth)
+	rand.Read(data)
+	csum = sha256.Sum256(data)
+	r := bytes.NewReader(data)
+	input.Body = r
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err = client.PutObject(ctx, input)
+	cancel()
+
+	return
+}
+
+func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: &bucket,
+		Key:    &key,
+	})
+	cancel()
+	return out, err
+}
+
+func isEqual(a, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i, d := range a {
+		if d != b[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+	for i, item := range list1 {
+		if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareParts(parts1, parts2 []types.Part) bool {
+	if len(parts1) != len(parts2) {
+		return false
+	}
+
+	for i, prt := range parts1 {
+		if *prt.PartNumber != *parts2[i].PartNumber {
+			return false
+		}
+		if *prt.ETag != *parts2[i].ETag {
+			return false
+		}
+	}
+	return true
+}
+
+func areTagsSame(tags1, tags2 []types.Tag) bool {
+	if len(tags1) != len(tags2) {
+		return false
+	}
+
+	for _, tag := range tags1 {
+		if !containsTag(tag, tags2) {
+			return false
+		}
+	}
+	return true
+}
+
+func containsTag(tag types.Tag, list []types.Tag) bool {
+	for _, item := range list {
+		if *item.Key == *tag.Key && *item.Value == *tag.Value {
+			return true
+		}
+	}
+	return false
+}
+
+func compareGrants(grts1, grts2 []types.Grant) bool {
+	if len(grts1) != len(grts2) {
+		return false
+	}
+
+	for i, grt := range grts1 {
+		if grt.Permission != grts2[i].Permission {
+			return false
+		}
+		if *grt.Grantee.ID != *grts2[i].Grantee.ID {
+			return false
+		}
+	}
+	return true
+}
+
+func execCommand(args ...string) ([]byte, error) {
+	cmd := exec.Command("./versitygw", args...)
+
+	return cmd.CombinedOutput()
+}
+
+func getString(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
+}
+
+func getPtr(str string) *string {
+	return &str
+}
+
+func areMapsSame(mp1, mp2 map[string]string) bool {
+	if len(mp1) != len(mp2) {
+		return false
+	}
+	for key, val := range mp1 {
+		if mp2[key] != val {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[*elem.Name] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[elem.Name]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareObjects(list1 []string, list2 []types.Object) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Prefix]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
+	dr := NewDataReader(size, size)
+	datafile := "rand.data"
+	w, err := os.Create(datafile)
+	if err != nil {
+		return parts, err
+	}
+	defer w.Close()
+
+	_, err = io.Copy(w, dr)
+	if err != nil {
+		return parts, err
+	}
+
+	fileInfo, err := w.Stat()
+	if err != nil {
+		return parts, err
+	}
+
+	partSize := fileInfo.Size() / int64(partCount)
+	var offset int64
+
+	for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
+		partStart := (partNumber - 1) * partSize
+		partEnd := partStart + partSize - 1
+		if partEnd > fileInfo.Size()-1 {
+			partEnd = fileInfo.Size() - 1
+		}
+		partBuffer := make([]byte, partEnd-partStart+1)
+		_, err := w.ReadAt(partBuffer, partStart)
+		if err != nil {
+			return parts, err
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		pn := int32(partNumber)
+		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     &bucket,
+			Key:        &key,
+			UploadId:   &uploadId,
+			Body:       bytes.NewReader(partBuffer),
+			PartNumber: &pn,
+		})
+		cancel()
+		if err != nil {
+			return parts, err
+		}
+		parts = append(parts, types.Part{
+			ETag:       out.ETag,
+			PartNumber: &pn,
+		})
+		offset += partSize
+	}
+
+	return parts, err
+}
+
+type user struct {
+	access string
+	secret string
+	role   string
+}
+
+func createUsers(s *S3Conf, users []user) error {
+	for _, usr := range users {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
+			return fmt.Errorf("failed to create a user account")
+		}
+	}
+	return nil
+}
+
+func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
+	for _, bucket := range buckets {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
+			return fmt.Errorf(string(out))
+		}
+	}
+
+	return nil
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func genRandString(length int) string {
+	source := rnd.NewSource(time.Now().UnixNano())
+	random := rnd.New(source)
+	result := make([]byte, length)
+	for i := range result {
+		result[i] = charset[random.Intn(len(charset))]
+	}
+	return string(result)
+}

runtests.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# make temp dirs
+rm -rf /tmp/gw
+mkdir /tmp/gw
+rm -rf /tmp/covdata
+mkdir /tmp/covdata
+
+# run server in background
+GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
+GW_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if server is still running
+if ! kill -0 $GW_PID; then
+	echo "server no longer running"
+	exit 1
+fi
+
+# run tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
+	echo "tests failed"
+	kill $GW_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_PID
+exit 0
+
+# if the above binary was built with -cover enabled (make testbin),
+# then the following can be used for code coverage reports:
+# go tool covdata percent -i=/tmp/covdata
+# go tool covdata textfmt -i=/tmp/covdata -o profile.txt
+# go tool cover -html=profile.txt
+

s3api/admin-router.go

@@ -0,0 +1,43 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+)
+
+type S3AdminRouter struct{}
+
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
+	controller := controllers.NewAdminController(iam, be)
+
+	// CreateUser admin api
+	app.Patch("/create-user", controller.CreateUser)
+
+	// DeleteUsers admin api
+	app.Patch("/delete-user", controller.DeleteUser)
+
+	// ListUsers admin api
+	app.Patch("/list-users", controller.ListUsers)
+
+	// ChangeBucketOwner admin api
+	app.Patch("/change-bucket-owner", controller.ChangeBucketOwner)
+
+	// ListBucketsAndOwners admin api
+	app.Patch("/list-buckets", controller.ListBuckets)
+}

s3api/admin-server.go

@@ -0,0 +1,72 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"crypto/tls"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+type S3AdminServer struct {
+	app     *fiber.App
+	backend backend.Backend
+	router  *S3AdminRouter
+	port    string
+	cert    *tls.Certificate
+}
+
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
+	server := &S3AdminServer{
+		app:     app,
+		backend: be,
+		router:  new(S3AdminRouter),
+		port:    port,
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
+
+	// Logging middlewares
+	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(nil))
+
+	// Authentication middlewares
+	app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(nil))
+	app.Use(middlewares.AclParser(be, nil))
+
+	server.router.Init(app, be, iam)
+
+	return server
+}
+
+type AdminOpt func(s *S3AdminServer)
+
+func WithAdminSrvTLS(cert tls.Certificate) AdminOpt {
+	return func(s *S3AdminServer) { s.cert = &cert }
+}
+
+func (sa *S3AdminServer) Serve() (err error) {
+	if sa.cert != nil {
+		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
+	}
+	return sa.app.Listen(sa.port)
+}

s3api/controllers/admin.go

@@ -0,0 +1,122 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+)
+
+type AdminController struct {
+	iam auth.IAMService
+	be  backend.Backend
+}
+
+func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
+	return AdminController{iam: iam, be: be}
+}
+
+func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	var usr auth.Account
+	err := json.Unmarshal(ctx.Body(), &usr)
+	if err != nil {
+		return fmt.Errorf("failed to parse request body: %w", err)
+	}
+
+	if usr.Role != "user" && usr.Role != "admin" {
+		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
+	}
+
+	err = c.iam.CreateAccount(usr)
+	if err != nil {
+		return fmt.Errorf("failed to create a user: %w", err)
+	}
+
+	return ctx.SendString("The user has been created successfully")
+}
+
+func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
+	access := ctx.Query("access")
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+
+	err := c.iam.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	return ctx.SendString("The user has been deleted successfully")
+}
+
+func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	accs, err := c.iam.ListUserAccounts()
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(accs)
+}
+
+func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	owner := ctx.Query("owner")
+	bucket := ctx.Query("bucket")
+
+	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return fmt.Errorf("user specified as the new bucket owner does not exist")
+	}
+
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
+	if err != nil {
+		return err
+	}
+
+	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+}
+
+func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+
+	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(buckets)
+}

s3api/controllers/admin_test.go

@@ -0,0 +1,481 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
+)
+
+func TestAdminController_CreateUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			CreateAccountFunc: func(account auth.Account) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/create-user", adminController.CreateUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	usr := auth.Account{
+		Access: "access",
+		Secret: "secret",
+		Role:   "invalid role",
+	}
+
+	user, _ := json.Marshal(&usr)
+
+	usr.Role = "admin"
+
+	succUsr, _ := json.Marshal(&usr)
+
+	appErr.Patch("/create-user", adminController.CreateUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-create-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-create-user-invalid-user-role",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-create-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.CreateUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.CreateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_DeleteUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			DeleteUserAccountFunc: func(access string) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/delete-user", adminController.DeleteUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/delete-user", adminController.DeleteUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-delete-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-delete-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.DeleteUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.DeleteUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ListUsers(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, fmt.Errorf("server error")
+			},
+		},
+	}
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/list-users", adminControllerErr.ListUsers)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-users", adminController.ListUsers)
+
+	appSucc := fiber.New()
+
+	appSucc.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appSucc.Patch("/list-users", adminController.ListUsers)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-list-users-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-iam-error",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-success",
+			app:  appSucc,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListUsers() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListUsers() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ChangeBucketOwner(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
+				return nil
+			},
+		},
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerIamErr := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, fmt.Errorf("unknown server error")
+			},
+		},
+	}
+
+	adminControllerIamAccDoesNotExist := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appIamErr := fiber.New()
+
+	appIamErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
+
+	appIamNoSuchUser := fiber.New()
+
+	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Change-bucket-owner-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-check-account-server-error",
+			app:  appIamErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-acc-does-not-exist",
+			app:  appIamNoSuchUser,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 201,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ChangeBucketOwner() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ChangeBucketOwner() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ListBuckets(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
+				return []s3response.Bucket{}, nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/list-buckets", adminController.ListBuckets)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "List-buckets-incorrect-role",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "List-buckets-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListBuckets() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}

s3api/controllers/iam_moq_test.go

@@ -0,0 +1,237 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package controllers
+
+import (
+	"github.com/versity/versitygw/auth"
+	"sync"
+)
+
+// Ensure, that IAMServiceMock does implement auth.IAMService.
+// If this is not the case, regenerate this file with moq.
+var _ auth.IAMService = &IAMServiceMock{}
+
+// IAMServiceMock is a mock implementation of auth.IAMService.
+//
+//	func TestSomethingThatUsesIAMService(t *testing.T) {
+//
+//		// make and configure a mocked auth.IAMService
+//		mockedIAMService := &IAMServiceMock{
+//			CreateAccountFunc: func(account auth.Account) error {
+//				panic("mock out the CreateAccount method")
+//			},
+//			DeleteUserAccountFunc: func(access string) error {
+//				panic("mock out the DeleteUserAccount method")
+//			},
+//			GetUserAccountFunc: func(access string) (auth.Account, error) {
+//				panic("mock out the GetUserAccount method")
+//			},
+//			ListUserAccountsFunc: func() ([]auth.Account, error) {
+//				panic("mock out the ListUserAccounts method")
+//			},
+//			ShutdownFunc: func() error {
+//				panic("mock out the Shutdown method")
+//			},
+//		}
+//
+//		// use mockedIAMService in code that requires auth.IAMService
+//		// and then make assertions.
+//
+//	}
+type IAMServiceMock struct {
+	// CreateAccountFunc mocks the CreateAccount method.
+	CreateAccountFunc func(account auth.Account) error
+
+	// DeleteUserAccountFunc mocks the DeleteUserAccount method.
+	DeleteUserAccountFunc func(access string) error
+
+	// GetUserAccountFunc mocks the GetUserAccount method.
+	GetUserAccountFunc func(access string) (auth.Account, error)
+
+	// ListUserAccountsFunc mocks the ListUserAccounts method.
+	ListUserAccountsFunc func() ([]auth.Account, error)
+
+	// ShutdownFunc mocks the Shutdown method.
+	ShutdownFunc func() error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// CreateAccount holds details about calls to the CreateAccount method.
+		CreateAccount []struct {
+			// Account is the account argument value.
+			Account auth.Account
+		}
+		// DeleteUserAccount holds details about calls to the DeleteUserAccount method.
+		DeleteUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+		}
+		// GetUserAccount holds details about calls to the GetUserAccount method.
+		GetUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+		}
+		// ListUserAccounts holds details about calls to the ListUserAccounts method.
+		ListUserAccounts []struct {
+		}
+		// Shutdown holds details about calls to the Shutdown method.
+		Shutdown []struct {
+		}
+	}
+	lockCreateAccount     sync.RWMutex
+	lockDeleteUserAccount sync.RWMutex
+	lockGetUserAccount    sync.RWMutex
+	lockListUserAccounts  sync.RWMutex
+	lockShutdown          sync.RWMutex
+}
+
+// CreateAccount calls CreateAccountFunc.
+func (mock *IAMServiceMock) CreateAccount(account auth.Account) error {
+	if mock.CreateAccountFunc == nil {
+		panic("IAMServiceMock.CreateAccountFunc: method is nil but IAMService.CreateAccount was just called")
+	}
+	callInfo := struct {
+		Account auth.Account
+	}{
+		Account: account,
+	}
+	mock.lockCreateAccount.Lock()
+	mock.calls.CreateAccount = append(mock.calls.CreateAccount, callInfo)
+	mock.lockCreateAccount.Unlock()
+	return mock.CreateAccountFunc(account)
+}
+
+// CreateAccountCalls gets all the calls that were made to CreateAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.CreateAccountCalls())
+func (mock *IAMServiceMock) CreateAccountCalls() []struct {
+	Account auth.Account
+} {
+	var calls []struct {
+		Account auth.Account
+	}
+	mock.lockCreateAccount.RLock()
+	calls = mock.calls.CreateAccount
+	mock.lockCreateAccount.RUnlock()
+	return calls
+}
+
+// DeleteUserAccount calls DeleteUserAccountFunc.
+func (mock *IAMServiceMock) DeleteUserAccount(access string) error {
+	if mock.DeleteUserAccountFunc == nil {
+		panic("IAMServiceMock.DeleteUserAccountFunc: method is nil but IAMService.DeleteUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+	}{
+		Access: access,
+	}
+	mock.lockDeleteUserAccount.Lock()
+	mock.calls.DeleteUserAccount = append(mock.calls.DeleteUserAccount, callInfo)
+	mock.lockDeleteUserAccount.Unlock()
+	return mock.DeleteUserAccountFunc(access)
+}
+
+// DeleteUserAccountCalls gets all the calls that were made to DeleteUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.DeleteUserAccountCalls())
+func (mock *IAMServiceMock) DeleteUserAccountCalls() []struct {
+	Access string
+} {
+	var calls []struct {
+		Access string
+	}
+	mock.lockDeleteUserAccount.RLock()
+	calls = mock.calls.DeleteUserAccount
+	mock.lockDeleteUserAccount.RUnlock()
+	return calls
+}
+
+// GetUserAccount calls GetUserAccountFunc.
+func (mock *IAMServiceMock) GetUserAccount(access string) (auth.Account, error) {
+	if mock.GetUserAccountFunc == nil {
+		panic("IAMServiceMock.GetUserAccountFunc: method is nil but IAMService.GetUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+	}{
+		Access: access,
+	}
+	mock.lockGetUserAccount.Lock()
+	mock.calls.GetUserAccount = append(mock.calls.GetUserAccount, callInfo)
+	mock.lockGetUserAccount.Unlock()
+	return mock.GetUserAccountFunc(access)
+}
+
+// GetUserAccountCalls gets all the calls that were made to GetUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.GetUserAccountCalls())
+func (mock *IAMServiceMock) GetUserAccountCalls() []struct {
+	Access string
+} {
+	var calls []struct {
+		Access string
+	}
+	mock.lockGetUserAccount.RLock()
+	calls = mock.calls.GetUserAccount
+	mock.lockGetUserAccount.RUnlock()
+	return calls
+}
+
+// ListUserAccounts calls ListUserAccountsFunc.
+func (mock *IAMServiceMock) ListUserAccounts() ([]auth.Account, error) {
+	if mock.ListUserAccountsFunc == nil {
+		panic("IAMServiceMock.ListUserAccountsFunc: method is nil but IAMService.ListUserAccounts was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockListUserAccounts.Lock()
+	mock.calls.ListUserAccounts = append(mock.calls.ListUserAccounts, callInfo)
+	mock.lockListUserAccounts.Unlock()
+	return mock.ListUserAccountsFunc()
+}
+
+// ListUserAccountsCalls gets all the calls that were made to ListUserAccounts.
+// Check the length with:
+//
+//	len(mockedIAMService.ListUserAccountsCalls())
+func (mock *IAMServiceMock) ListUserAccountsCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockListUserAccounts.RLock()
+	calls = mock.calls.ListUserAccounts
+	mock.lockListUserAccounts.RUnlock()
+	return calls
+}
+
+// Shutdown calls ShutdownFunc.
+func (mock *IAMServiceMock) Shutdown() error {
+	if mock.ShutdownFunc == nil {
+		panic("IAMServiceMock.ShutdownFunc: method is nil but IAMService.Shutdown was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockShutdown.Lock()
+	mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo)
+	mock.lockShutdown.Unlock()
+	return mock.ShutdownFunc()
+}
+
+// ShutdownCalls gets all the calls that were made to Shutdown.
+// Check the length with:
+//
+//	len(mockedIAMService.ShutdownCalls())
+func (mock *IAMServiceMock) ShutdownCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockShutdown.RLock()
+	calls = mock.calls.Shutdown
+	mock.lockShutdown.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -0,0 +1,61 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3log"
+)
+
+func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
+		path := ctx.Path()
+		pathParts := strings.Split(path, "/")
+		bucket := pathParts[1]
+		if path == "/" && ctx.Method() == http.MethodGet {
+			return ctx.Next()
+		}
+		if ctx.Method() == http.MethodPatch {
+			return ctx.Next()
+		}
+		if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
+			if err := auth.IsAdmin(acct, isRoot); err != nil {
+				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
+			}
+			return ctx.Next()
+		}
+		//TODO: provide correct action names for the logger, after implementing DetectAction middleware
+		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
+		parsedAcl, err := auth.ParseACL(data)
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
+		ctx.Locals("parsedAcl", parsedAcl)
+		return ctx.Next()
+	}
+}

s3api/middlewares/authentication.go

@@ -0,0 +1,186 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+const (
+	iso8601Format = "20060102T150405Z"
+)
+
+type RootUserConfig struct {
+	Access string
+	Secret string
+}
+
+func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
+	acct := accounts{root: root, iam: iam}
+
+	return func(ctx *fiber.Ctx) error {
+		ctx.Locals("region", region)
+		ctx.Locals("startTime", time.Now())
+		authorization := ctx.Get("Authorization")
+		if authorization == "" {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
+		}
+
+		authData, err := utils.ParseAuthorization(authorization)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		if authData.Algorithm != "AWS4-HMAC-SHA256" {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
+		}
+
+		if authData.Region != region {
+			return sendResponse(ctx, s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
+				HTTPStatusCode: http.StatusForbidden,
+			}, logger)
+		}
+
+		ctx.Locals("isRoot", authData.Access == root.Access)
+
+		account, err := acct.getAccount(authData.Access)
+		if err == auth.ErrNoSuchUser {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
+		}
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+		ctx.Locals("account", account)
+
+		// Check X-Amz-Date header
+		date := ctx.Get("X-Amz-Date")
+		if date == "" {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
+		}
+
+		// Parse the date and check the date validity
+		tdate, err := time.Parse(iso8601Format, date)
+		if err != nil {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
+		}
+
+		if date[:8] != authData.Date {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
+		}
+
+		// Validate the dates difference
+		err = validateDate(tdate)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		if utils.IsBigDataAction(ctx) {
+			// for streaming PUT actions, authorization is deferred
+			// until end of stream due to need to get length and
+			// checksum of the stream to validate authorization
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
+			})
+			return ctx.Next()
+		}
+
+		hashPayload := ctx.Get("X-Amz-Content-Sha256")
+		if !utils.IsSpecialPayload(hashPayload) {
+			// Calculate the hash of the request payload
+			hashedPayload := sha256.Sum256(ctx.Body())
+			hexPayload := hex.EncodeToString(hashedPayload[:])
+
+			// Compare the calculated hash with the hash provided
+			if hashPayload != hexPayload {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
+			}
+		}
+
+		var contentLength int64
+		contentLengthStr := ctx.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+			if err != nil {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
+			}
+		}
+
+		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		return ctx.Next()
+	}
+}
+
+type accounts struct {
+	root RootUserConfig
+	iam  auth.IAMService
+}
+
+func (a accounts) getAccount(access string) (auth.Account, error) {
+	if access == a.root.Access {
+		return auth.Account{
+			Access: a.root.Access,
+			Secret: a.root.Secret,
+			Role:   "admin",
+		}, nil
+	}
+
+	return a.iam.GetUserAccount(access)
+}
+
+func validateDate(date time.Time) error {
+	now := time.Now().UTC()
+	diff := date.Unix() - now.Unix()
+
+	// Checks the dates difference to be less than a minute
+	if diff > 60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+	if diff < -60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+
+	return nil
+}
+
+func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
+	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+}

s3api/middlewares/body-reader.go

@@ -0,0 +1,31 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
+	r, ok := ctx.Locals("body-reader").(io.Reader)
+	if !ok {
+		r = ctx.Request().BodyStream()
+	}
+
+	r = wr(r)
+	ctx.Locals("body-reader", r)
+}

s3api/middlewares/logger.go

@@ -0,0 +1,44 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func RequestLogger(isDebug bool) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		ctx.Locals("isDebug", isDebug)
+		if isDebug {
+			log.Println("Request headers: ")
+			ctx.Request().Header.VisitAll(func(key, val []byte) {
+				log.Printf("%s: %s", key, val)
+			})
+
+			if ctx.Request().URI().QueryArgs().Len() != 0 {
+				fmt.Println()
+				log.Println("Request query arguments: ")
+				ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
+					log.Printf("%s: %s", key, val)
+				})
+			}
+		}
+
+		return ctx.Next()
+	}
+}

s3api/middlewares/md5.go

@@ -0,0 +1,52 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"crypto/md5"
+	"io"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		incomingSum := ctx.Get("Content-Md5")
+		if incomingSum == "" {
+			return ctx.Next()
+		}
+
+		if utils.IsBigDataAction(ctx) {
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				r, _ = utils.NewHashReader(r, incomingSum, utils.HashTypeMd5)
+				return r
+			})
+			return ctx.Next()
+		}
+
+		sum := md5.Sum(ctx.Body())
+		calculatedSum := utils.Md5SumString(sum[:])
+
+		if incomingSum != calculatedSum {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
+		}
+
+		return ctx.Next()
+	}
+}

s3api/middlewares/url-decoder.go

@@ -0,0 +1,36 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/url"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		reqURL := ctx.Request().URI().String()
+		decoded, err := url.Parse(reqURL)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+		}
+		ctx.Path(decoded.Path)
+		return ctx.Next()
+	}
+}

s3api/router.go

@@ -0,0 +1,103 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3event"
+	"github.com/versity/versitygw/s3log"
+)
+
+type S3ApiRouter struct {
+	WithAdmSrv bool
+}
+
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
+	s3ApiController := controllers.New(be, iam, logger, evs)
+
+	if sa.WithAdmSrv {
+		adminController := controllers.NewAdminController(iam, be)
+
+		// CreateUser admin api
+		app.Patch("/create-user", adminController.CreateUser)
+
+		// DeleteUsers admin api
+		app.Patch("/delete-user", adminController.DeleteUser)
+
+		// ListUsers admin api
+		app.Patch("/list-users", adminController.ListUsers)
+
+		// ChangeBucketOwner admin api
+		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+		// ListBucketsAndOwners admin api
+		app.Patch("/list-buckets", adminController.ListBuckets)
+	}
+
+	// ListBuckets action
+	app.Get("/", s3ApiController.ListBuckets)
+
+	// CreateBucket action
+	// PutBucketAcl action
+	app.Put("/:bucket", s3ApiController.PutBucketActions)
+
+	// DeleteBucket action
+	app.Delete("/:bucket", s3ApiController.DeleteBucket)
+
+	// HeadBucket
+	app.Head("/:bucket", s3ApiController.HeadBucket)
+
+	// GetBucketAcl action
+	// ListMultipartUploads action
+	// ListObjects action
+	// ListObjectsV2 action
+	app.Get("/:bucket", s3ApiController.ListActions)
+
+	// HeadObject action
+	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
+
+	// GetObjectAcl action
+	// GetObject action
+	// ListObjectParts action
+	// GetObjectTagging action
+	// ListParts action
+	// GetObjectAttributes action
+	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
+
+	// DeleteObject action
+	// AbortMultipartUpload action
+	// DeleteObjectTagging action
+	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
+
+	// DeleteObjects action
+	app.Post("/:bucket", s3ApiController.DeleteObjects)
+
+	// CompleteMultipartUpload action
+	// CreateMultipartUpload
+	// RestoreObject action
+	// SelectObjectContent action
+	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
+
+	// CopyObject action
+	// PutObject action
+	// UploadPart action
+	// UploadPartCopy action
+	// PutObjectTagging action
+	// PutObjectAcl action
+	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
+}

s3api/router_test.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+)
+
+func TestS3ApiRouter_Init(t *testing.T) {
+	type args struct {
+		app *fiber.App
+		be  backend.Backend
+		iam auth.IAMService
+	}
+	tests := []struct {
+		name string
+		sa   *S3ApiRouter
+		args args
+	}{
+		{
+			name: "Initialize S3 api router",
+			sa:   &S3ApiRouter{},
+			args: args{
+				app: fiber.New(),
+				be:  backend.BackendUnsupported{},
+				iam: &auth.IAMServiceInternal{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
+		})
+	}
+}

s3api/server.go

@@ -0,0 +1,88 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"crypto/tls"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3event"
+	"github.com/versity/versitygw/s3log"
+)
+
+type S3ApiServer struct {
+	app     *fiber.App
+	backend backend.Backend
+	router  *S3ApiRouter
+	port    string
+	cert    *tls.Certificate
+	debug   bool
+}
+
+func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
+	server := &S3ApiServer{
+		app:     app,
+		backend: be,
+		router:  new(S3ApiRouter),
+		port:    port,
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
+
+	// Logging middlewares
+	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(l))
+	app.Use(middlewares.RequestLogger(server.debug))
+
+	// Authentication middlewares
+	app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
+	app.Use(middlewares.VerifyMD5Body(l))
+	app.Use(middlewares.AclParser(be, l))
+
+	server.router.Init(app, be, iam, l, evs)
+
+	return server, nil
+}
+
+// Option sets various options for New()
+type Option func(*S3ApiServer)
+
+// WithTLS sets TLS Credentials
+func WithTLS(cert tls.Certificate) Option {
+	return func(s *S3ApiServer) { s.cert = &cert }
+}
+
+// WithAdminServer runs admin endpoints with the gateway in the same network
+func WithAdminServer() Option {
+	return func(s *S3ApiServer) { s.router.WithAdmSrv = true }
+}
+
+// WithDebug sets debug output
+func WithDebug() Option {
+	return func(s *S3ApiServer) { s.debug = true }
+}
+
+func (sa *S3ApiServer) Serve() (err error) {
+	if sa.cert != nil {
+		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
+	}
+	return sa.app.Listen(sa.port)
+}

s3api/server_test.go

@@ -0,0 +1,114 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"crypto/tls"
+	"reflect"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+func TestNew(t *testing.T) {
+	type args struct {
+		app  *fiber.App
+		be   backend.Backend
+		port string
+		root middlewares.RootUserConfig
+	}
+
+	app := fiber.New()
+	be := backend.BackendUnsupported{}
+	router := S3ApiRouter{}
+	port := ":7070"
+
+	tests := []struct {
+		name            string
+		args            args
+		wantS3ApiServer *S3ApiServer
+		wantErr         bool
+	}{
+		{
+			name: "Create S3 api server",
+			args: args{
+				app:  app,
+				be:   be,
+				port: port,
+				root: middlewares.RootUserConfig{},
+			},
+			wantS3ApiServer: &S3ApiServer{
+				app:     app,
+				port:    port,
+				router:  &router,
+				backend: be,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
+				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotS3ApiServer, tt.wantS3ApiServer) {
+				t.Errorf("New() = %v, want %v", gotS3ApiServer, tt.wantS3ApiServer)
+			}
+		})
+	}
+}
+
+func TestS3ApiServer_Serve(t *testing.T) {
+	tests := []struct {
+		name    string
+		sa      *S3ApiServer
+		wantErr bool
+	}{
+		{
+			name:    "Serve-invalid-address",
+			wantErr: true,
+			sa: &S3ApiServer{
+				app:     fiber.New(),
+				backend: backend.BackendUnsupported{},
+				port:    "Invalid address",
+				router:  &S3ApiRouter{},
+			},
+		},
+		{
+			name:    "Serve-invalid-address-with-certificate",
+			wantErr: true,
+			sa: &S3ApiServer{
+				app:     fiber.New(),
+				backend: backend.BackendUnsupported{},
+				port:    "Invalid address",
+				router:  &S3ApiRouter{},
+				cert:    &tls.Certificate{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.sa.Serve(); (err != nil) != tt.wantErr {
+				t.Errorf("S3ApiServer.Serve() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

s3api/utils/auth-reader.go

@@ -0,0 +1,261 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go/logging"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	iso8601Format = "20060102T150405Z"
+	yyyymmdd      = "20060102"
+)
+
+// AuthReader is an io.Reader that validates the request authorization
+// once the underlying reader returns io.EOF.  This is needed for streaming
+// data requests where the data size and checksum are not known until
+// the data is completely read.
+type AuthReader struct {
+	ctx    *fiber.Ctx
+	auth   AuthData
+	secret string
+	size   int
+	r      *HashReader
+	debug  bool
+}
+
+// NewAuthReader initializes an io.Reader that will verify the request
+// v4 auth when the underlying reader returns io.EOF. This postpones the
+// authorization check until the reader is consumed. So it is important that
+// the consumer of this reader checks for the auth errors while reading.
+func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *AuthReader {
+	var hr *HashReader
+	hashPayload := ctx.Get("X-Amz-Content-Sha256")
+	if !IsSpecialPayload(hashPayload) {
+		hr, _ = NewHashReader(r, "", HashTypeSha256)
+	} else {
+		hr, _ = NewHashReader(r, "", HashTypeNone)
+	}
+
+	return &AuthReader{
+		ctx:    ctx,
+		r:      hr,
+		auth:   auth,
+		secret: secret,
+		debug:  debug,
+	}
+}
+
+// Read allows *AuthReader to be used as an io.Reader
+func (ar *AuthReader) Read(p []byte) (int, error) {
+	n, err := ar.r.Read(p)
+	ar.size += n
+
+	if errors.Is(err, io.EOF) {
+		verr := ar.validateSignature()
+		if verr != nil {
+			return n, verr
+		}
+	}
+
+	return n, err
+}
+
+func (ar *AuthReader) validateSignature() error {
+	date := ar.ctx.Get("X-Amz-Date")
+	if date == "" {
+		return s3err.GetAPIError(s3err.ErrMissingDateHeader)
+	}
+
+	hashPayload := ar.ctx.Get("X-Amz-Content-Sha256")
+	if !IsSpecialPayload(hashPayload) {
+		hexPayload := ar.r.Sum()
+
+		// Compare the calculated hash with the hash provided
+		if hashPayload != hexPayload {
+			return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+		}
+	}
+
+	// Parse the date and check the date validity
+	tdate, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrMalformedDate)
+	}
+
+	return CheckValidSignature(ar.ctx, ar.auth, ar.secret, hashPayload, tdate, int64(ar.size), ar.debug)
+}
+
+const (
+	service = "s3"
+)
+
+// CheckValidSignature validates the ctx v4 auth signature
+func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64, debug bool) error {
+	signedHdrs := strings.Split(auth.SignedHeaders, ";")
+
+	// Create a new http request instance from fasthttp request
+	req, err := createHttpRequestFromCtx(ctx, signedHdrs, contentLen)
+	if err != nil {
+		return fmt.Errorf("create http request from context: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{
+		AccessKeyID:     auth.Access,
+		SecretAccessKey: secret,
+	}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
+		options.DisableURIPathEscaping = true
+		if debug {
+			options.LogSigning = true
+			options.Logger = logging.NewStandardLogger(os.Stderr)
+		}
+	})
+	if signErr != nil {
+		return fmt.Errorf("sign generated http request: %w", err)
+	}
+
+	genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
+	if err != nil {
+		return err
+	}
+
+	if auth.Signature != genAuth.Signature {
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+
+	return nil
+}
+
+// AuthData is the parsed authorization data from the header
+type AuthData struct {
+	Algorithm     string
+	Access        string
+	Region        string
+	SignedHeaders string
+	Signature     string
+	Date          string
+}
+
+// ParseAuthorization returns the parsed fields for the aws v4 auth header
+// example authorization string from aws docs:
+// Authorization: AWS4-HMAC-SHA256
+// Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,
+// SignedHeaders=host;range;x-amz-date,
+// Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
+func ParseAuthorization(authorization string) (AuthData, error) {
+	a := AuthData{}
+
+	// authorization must start with:
+	// Authorization: <ALGORITHM>
+	// followed by key=value pairs separated by ","
+	authParts := strings.Fields(authorization)
+	for i, el := range authParts {
+		authParts[i] = strings.TrimSpace(el)
+	}
+
+	if len(authParts) < 3 {
+		return a, s3err.GetAPIError(s3err.ErrMissingFields)
+	}
+
+	algo := authParts[0]
+
+	kvData := strings.Join(authParts[1:], "")
+	kvPairs := strings.Split(kvData, ",")
+	// we are expecting at least Credential, SignedHeaders, and Signature
+	// key value pairs here
+	if len(kvPairs) < 3 {
+		return a, s3err.GetAPIError(s3err.ErrMissingFields)
+	}
+
+	var access, region, signedHeaders, signature, date string
+
+	for _, kv := range kvPairs {
+		keyValue := strings.Split(kv, "=")
+		if len(keyValue) != 2 {
+			switch {
+			case strings.HasPrefix(kv, "Credential"):
+				return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+			case strings.HasPrefix(kv, "SignedHeaders"):
+				return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+			}
+			return a, s3err.GetAPIError(s3err.ErrMissingFields)
+		}
+		key := strings.TrimSpace(keyValue[0])
+		value := strings.TrimSpace(keyValue[1])
+
+		switch key {
+		case "Credential":
+			creds := strings.Split(value, "/")
+			if len(creds) != 5 {
+				return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+			}
+			if creds[3] != "s3" {
+				return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
+			}
+			if creds[4] != "aws4_request" {
+				return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
+			}
+			_, err := time.Parse(yyyymmdd, creds[1])
+			if err != nil {
+				return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
+			}
+			access = creds[0]
+			date = creds[1]
+			region = creds[2]
+		case "SignedHeaders":
+			signedHeaders = value
+		case "Signature":
+			signature = value
+		}
+	}
+
+	return AuthData{
+		Algorithm:     algo,
+		Access:        access,
+		Region:        region,
+		SignedHeaders: signedHeaders,
+		Signature:     signature,
+		Date:          date,
+	}, nil
+}
+
+var (
+	specialValues = map[string]bool{
+		"UNSIGNED-PAYLOAD":                                 true,
+		"STREAMING-UNSIGNED-PAYLOAD-TRAILER":               true,
+		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD":               true,
+		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER":       true,
+		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD":         true,
+		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
+	}
+)
+
+// IsSpecialPayload checks for streaming/unsigned authorization types
+func IsSpecialPayload(str string) bool {
+	return specialValues[str]
+}

s3api/utils/csum-reader.go

@@ -0,0 +1,130 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"hash"
+	"io"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+// HashType identifies the checksum algorithm to be used
+type HashType string
+
+const (
+	// HashTypeMd5 generates MD5 checksum for the data stream
+	HashTypeMd5 = "md5"
+	// HashTypeSha256 generates SHA256 checksum for the data stream
+	HashTypeSha256 = "sha256"
+	// HashTypeNone is a no-op checksum for the data stream
+	HashTypeNone = "none"
+)
+
+// HashReader is an io.Reader that calculates the checksum
+// as the data is read
+type HashReader struct {
+	hashType HashType
+	hash     hash.Hash
+	r        io.Reader
+	sum      string
+}
+
+var (
+	errInvalidHashType = errors.New("unsupported or invalid checksum type")
+)
+
+// NewHashReader intializes an io.Reader from an underlying io.Reader that
+// calculates the checksum while the reader is being read from. If the
+// sum provided is not "", the reader will return an error when the underlying
+// reader returns io.EOF if the checksum does not match the provided expected
+// checksum.  If the provided sum is "", then the Sum() method can still
+// be used to get the current checksum for the data read so far.
+func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
+	var hash hash.Hash
+	switch ht {
+	case HashTypeMd5:
+		hash = md5.New()
+	case HashTypeSha256:
+		hash = sha256.New()
+	case HashTypeNone:
+		hash = noop{}
+	default:
+		return nil, errInvalidHashType
+	}
+
+	return &HashReader{
+		hash:     hash,
+		r:        r,
+		sum:      expectedSum,
+		hashType: ht,
+	}, nil
+}
+
+// Read allows *HashReader to be used as an io.Reader
+func (hr *HashReader) Read(p []byte) (int, error) {
+	n, readerr := hr.r.Read(p)
+	_, err := hr.hash.Write(p[:n])
+	if err != nil {
+		return n, err
+	}
+	if errors.Is(readerr, io.EOF) && hr.sum != "" {
+		switch hr.hashType {
+		case HashTypeMd5:
+			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
+			}
+		case HashTypeSha256:
+			sum := hex.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+			}
+		default:
+			return n, errInvalidHashType
+		}
+	}
+	return n, readerr
+}
+
+// Sum returns the checksum hash of the data read so far
+func (hr *HashReader) Sum() string {
+	switch hr.hashType {
+	case HashTypeMd5:
+		return Md5SumString(hr.hash.Sum(nil))
+	case HashTypeSha256:
+		return hex.EncodeToString(hr.hash.Sum(nil))
+	default:
+		return ""
+	}
+}
+
+// Md5SumString converts the hash bytes to the string checksum value
+func Md5SumString(b []byte) string {
+	return base64.StdEncoding.EncodeToString(b)
+}
+
+type noop struct{}
+
+func (n noop) Write(p []byte) (int, error) { return 0, nil }
+func (n noop) Sum(b []byte) []byte         { return []byte{} }
+func (n noop) Reset()                      {}
+func (n noop) Size() int                   { return 0 }
+func (n noop) BlockSize() int              { return 1 }

s3api/utils/logger.go

@@ -0,0 +1,55 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func LogCtxDetails(ctx *fiber.Ctx, respBody []byte) {
+	isDebug, ok := ctx.Locals("isDebug").(bool)
+	_, notLogReqBody := ctx.Locals("logReqBody").(bool)
+	_, notLogResBody := ctx.Locals("logResBody").(bool)
+	if isDebug && ok {
+		// Log request body
+		if !notLogReqBody {
+			fmt.Println()
+			log.Printf("Request Body: %s", ctx.Request().Body())
+		}
+
+		// Log path parameters
+		fmt.Println()
+		log.Println("Path parameters: ")
+		for key, val := range ctx.AllParams() {
+			log.Printf("%s: %s", key, val)
+		}
+
+		// Log response headers
+		fmt.Println()
+		log.Println("Response Headers: ")
+		ctx.Response().Header.VisitAll(func(key, val []byte) {
+			log.Printf("%s: %s", key, val)
+		})
+
+		// Log response body
+		if !notLogResBody && len(respBody) > 0 {
+			fmt.Println()
+			log.Printf("Response body %s", ctx.Response().Body())
+		}
+	}
+}

s3api/utils/utils.go

@@ -0,0 +1,151 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	bucketNameRegexp   = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]+[a-z0-9]$`)
+	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
+)
+
+func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
+	metadata = make(map[string]string)
+	headers.DisableNormalizing()
+	headers.VisitAllInOrder(func(key, value []byte) {
+		hKey := string(key)
+		if strings.HasPrefix(strings.ToLower(hKey), "x-amz-meta-") {
+			trimmedKey := hKey[11:]
+			headerValue := string(value)
+			metadata[trimmedKey] = headerValue
+		}
+	})
+	headers.EnableNormalizing()
+
+	return
+}
+
+func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
+	req := ctx.Request()
+	var body io.Reader
+	if IsBigDataAction(ctx) {
+		body = req.BodyStream()
+	} else {
+		body = bytes.NewReader(req.Body())
+	}
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
+	if err != nil {
+		return nil, errors.New("error in creating an http request")
+	}
+
+	// Set the request headers
+	req.Header.VisitAll(func(key, value []byte) {
+		keyStr := string(key)
+		if includeHeader(keyStr, signedHdrs) {
+			httpReq.Header.Add(keyStr, string(value))
+		}
+	})
+
+	// Check if Content-Length in signed headers
+	// If content length is non 0, then the header will be included
+	if !includeHeader("Content-Length", signedHdrs) {
+		httpReq.ContentLength = 0
+	} else {
+		httpReq.ContentLength = contentLength
+	}
+
+	// Set the Host header
+	httpReq.Host = string(req.Header.Host())
+
+	return httpReq, nil
+}
+
+func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
+	ctx.Response().Header.DisableNormalizing()
+	for key, val := range meta {
+		ctx.Response().Header.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
+	}
+	ctx.Response().Header.EnableNormalizing()
+}
+
+func ParseUint(str string) (int32, error) {
+	if str == "" {
+		return 1000, nil
+	}
+	num, err := strconv.ParseUint(str, 10, 16)
+	if err != nil {
+		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+	}
+	return int32(num), nil
+}
+
+type CustomHeader struct {
+	Key   string
+	Value string
+}
+
+func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
+	for _, header := range headers {
+		ctx.Set(header.Key, header.Value)
+	}
+}
+
+func IsValidBucketName(bucket string) bool {
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	// Checks to contain only digits, lowercase letters, dot, hyphen.
+	// Checks to start and end with only digits and lowercase letters.
+	if !bucketNameRegexp.MatchString(bucket) {
+		return false
+	}
+	// Checks not to be a valid IP address
+	if bucketNameIpRegexp.MatchString(bucket) {
+		return false
+	}
+	return true
+}
+
+func includeHeader(hdr string, signedHdrs []string) bool {
+	for _, shdr := range signedHdrs {
+		if strings.EqualFold(hdr, shdr) {
+			return true
+		}
+	}
+	return false
+}
+
+func IsBigDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}

s3api/utils/utils_test.go

@@ -0,0 +1,263 @@
+package utils
+
+import (
+	"bytes"
+	"net/http"
+	"reflect"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
+)
+
+func TestCreateHttpRequestFromCtx(t *testing.T) {
+	type args struct {
+		ctx *fiber.Ctx
+	}
+
+	app := fiber.New()
+
+	// Expected output, Case 1
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
+	req := ctx.Request()
+	request, _ := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
+
+	// Case 2
+	ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
+	req2 := ctx2.Request()
+	req2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
+
+	request2, _ := http.NewRequest(string(req2.Header.Method()), req2.URI().String(), bytes.NewReader(req2.Body()))
+	request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
+
+	tests := []struct {
+		name    string
+		args    args
+		want    *http.Request
+		wantErr bool
+	}{
+		{
+			name: "Success-response",
+			args: args{
+				ctx: ctx,
+			},
+			want:    request,
+			wantErr: false,
+		},
+		{
+			name: "Success-response-With-Headers",
+			args: args{
+				ctx: ctx2,
+			},
+			want:    request2,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if !reflect.DeepEqual(got.Header, tt.want.Header) {
+				t.Errorf("CreateHttpRequestFromCtx() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetUserMetaData(t *testing.T) {
+	type args struct {
+		headers *fasthttp.RequestHeader
+	}
+
+	app := fiber.New()
+
+	// Case 1
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
+	req := ctx.Request()
+
+	tests := []struct {
+		name         string
+		args         args
+		wantMetadata map[string]string
+	}{
+		{
+			name: "Success-empty-response",
+			args: args{
+				headers: &req.Header,
+			},
+			wantMetadata: map[string]string{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotMetadata := GetUserMetaData(tt.args.headers); !reflect.DeepEqual(gotMetadata, tt.wantMetadata) {
+				t.Errorf("GetUserMetaData() = %v, want %v", gotMetadata, tt.wantMetadata)
+			}
+		})
+	}
+}
+
+func Test_includeHeader(t *testing.T) {
+	type args struct {
+		hdr        string
+		signedHdrs []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "include-header-falsy-case",
+			args: args{
+				hdr:        "Content-Type",
+				signedHdrs: []string{"X-Amz-Acl", "Content-Encoding"},
+			},
+			want: false,
+		},
+		{
+			name: "include-header-falsy-case",
+			args: args{
+				hdr:        "Content-Type",
+				signedHdrs: []string{"X-Amz-Acl", "Content-Type"},
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := includeHeader(tt.args.hdr, tt.args.signedHdrs); got != tt.want {
+				t.Errorf("includeHeader() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsValidBucketName(t *testing.T) {
+	type args struct {
+		bucket string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "IsValidBucketName-short-name",
+			args: args{
+				bucket: "a",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-start-with-hyphen",
+			args: args{
+				bucket: "-bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-start-with-dot",
+			args: args{
+				bucket: ".bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-contain-invalid-character",
+			args: args{
+				bucket: "my@bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-end-with-hyphen",
+			args: args{
+				bucket: "bucket-",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-end-with-dot",
+			args: args{
+				bucket: "bucket.",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-valid-bucket-name",
+			args: args{
+				bucket: "my-bucket",
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidBucketName(tt.args.bucket); got != tt.want {
+				t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseUint(t *testing.T) {
+	type args struct {
+		str string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    int32
+		wantErr bool
+	}{
+		{
+			name: "Parse-uint-empty-string",
+			args: args{
+				str: "",
+			},
+			want:    1000,
+			wantErr: false,
+		},
+		{
+			name: "Parse-uint-invalid-number-string",
+			args: args{
+				str: "bla",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-invalid-negative-number",
+			args: args{
+				str: "-5",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-success",
+			args: args{
+				str: "23",
+			},
+			want:    23,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseUint(tt.args.str)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseMaxKeys() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ParseMaxKeys() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -0,0 +1,446 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3err
+
+import (
+	"bytes"
+	"encoding/xml"
+	"net/http"
+)
+
+// APIError structure
+type APIError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// APIErrorResponse - error response format
+type APIErrorResponse struct {
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string
+	Message    string
+	Key        string `xml:"Key,omitempty" json:"Key,omitempty"`
+	BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
+	Resource   string
+	Region     string `xml:"Region,omitempty" json:"Region,omitempty"`
+	RequestID  string `xml:"RequestId" json:"RequestId"`
+	HostID     string `xml:"HostId" json:"HostId"`
+}
+
+func (A APIError) Error() string {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	_ = e.Encode(A)
+	return bytesBuffer.String()
+}
+
+// ErrorCode type of error status.
+type ErrorCode int
+
+// Error codes, see full list at http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+const (
+	ErrNone ErrorCode = iota
+	ErrAccessDenied
+	ErrMethodNotAllowed
+	ErrBucketNotEmpty
+	ErrBucketAlreadyExists
+	ErrBucketAlreadyOwnedByYou
+	ErrNoSuchBucket
+	ErrNoSuchKey
+	ErrNoSuchUpload
+	ErrInvalidBucketName
+	ErrInvalidDigest
+	ErrInvalidMaxKeys
+	ErrInvalidMaxUploads
+	ErrInvalidMaxParts
+	ErrInvalidPartNumberMarker
+	ErrInvalidPart
+	ErrInternalError
+	ErrInvalidCopyDest
+	ErrInvalidCopySource
+	ErrInvalidTag
+	ErrAuthHeaderEmpty
+	ErrSignatureVersionNotSupported
+	ErrMalformedPOSTRequest
+	ErrPOSTFileRequired
+	ErrPostPolicyConditionInvalidFormat
+	ErrEntityTooSmall
+	ErrEntityTooLarge
+	ErrMissingFields
+	ErrMissingCredTag
+	ErrCredMalformed
+	ErrMalformedXML
+	ErrMalformedDate
+	ErrMalformedPresignedDate
+	ErrMalformedCredentialDate
+	ErrMissingSignHeadersTag
+	ErrMissingSignTag
+	ErrUnsignedHeaders
+	ErrInvalidQueryParams
+	ErrInvalidQuerySignatureAlgo
+	ErrExpiredPresignRequest
+	ErrMalformedExpires
+	ErrNegativeExpires
+	ErrMaximumExpires
+	ErrSignatureDoesNotMatch
+	ErrSignatureDateDoesNotMatch
+	ErrSignatureTerminationStr
+	ErrSignatureIncorrService
+	ErrContentSHA256Mismatch
+	ErrInvalidAccessKeyID
+	ErrRequestNotReadyYet
+	ErrMissingDateHeader
+	ErrInvalidRequest
+	ErrAuthNotSetup
+	ErrNotImplemented
+	ErrPreconditionFailed
+	ErrInvalidObjectState
+	ErrInvalidRange
+	ErrInvalidURI
+
+	// Non-AWS errors
+	ErrExistingObjectIsDirectory
+	ErrObjectParentIsFile
+	ErrDirectoryObjectContainsData
+)
+
+var errorCodeResponse = map[ErrorCode]APIError{
+	ErrAccessDenied: {
+		Code:           "AccessDenied",
+		Description:    "Access Denied.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMethodNotAllowed: {
+		Code:           "MethodNotAllowed",
+		Description:    "The specified method is not allowed against this resource.",
+		HTTPStatusCode: http.StatusMethodNotAllowed,
+	},
+	ErrBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrBucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrBucketAlreadyOwnedByYou: {
+		Code:           "BucketAlreadyOwnedByYou",
+		Description:    "Your previous request to create the named bucket succeeded and you already own it.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrInvalidBucketName: {
+		Code:           "InvalidBucketName",
+		Description:    "The specified bucket is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidDigest: {
+		Code:           "InvalidDigest",
+		Description:    "The Content-Md5 you specified is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxUploads: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxKeys: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxParts: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartNumberMarker: {
+		Code:           "InvalidArgument",
+		Description:    "Argument partNumberMarker must be an integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNoSuchBucket: {
+		Code:           "NoSuchBucket",
+		Description:    "The specified bucket does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchKey: {
+		Code:           "NoSuchKey",
+		Description:    "The specified key does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchUpload: {
+		Code:           "NoSuchUpload",
+		Description:    "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrInternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrInvalidPart: {
+		Code:           "InvalidPart",
+		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCopyDest: {
+		Code:           "InvalidRequest",
+		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCopySource: {
+		Code:           "InvalidArgument",
+		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidTag: {
+		Code:           "InvalidArgument",
+		Description:    "The Tag value you have provided is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedXML: {
+		Code:           "MalformedXML",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthHeaderEmpty: {
+		Code:           "InvalidArgument",
+		Description:    "Authorization header is invalid -- one and only one ' ' (space) required.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSignatureVersionNotSupported: {
+		Code:           "InvalidRequest",
+		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPOSTRequest: {
+		Code:           "MalformedPOSTRequest",
+		Description:    "The body of your POST request is not well-formed multipart/form-data.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPOSTFileRequired: {
+		Code:           "InvalidArgument",
+		Description:    "POST requires exactly one file upload per request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPostPolicyConditionInvalidFormat: {
+		Code:           "PostPolicyInvalidKeyName",
+		Description:    "Invalid according to Policy: Policy Condition failed",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrEntityTooSmall: {
+		Code:           "EntityTooSmall",
+		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrEntityTooLarge: {
+		Code:           "EntityTooLarge",
+		Description:    "Your proposed upload exceeds the maximum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingFields: {
+		Code:           "MissingFields",
+		Description:    "Missing fields in request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingCredTag: {
+		Code:           "InvalidRequest",
+		Description:    "Missing Credential field for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrCredMalformed: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedDate: {
+		Code:           "MalformedDate",
+		Description:    "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPresignedDate: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignHeadersTag: {
+		Code:           "InvalidArgument",
+		Description:    "Signature header missing SignedHeaders field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignTag: {
+		Code:           "AccessDenied",
+		Description:    "Signature header missing Signature field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrUnsignedHeaders: {
+		Code:           "AccessDenied",
+		Description:    "There were headers present in the request which were not signed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQueryParams: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQuerySignatureAlgo: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExpiredPresignRequest: {
+		Code:           "AccessDenied",
+		Description:    "Request has expired",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMalformedExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires should be a number",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNegativeExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be non-negative",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMaximumExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidAccessKeyID: {
+		Code:           "InvalidAccessKeyId",
+		Description:    "The access key ID you provided does not exist in our records.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrRequestNotReadyYet: {
+		Code:           "AccessDenied",
+		Description:    "Request is not valid yet",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureDateDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureTerminationStr: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureIncorrService: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped to correct service: s3",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrContentSHA256Mismatch: {
+		Code:           "XAmzContentSHA256Mismatch",
+		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingDateHeader: {
+		Code:           "AccessDenied",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequest: {
+		Code:           "InvalidRequest",
+		Description:    "Invalid Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthNotSetup: {
+		Code:           "InvalidRequest",
+		Description:    "Signed request requires setting up SeaweedFS S3 authentication",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNotImplemented: {
+		Code:           "NotImplemented",
+		Description:    "A header you provided implies functionality that is not implemented",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrPreconditionFailed: {
+		Code:           "PreconditionFailed",
+		Description:    "At least one of the pre-conditions you specified did not hold",
+		HTTPStatusCode: http.StatusPreconditionFailed,
+	},
+	ErrInvalidObjectState: {
+		Code:           "InvalidObjectState",
+		Description:    "The operation is not valid for the current state of the object",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidRange: {
+		Code:           "InvalidRange",
+		Description:    "The requested range is not valid for the request. Try another range.",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
+	ErrInvalidURI: {
+		Code:           "InvalidURI",
+		Description:    "The specified URI couldn't be parsed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExistingObjectIsDirectory: {
+		Code:           "ExistingObjectIsDirectory",
+		Description:    "Existing Object is a directory.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrObjectParentIsFile: {
+		Code:           "ObjectParentIsFile",
+		Description:    "Object parent already exists as a file.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrDirectoryObjectContainsData: {
+		Code:           "DirectoryObjectContainsData",
+		Description:    "Directory object contains data payload.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+}
+
+// GetAPIError provides API Error for input API error code.
+func GetAPIError(code ErrorCode) APIError {
+	return errorCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func GetAPIErrorResponse(err APIError, resource, requestID, hostID string) []byte {
+	return encodeResponse(APIErrorResponse{
+		Code:       err.Code,
+		Message:    err.Description,
+		BucketName: "",
+		Key:        "",
+		Resource:   resource,
+		Region:     "",
+		RequestID:  requestID,
+		HostID:     hostID,
+	})
+}
+
+// Encodes the response headers into XML format.
+func encodeResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}

s3event/event.go

@@ -0,0 +1,130 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"fmt"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type S3EventSender interface {
+	SendEvent(ctx *fiber.Ctx, meta EventMeta)
+}
+
+type EventMeta struct {
+	BucketOwner string
+	EventName   EventType
+	ObjectSize  int64
+	ObjectETag  *string
+	VersionId   *string
+}
+
+type EventFields struct {
+	Records []EventSchema
+}
+
+type EventType string
+
+const (
+	EventObjectPut               EventType = "s3:ObjectCreated:Put"
+	EventObjectCopy              EventType = "s3:ObjectCreated:Copy"
+	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
+	EventObjectDelete            EventType = "s3:ObjectRemoved:Delete"
+	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
+	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
+	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
+	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
+	// Not supported
+	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
+)
+
+type EventSchema struct {
+	EventVersion      string                `json:"eventVersion"`
+	EventSource       string                `json:"eventSource"`
+	AwsRegion         string                `json:"awsRegion"`
+	EventTime         string                `json:"eventTime"`
+	EventName         EventType             `json:"eventName"`
+	UserIdentity      EventUserIdentity     `json:"userIdentity"`
+	RequestParameters EventRequestParams    `json:"requestParameters"`
+	ResponseElements  EventResponseElements `json:"responseElements"`
+	S3                EventS3Data           `json:"s3"`
+	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
+}
+
+type EventUserIdentity struct {
+	PrincipalId string `json:"PrincipalId"`
+}
+
+type EventRequestParams struct {
+	SourceIPAddress string `json:"sourceIPAddress"`
+}
+
+type EventResponseElements struct {
+	RequestId string `json:"x-amz-request-id"`
+	HostId    string `json:"x-amz-id-2"`
+}
+
+type EventS3Data struct {
+	S3SchemaVersion string            `json:"s3SchemaVersion"`
+	ConfigurationId string            `json:"configurationId"`
+	Bucket          EventS3BucketData `json:"bucket"`
+	Object          EventObjectData   `json:"object"`
+}
+
+type EventGlacierData struct {
+	RestoreEventData EventRestoreData `json:"restoreEventData"`
+}
+
+type EventRestoreData struct {
+	LifecycleRestorationExpiryTime string `json:"lifecycleRestorationExpiryTime"`
+	LifecycleRestoreStorageClass   string `json:"lifecycleRestoreStorageClass"`
+}
+
+type EventS3BucketData struct {
+	Name          string            `json:"name"`
+	OwnerIdentity EventUserIdentity `json:"ownerIdentity"`
+	Arn           string            `json:"arn"`
+}
+
+type EventObjectData struct {
+	Key       string  `json:"key"`
+	Size      int64   `json:"size"`
+	ETag      *string `json:"eTag"`
+	VersionId *string `json:"versionId"`
+	Sequencer string  `json:"sequencer"`
+}
+
+type EventConfig struct {
+	KafkaURL      string
+	KafkaTopic    string
+	KafkaTopicKey string
+	NatsURL       string
+	NatsTopic     string
+}
+
+func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
+	if cfg.KafkaURL != "" && cfg.NatsURL != "" {
+		return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
+	}
+	if cfg.NatsURL != "" {
+		return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
+	}
+	if cfg.KafkaURL != "" {
+		return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
+	}
+	return nil, nil
+}

s3event/kafka.go

@@ -0,0 +1,153 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/segmentio/kafka-go"
+)
+
+var sequencer = 0
+
+type Kafka struct {
+	key    string
+	writer *kafka.Writer
+	mu     sync.Mutex
+}
+
+func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
+	if topic == "" {
+		return nil, fmt.Errorf("kafka message topic should be specified")
+	}
+
+	w := kafka.NewWriter(kafka.WriterConfig{
+		Brokers:      []string{url},
+		Topic:        topic,
+		Balancer:     &kafka.LeastBytes{},
+		BatchTimeout: 5 * time.Millisecond,
+	})
+
+	msg := map[string]string{
+		"Service": "S3",
+		"Event":   "s3:TestEvent",
+		"Time":    time.Now().Format(time.RFC3339),
+		"Bucket":  "Test-Bucket",
+	}
+
+	msgJSON, err := json.Marshal(msg)
+	if err != nil {
+		return nil, err
+	}
+
+	message := kafka.Message{
+		Key:   []byte(key),
+		Value: msgJSON,
+	}
+
+	ctx := context.Background()
+
+	err = w.WriteMessages(ctx, message)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Kafka{
+		key:    key,
+		writer: w,
+	}, nil
+}
+
+func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+
+	schema := EventSchema{
+		EventVersion: "2.2",
+		EventSource:  "aws:s3",
+		AwsRegion:    ctx.Locals("region").(string),
+		EventTime:    time.Now().Format(time.RFC3339),
+		EventName:    meta.EventName,
+		UserIdentity: EventUserIdentity{
+			PrincipalId: ctx.Locals("access").(string),
+		},
+		RequestParameters: EventRequestParams{
+			SourceIPAddress: ctx.IP(),
+		},
+		ResponseElements: EventResponseElements{
+			RequestId: ctx.Get("X-Amz-Request-Id"),
+			HostId:    ctx.Get("X-Amx-Id-2"),
+		},
+		S3: EventS3Data{
+			S3SchemaVersion: "1.0",
+			// This field will come up after implementing per bucket notifications
+			ConfigurationId: "kafka-global",
+			Bucket: EventS3BucketData{
+				Name: bucket,
+				OwnerIdentity: EventUserIdentity{
+					PrincipalId: ctx.Locals("access").(string),
+				},
+				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+			},
+			Object: EventObjectData{
+				Key:       object,
+				Size:      meta.ObjectSize,
+				ETag:      meta.ObjectETag,
+				VersionId: meta.VersionId,
+				Sequencer: genSequencer(),
+			},
+		},
+		GlacierEventData: EventGlacierData{
+			// Not supported
+			RestoreEventData: EventRestoreData{},
+		},
+	}
+
+	ks.send([]EventSchema{schema})
+}
+
+func (ks *Kafka) send(evnt []EventSchema) {
+	msg, err := json.Marshal(evnt)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
+		return
+	}
+
+	message := kafka.Message{
+		Key:   []byte(ks.key),
+		Value: msg,
+	}
+
+	ctx := context.Background()
+	err = ks.writer.WriteMessages(ctx, message)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
+	}
+}
+
+func genSequencer() string {
+	sequencer = sequencer + 1
+	return fmt.Sprintf("%X", sequencer)
+}

s3event/nats.go

@@ -0,0 +1,112 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/nats-io/nats.go"
+)
+
+type NatsEventSender struct {
+	topic  string
+	client *nats.Conn
+	mu     sync.Mutex
+}
+
+func InitNatsEventService(url, topic string) (S3EventSender, error) {
+	if topic == "" {
+		return nil, fmt.Errorf("nats message topic should be specified")
+	}
+
+	client, err := nats.Connect(url)
+	if err != nil {
+		return nil, err
+	}
+
+	return &NatsEventSender{
+		topic:  topic,
+		client: client,
+	}, nil
+}
+
+func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
+	ns.mu.Lock()
+	defer ns.mu.Unlock()
+
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+
+	schema := EventSchema{
+		EventVersion: "2.2",
+		EventSource:  "aws:s3",
+		AwsRegion:    ctx.Locals("region").(string),
+		EventTime:    time.Now().Format(time.RFC3339),
+		EventName:    meta.EventName,
+		UserIdentity: EventUserIdentity{
+			PrincipalId: ctx.Locals("access").(string),
+		},
+		RequestParameters: EventRequestParams{
+			SourceIPAddress: ctx.IP(),
+		},
+		ResponseElements: EventResponseElements{
+			RequestId: ctx.Get("X-Amz-Request-Id"),
+			HostId:    ctx.Get("X-Amx-Id-2"),
+		},
+		S3: EventS3Data{
+			S3SchemaVersion: "1.0",
+			// This field will come up after implementing per bucket notifications
+			ConfigurationId: "nats-global",
+			Bucket: EventS3BucketData{
+				Name: bucket,
+				OwnerIdentity: EventUserIdentity{
+					PrincipalId: ctx.Locals("access").(string),
+				},
+				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+			},
+			Object: EventObjectData{
+				Key:       object,
+				Size:      meta.ObjectSize,
+				ETag:      meta.ObjectETag,
+				VersionId: meta.VersionId,
+				Sequencer: genSequencer(),
+			},
+		},
+		GlacierEventData: EventGlacierData{
+			// Not supported
+			RestoreEventData: EventRestoreData{},
+		},
+	}
+
+	ns.send([]EventSchema{schema})
+}
+
+func (ns *NatsEventSender) send(evnt []EventSchema) {
+	msg, err := json.Marshal(evnt)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
+	}
+
+	err = ns.client.Publish(ns.topic, msg)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
+	}
+}

s3log/audit-logger.go

@@ -0,0 +1,112 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"crypto/tls"
+	"encoding/hex"
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type AuditLogger interface {
+	Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
+	HangUp() error
+	Shutdown() error
+}
+
+type LogMeta struct {
+	BucketOwner string
+	ObjectSize  int64
+	Action      string
+}
+
+type LogConfig struct {
+	LogFile    string
+	WebhookURL string
+}
+
+type LogFields struct {
+	BucketOwner        string
+	Bucket             string
+	Time               time.Time
+	RemoteIP           string
+	Requester          string
+	RequestID          string
+	Operation          string
+	Key                string
+	RequestURI         string
+	HttpStatus         int
+	ErrorCode          string
+	BytesSent          int
+	ObjectSize         int64
+	TotalTime          int64
+	TurnAroundTime     int64
+	Referer            string
+	UserAgent          string
+	VersionID          string
+	HostID             string
+	SignatureVersion   string
+	CipherSuite        string
+	AuthenticationType string
+	HostHeader         string
+	TLSVersion         string
+	AccessPointARN     string
+	AclRequired        string
+}
+
+func InitLogger(cfg *LogConfig) (AuditLogger, error) {
+	if cfg.WebhookURL != "" && cfg.LogFile != "" {
+		return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
+	}
+	if cfg.WebhookURL != "" {
+		return InitWebhookLogger(cfg.WebhookURL)
+	}
+	if cfg.LogFile != "" {
+		return InitFileLogger(cfg.LogFile)
+	}
+
+	return nil, nil
+}
+
+func genID() string {
+	src := rand.New(rand.NewSource(time.Now().UnixNano()))
+	b := make([]byte, 8)
+
+	if _, err := src.Read(b); err != nil {
+		panic(err)
+	}
+
+	return strings.ToUpper(hex.EncodeToString(b))
+}
+
+func getTLSVersionName(version uint16) string {
+	switch version {
+	case tls.VersionTLS10:
+		return "TLSv1.0"
+	case tls.VersionTLS11:
+		return "TLSv1.1"
+	case tls.VersionTLS12:
+		return "TLSv1.2"
+	case tls.VersionTLS13:
+		return "TLSv1.3"
+	default:
+		return ""
+	}
+}

s3log/file.go

@@ -0,0 +1,230 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"crypto/tls"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	logFileMode = 0600
+	timeFormat  = "02/January/2006:15:04:05 -0700"
+)
+
+// FileLogger is a local file audit log
+type FileLogger struct {
+	logfile string
+	f       *os.File
+	gotErr  bool
+	mu      sync.Mutex
+}
+
+var _ AuditLogger = &FileLogger{}
+
+// InitFileLogger initializes audit logs to local file
+func InitFileLogger(logname string) (AuditLogger, error) {
+	f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("open log: %w", err)
+	}
+
+	f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return &FileLogger{logfile: logname, f: f}, nil
+}
+
+// Log sends log message to file logger
+func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if f.gotErr {
+		return
+	}
+
+	lf := LogFields{}
+
+	access := "-"
+	reqURI := ctx.OriginalURL()
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+	errorCode := ""
+	httpStatus := 200
+	startTime := ctx.Locals("startTime").(time.Time)
+	tlsConnState := ctx.Context().TLSConnectionState()
+	if tlsConnState != nil {
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
+	}
+
+	if err != nil {
+		serr, ok := err.(s3err.APIError)
+		if ok {
+			errorCode = serr.Code
+			httpStatus = serr.HTTPStatusCode
+		} else {
+			errorCode = err.Error()
+			httpStatus = 500
+		}
+	}
+
+	switch ctx.Locals("access").(type) {
+	case string:
+		access = ctx.Locals("access").(string)
+	}
+
+	lf.BucketOwner = meta.BucketOwner
+	lf.Bucket = bucket
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.Key = object
+	lf.RequestURI = reqURI
+	lf.HttpStatus = httpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.ObjectSize = meta.ObjectSize
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.VersionID = ctx.Query("versionId")
+	lf.HostID = ctx.Get("X-Amz-Id-2")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
+	lf.AclRequired = "Yes"
+
+	f.writeLog(lf)
+}
+
+func (f *FileLogger) writeLog(lf LogFields) {
+	if lf.BucketOwner == "" {
+		lf.BucketOwner = "-"
+	}
+	if lf.Bucket == "" {
+		lf.Bucket = "-"
+	}
+	if lf.RemoteIP == "" {
+		lf.RemoteIP = "-"
+	}
+	if lf.Requester == "" {
+		lf.Requester = "-"
+	}
+	if lf.Operation == "" {
+		lf.Operation = "-"
+	}
+	if lf.Key == "" {
+		lf.Key = "-"
+	}
+	if lf.RequestURI == "" {
+		lf.RequestURI = "-"
+	}
+	if lf.ErrorCode == "" {
+		lf.ErrorCode = "-"
+	}
+	if lf.Referer == "" {
+		lf.Referer = "-"
+	}
+	if lf.UserAgent == "" {
+		lf.UserAgent = "-"
+	}
+	if lf.VersionID == "" {
+		lf.VersionID = "-"
+	}
+	if lf.HostID == "" {
+		lf.HostID = "-"
+	}
+	if lf.CipherSuite == "" {
+		lf.CipherSuite = "-"
+	}
+	if lf.HostHeader == "" {
+		lf.HostHeader = "-"
+	}
+	if lf.TLSVersion == "" {
+		lf.TLSVersion = "-"
+	}
+
+	log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
+		lf.BucketOwner,
+		lf.Bucket,
+		fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
+		lf.RemoteIP,
+		lf.Requester,
+		lf.RequestID,
+		lf.Operation,
+		lf.Key,
+		lf.RequestURI,
+		lf.HttpStatus,
+		lf.ErrorCode,
+		lf.BytesSent,
+		lf.ObjectSize,
+		lf.TotalTime,
+		lf.TurnAroundTime,
+		lf.Referer,
+		lf.UserAgent,
+		lf.VersionID,
+		lf.HostID,
+		lf.SignatureVersion,
+		lf.CipherSuite,
+		lf.AuthenticationType,
+		lf.HostHeader,
+		lf.TLSVersion,
+		lf.AccessPointARN,
+		lf.AclRequired,
+	)
+
+	_, err := f.f.WriteString(log)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
+		// TODO: do we need to terminate on log error?
+		// set err for now so that we don't spew errors
+		f.gotErr = true
+	}
+}
+
+// HangUp closes current logfile handle and opens a new one
+// typically needed for log rotations
+func (f *FileLogger) HangUp() error {
+	err := f.f.Close()
+	if err != nil {
+		return fmt.Errorf("close log: %w", err)
+	}
+
+	f.f, err = os.OpenFile(f.logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("open log: %w", err)
+	}
+
+	f.f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return nil
+}
+
+// Shutdown closes logfile handle
+func (f *FileLogger) Shutdown() error {
+	return f.f.Close()
+}

s3log/webhook.go

@@ -0,0 +1,156 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+// WebhookLogger is a webhook URL audit log
+type WebhookLogger struct {
+	mu  sync.Mutex
+	url string
+}
+
+var _ AuditLogger = &WebhookLogger{}
+
+// InitWebhookLogger initializes audit logs to webhook URL
+func InitWebhookLogger(url string) (AuditLogger, error) {
+	client := &http.Client{
+		Timeout: 3 * time.Second,
+	}
+	_, err := client.Post(url, "application/json", nil)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			return nil, fmt.Errorf("unreachable webhook url: %w", err)
+		}
+	}
+	return &WebhookLogger{
+		url: url,
+	}, nil
+}
+
+// Log sends log message to webhook
+func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
+	wl.mu.Lock()
+	defer wl.mu.Unlock()
+
+	lf := LogFields{}
+
+	access := "-"
+	reqURI := ctx.OriginalURL()
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+	errorCode := ""
+	httpStatus := 200
+	startTime := ctx.Locals("startTime").(time.Time)
+	tlsConnState := ctx.Context().TLSConnectionState()
+	if tlsConnState != nil {
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
+	}
+
+	if err != nil {
+		serr, ok := err.(s3err.APIError)
+		if ok {
+			errorCode = serr.Code
+			httpStatus = serr.HTTPStatusCode
+		} else {
+			errorCode = err.Error()
+			httpStatus = 500
+		}
+	}
+
+	switch ctx.Locals("access").(type) {
+	case string:
+		access = ctx.Locals("access").(string)
+	}
+
+	lf.BucketOwner = meta.BucketOwner
+	lf.Bucket = bucket
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.Key = object
+	lf.RequestURI = reqURI
+	lf.HttpStatus = httpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.ObjectSize = meta.ObjectSize
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.VersionID = ctx.Query("versionId")
+	lf.HostID = ctx.Get("X-Amz-Id-2")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
+	lf.AclRequired = "Yes"
+
+	wl.sendLog(lf)
+}
+
+func (wl *WebhookLogger) sendLog(lf LogFields) {
+	jsonLog, err := json.Marshal(lf)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse the log data: %v\n", err.Error())
+	}
+
+	req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	go makeRequest(req)
+}
+
+func makeRequest(req *http.Request) {
+	client := &http.Client{
+		Timeout: 1 * time.Second,
+	}
+	_, err := client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			fmt.Fprintf(os.Stderr, "error sending webhook log: %v\n", err)
+		}
+	}
+}
+
+// HangUp does nothing for webhooks
+func (wl *WebhookLogger) HangUp() error {
+	return nil
+}
+
+// Shutdown does nothing for webhooks
+func (wl *WebhookLogger) Shutdown() error {
+	return nil
+}

s3response/AmazonS3.xsd

@@ -0,0 +1,692 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema
+  xmlns:tns="http://s3.amazonaws.com/doc/2006-03-01/"
+  xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+  elementFormDefault="qualified"
+  targetNamespace="http://s3.amazonaws.com/doc/2006-03-01/">
+
+  <xsd:element name="CreateBucket">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="MetadataEntry">
+    <xsd:sequence>
+      <xsd:element name="Name" type="xsd:string"/>
+      <xsd:element name="Value" type="xsd:string"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:element name="CreateBucketResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="CreateBucketReturn" type="tns:CreateBucketResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="Status">
+    <xsd:sequence>
+      <xsd:element name="Code" type="xsd:int"/>
+      <xsd:element name="Description" type="xsd:string"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="Result">
+    <xsd:sequence>
+      <xsd:element name="Status" type="tns:Status"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="CreateBucketResult">
+    <xsd:sequence>
+      <xsd:element name="BucketName" type="xsd:string"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:element name="DeleteBucket">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+  <xsd:element name="DeleteBucketResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="DeleteBucketResponse" type="tns:Status"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="BucketLoggingStatus">
+    <xsd:sequence>
+      <xsd:element name="LoggingEnabled" type="tns:LoggingSettings" minOccurs="0"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="LoggingSettings">
+    <xsd:sequence>
+      <xsd:element name="TargetBucket" type="xsd:string"/>
+      <xsd:element name="TargetPrefix" type="xsd:string"/>
+      <xsd:element name="TargetGrants" type="tns:AccessControlList" minOccurs="0"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:element name="GetBucketLoggingStatus">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+  
+  <xsd:element name="GetBucketLoggingStatusResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="GetBucketLoggingStatusResponse" type="tns:BucketLoggingStatus"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+  
+  <xsd:element name="SetBucketLoggingStatus">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="BucketLoggingStatus" type="tns:BucketLoggingStatus"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="SetBucketLoggingStatusResponse">
+    <xsd:complexType>
+      <xsd:sequence/>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="GetObjectAccessControlPolicy">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="GetObjectAccessControlPolicyResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="GetObjectAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="GetBucketAccessControlPolicy">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="GetBucketAccessControlPolicyResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="GetBucketAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType abstract="true" name="Grantee"/>
+
+  <xsd:complexType name="User" abstract="true">
+    <xsd:complexContent>
+      <xsd:extension base="tns:Grantee"/>
+    </xsd:complexContent>
+  </xsd:complexType>
+
+  <xsd:complexType name="AmazonCustomerByEmail">
+    <xsd:complexContent>
+      <xsd:extension base="tns:User">
+        <xsd:sequence>
+          <xsd:element name="EmailAddress" type="xsd:string"/>              
+        </xsd:sequence>
+      </xsd:extension>
+    </xsd:complexContent>
+  </xsd:complexType>
+
+  <xsd:complexType name="CanonicalUser">
+    <xsd:complexContent>
+      <xsd:extension base="tns:User">
+        <xsd:sequence>
+          <xsd:element name="ID" type="xsd:string"/>              
+          <xsd:element name="DisplayName" type="xsd:string" minOccurs="0"/>
+        </xsd:sequence>
+      </xsd:extension>
+    </xsd:complexContent>
+  </xsd:complexType>
+
+  <xsd:complexType name="Group">
+    <xsd:complexContent>
+      <xsd:extension base="tns:Grantee">
+        <xsd:sequence>
+          <xsd:element name="URI" type="xsd:string"/>
+        </xsd:sequence>
+      </xsd:extension>
+    </xsd:complexContent>
+  </xsd:complexType>
+
+  <xsd:simpleType name="Permission">
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="READ"/>
+      <xsd:enumeration value="WRITE"/>
+      <xsd:enumeration value="READ_ACP"/>
+      <xsd:enumeration value="WRITE_ACP"/>
+      <xsd:enumeration value="FULL_CONTROL"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="StorageClass">
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="STANDARD"/>
+      <xsd:enumeration value="REDUCED_REDUNDANCY"/>
+      <xsd:enumeration value="GLACIER"/>
+      <xsd:enumeration value="UNKNOWN"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:complexType name="Grant">
+    <xsd:sequence>
+      <xsd:element name="Grantee" type="tns:Grantee"/>
+      <xsd:element name="Permission" type="tns:Permission"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="AccessControlList">
+    <xsd:sequence>
+      <xsd:element name="Grant" type="tns:Grant" minOccurs="0" maxOccurs="100"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="CreateBucketConfiguration">
+    <xsd:sequence>
+      <xsd:element name="LocationConstraint" type="tns:LocationConstraint"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="LocationConstraint">
+    <xsd:simpleContent>
+      <xsd:extension base="xsd:string"/>
+    </xsd:simpleContent>
+  </xsd:complexType>
+
+  <xsd:complexType name="AccessControlPolicy">
+    <xsd:sequence>
+      <xsd:element name="Owner" type="tns:CanonicalUser"/>
+      <xsd:element name="AccessControlList" type="tns:AccessControlList"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:element name="SetObjectAccessControlPolicy">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element name="AccessControlList" type="tns:AccessControlList"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="SetObjectAccessControlPolicyResponse">
+    <xsd:complexType>
+      <xsd:sequence/>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="SetBucketAccessControlPolicy">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="SetBucketAccessControlPolicyResponse">
+    <xsd:complexType>
+      <xsd:sequence/>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="GetObject">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element name="GetMetadata" type="xsd:boolean"/>
+        <xsd:element name="GetData" type="xsd:boolean"/>
+        <xsd:element name="InlineData" type="xsd:boolean"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+        
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="GetObjectResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="GetObjectResult">
+    <xsd:complexContent>
+      <xsd:extension base="tns:Result">
+        <xsd:sequence>
+          <xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
+          <xsd:element name="Data" type="xsd:base64Binary" nillable="true"/>
+          <xsd:element name="LastModified" type="xsd:dateTime"/>
+          <xsd:element name="ETag" type="xsd:string"/>
+        </xsd:sequence>
+      </xsd:extension>
+    </xsd:complexContent>
+  </xsd:complexType>
+
+  <xsd:element name="GetObjectExtended">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element name="GetMetadata" type="xsd:boolean"/>
+        <xsd:element name="GetData" type="xsd:boolean"/>
+        <xsd:element name="InlineData" type="xsd:boolean"/>
+        <xsd:element name="ByteRangeStart" type="xsd:long" minOccurs="0"/>
+        <xsd:element name="ByteRangeEnd" type="xsd:long" minOccurs="0"/>
+        <xsd:element name="IfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="IfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="IfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
+        <xsd:element name="IfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
+        <xsd:element name="ReturnCompleteObjectOnConditionFailure" type="xsd:boolean" minOccurs="0"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="GetObjectExtendedResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="PutObject">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
+        <xsd:element name="ContentLength" type="xsd:long"/>
+        <xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
+        <xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="PutObjectResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="PutObjectResponse" type="tns:PutObjectResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="PutObjectResult">
+    <xsd:sequence>
+      <xsd:element name="ETag" type="xsd:string"/>
+      <xsd:element name="LastModified" type="xsd:dateTime"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:element name="PutObjectInline">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element minOccurs="0" maxOccurs="100" name="Metadata" type="tns:MetadataEntry"/>
+        <xsd:element name="Data" type="xsd:base64Binary"/>
+        <xsd:element name="ContentLength" type="xsd:long"/>
+        <xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
+        <xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="PutObjectInlineResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="PutObjectInlineResponse" type="tns:PutObjectResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="DeleteObject">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="DeleteObjectResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="DeleteObjectResponse" type="tns:Status"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="ListBucket">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Prefix" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Marker" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="MaxKeys" type="xsd:int" minOccurs="0"/>
+        <xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="ListBucketResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="ListBucketResponse" type="tns:ListBucketResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="ListVersionsResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="ListVersionsResponse" type="tns:ListVersionsResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>  
+
+  <xsd:complexType name="ListEntry">
+    <xsd:sequence>
+      <xsd:element name="Key" type="xsd:string"/>
+      <xsd:element name="LastModified" type="xsd:dateTime"/>
+      <xsd:element name="ETag" type="xsd:string"/>
+      <xsd:element name="Size" type="xsd:long"/>
+      <xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
+      <xsd:element name="StorageClass" type="tns:StorageClass"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="VersionEntry">
+    <xsd:sequence>
+      <xsd:element name="Key" type="xsd:string"/>
+      <xsd:element name="VersionId" type="xsd:string"/>
+      <xsd:element name="IsLatest" type="xsd:boolean"/>
+      <xsd:element name="LastModified" type="xsd:dateTime"/>
+      <xsd:element name="ETag" type="xsd:string"/>
+      <xsd:element name="Size" type="xsd:long"/>
+      <xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
+      <xsd:element name="StorageClass" type="tns:StorageClass"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="DeleteMarkerEntry">
+    <xsd:sequence>
+      <xsd:element name="Key" type="xsd:string"/>
+      <xsd:element name="VersionId" type="xsd:string"/>
+      <xsd:element name="IsLatest" type="xsd:boolean"/>
+      <xsd:element name="LastModified" type="xsd:dateTime"/>
+      <xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="PrefixEntry">
+    <xsd:sequence>
+      <xsd:element name="Prefix" type="xsd:string"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="ListBucketResult">
+    <xsd:sequence>
+      <xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="Name" type="xsd:string"/>
+      <xsd:element name="Prefix" type="xsd:string"/>
+      <xsd:element name="Marker" type="xsd:string"/>
+      <xsd:element name="NextMarker" type="xsd:string" minOccurs="0"/>
+      <xsd:element name="MaxKeys" type="xsd:int"/>
+      <xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
+      <xsd:element name="IsTruncated" type="xsd:boolean"/>
+      <xsd:element name="Contents" type="tns:ListEntry" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="ListVersionsResult">
+    <xsd:sequence>
+      <xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="Name" type="xsd:string"/>
+      <xsd:element name="Prefix" type="xsd:string"/>
+      <xsd:element name="KeyMarker" type="xsd:string"/>
+      <xsd:element name="VersionIdMarker" type="xsd:string"/>
+      <xsd:element name="NextKeyMarker" type="xsd:string" minOccurs="0"/>
+      <xsd:element name="NextVersionIdMarker" type="xsd:string" minOccurs="0"/>
+      <xsd:element name="MaxKeys" type="xsd:int"/>
+      <xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
+      <xsd:element name="IsTruncated" type="xsd:boolean"/>
+      <xsd:choice minOccurs="0" maxOccurs="unbounded">
+          <xsd:element name="Version" type="tns:VersionEntry"/>
+          <xsd:element name="DeleteMarker" type="tns:DeleteMarkerEntry"/>
+      </xsd:choice>
+      <xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
+    </xsd:sequence>
+  </xsd:complexType>  
+
+  <xsd:element name="ListAllMyBuckets">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="ListAllMyBucketsResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="ListAllMyBucketsResponse" type="tns:ListAllMyBucketsResult"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="ListAllMyBucketsEntry">
+    <xsd:sequence>
+      <xsd:element name="Name" type="xsd:string"/>
+      <xsd:element name="CreationDate" type="xsd:dateTime"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="ListAllMyBucketsResult">
+    <xsd:sequence>
+      <xsd:element name="Owner" type="tns:CanonicalUser"/>
+      <xsd:element name="Buckets" type="tns:ListAllMyBucketsList"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="ListAllMyBucketsList">
+    <xsd:sequence>
+      <xsd:element name="Bucket" type="tns:ListAllMyBucketsEntry" minOccurs="0" maxOccurs="unbounded"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:element name="PostResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="Location" type="xsd:anyURI"/>
+        <xsd:element name="Bucket" type="xsd:string"/>
+        <xsd:element name="Key" type="xsd:string"/>
+        <xsd:element name="ETag" type="xsd:string"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:simpleType name="MetadataDirective">
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="COPY"/>
+      <xsd:enumeration value="REPLACE"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:element name="CopyObject">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="SourceBucket" type="xsd:string"/>
+        <xsd:element name="SourceKey" type="xsd:string"/>
+        <xsd:element name="DestinationBucket" type="xsd:string"/>
+        <xsd:element name="DestinationKey" type="xsd:string"/>
+        <xsd:element name="MetadataDirective" type="tns:MetadataDirective" minOccurs="0"/>
+        <xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
+        <xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
+	<xsd:element name="CopySourceIfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="CopySourceIfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="CopySourceIfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
+        <xsd:element name="CopySourceIfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
+        <xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
+        <xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
+        <xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
+        <xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="CopyObjectResponse">
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:element name="CopyObjectResult" type="tns:CopyObjectResult" />
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="CopyObjectResult">
+    <xsd:sequence>
+      <xsd:element name="LastModified" type="xsd:dateTime"/>
+      <xsd:element name="ETag" type="xsd:string"/>      
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="RequestPaymentConfiguration">
+    <xsd:sequence>
+      <xsd:element name="Payer" type="tns:Payer" minOccurs="1" maxOccurs="1"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:simpleType name="Payer">
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="BucketOwner"/>
+      <xsd:enumeration value="Requester"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+  
+  <xsd:complexType name="VersioningConfiguration">
+    <xsd:sequence>
+      <xsd:element name="Status" type="tns:VersioningStatus" minOccurs="0"/>
+      <xsd:element name="MfaDelete" type="tns:MfaDeleteStatus" minOccurs="0"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:simpleType name="MfaDeleteStatus">
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="Enabled"/>
+      <xsd:enumeration value="Disabled"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="VersioningStatus">
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="Enabled"/>
+      <xsd:enumeration value="Suspended"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:complexType name="NotificationConfiguration">
+    <xsd:sequence>
+      <xsd:element name="TopicConfiguration" minOccurs="0" maxOccurs="unbounded" type="tns:TopicConfiguration"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="TopicConfiguration">
+    <xsd:sequence>
+      <xsd:element name="Topic" minOccurs="1" maxOccurs="1" type="xsd:string"/>
+      <xsd:element name="Event" minOccurs="1" maxOccurs="unbounded" type="xsd:string"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+</xsd:schema>

s3response/README.txt

@@ -0,0 +1,6 @@
+https://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd
+
+see https://blog.aqwari.net/xml-schema-go/
+
+go install aqwari.net/xml/cmd/xsdgen@latest
+xsdgen -o s3api_xsd_generated.go -pkg s3response AmazonS3.xsd

s3response/s3response.go

@@ -0,0 +1,141 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3response
+
+import (
+	"encoding/xml"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+// Part describes part metadata.
+type Part struct {
+	PartNumber   int
+	LastModified string
+	ETag         string
+	Size         int64
+}
+
+// ListPartsResponse - s3 api list parts response.
+type ListPartsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+
+	Initiator Initiator
+	Owner     Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
+}
+
+// ListMultipartUploadsResponse - s3 api list multipart uploads response.
+type ListMultipartUploadsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
+
+	Bucket             string
+	KeyMarker          string
+	UploadIDMarker     string `xml:"UploadIdMarker"`
+	NextKeyMarker      string
+	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
+	Delimiter          string
+	Prefix             string
+	EncodingType       string `xml:"EncodingType,omitempty"`
+	MaxUploads         int
+	IsTruncated        bool
+
+	// List of pending uploads.
+	Uploads []Upload `xml:"Upload"`
+
+	// Delimed common prefixes.
+	CommonPrefixes []CommonPrefix
+}
+
+// Upload desribes in progress multipart upload
+type Upload struct {
+	Key          string
+	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
+	StorageClass string
+	Initiated    string
+}
+
+// CommonPrefix ListObjectsResponse common prefixes (directory abstraction)
+type CommonPrefix struct {
+	Prefix string
+}
+
+// Initiator same fields as Owner
+type Initiator Owner
+
+// Owner bucket ownership
+type Owner struct {
+	ID          string
+	DisplayName string
+}
+
+type Tag struct {
+	Key   string `xml:"Key"`
+	Value string `xml:"Value"`
+}
+
+type TagSet struct {
+	Tags []Tag `xml:"Tag"`
+}
+
+type Tagging struct {
+	TagSet TagSet `xml:"TagSet"`
+}
+
+type DeleteObjects struct {
+	Objects []types.ObjectIdentifier `xml:"Object"`
+}
+
+type DeleteObjectsResult struct {
+	Deleted []types.DeletedObject
+	Error   []types.Error
+}
+type SelectObjectContentPayload struct {
+	Expression          *string
+	ExpressionType      types.ExpressionType
+	RequestProgress     *types.RequestProgress
+	InputSerialization  *types.InputSerialization
+	OutputSerialization *types.OutputSerialization
+	ScanRange           *types.ScanRange
+}
+
+type SelectObjectContentResult struct {
+	Records  *types.RecordsEvent
+	Stats    *types.StatsEvent
+	Progress *types.ProgressEvent
+	Cont     *types.ContinuationEvent
+	End      *types.EndEvent
+}
+
+type Bucket struct {
+	Name  string `json:"name"`
+	Owner string `json:"owner"`
+}

s3select/message-handler.go

@@ -0,0 +1,358 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3select
+
+import (
+	"bufio"
+	"context"
+	"encoding/binary"
+	"encoding/xml"
+	"fmt"
+	"hash/crc32"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// Protocol definition for messages can be found here:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
+
+var (
+	// From ptotocol def:
+	// Enum indicating the header value type.
+	// For Amazon S3 Select, this is always 7.
+	headerValueType = byte(7)
+)
+
+func intToTwoBytes(i int) []byte {
+	return []byte{byte(i >> 8), byte(i)}
+}
+
+func generateHeader(messages ...string) []byte {
+	var header []byte
+
+	for i, message := range messages {
+		if i%2 == 1 {
+			header = append(header, headerValueType)
+			header = append(header, intToTwoBytes(len(message))...)
+		} else {
+			header = append(header, byte(len(message)))
+		}
+		header = append(header, message...)
+	}
+
+	return header
+}
+
+func generateOctetHeader(message string) []byte {
+	return generateHeader(
+		":message-type",
+		"event",
+		":content-type",
+		"application/octet-stream",
+		":event-type",
+		message)
+}
+
+func generateTextHeader(message string) []byte {
+	return generateHeader(
+		":message-type",
+		"event",
+		":content-type",
+		"text/xml",
+		":event-type",
+		message)
+}
+
+func generateNoContentHeader(message string) []byte {
+	return generateHeader(
+		":message-type",
+		"event",
+		":event-type",
+		message)
+}
+
+const (
+	// 4 bytes total byte len +
+	// 4 bytes headers bytes len +
+	// 4 bytes prelude CRC
+	preludeLen = 12
+	// CRC is uint32
+	msgCrcLen = 4
+)
+
+var (
+	recordsHeader       = generateOctetHeader("Records")
+	continuationHeader  = generateNoContentHeader("Cont")
+	continuationMessage = genMessage(continuationHeader, []byte{})
+	progressHeader      = generateTextHeader("Progress")
+	statsHeader         = generateTextHeader("Stats")
+	endHeader           = generateNoContentHeader("End")
+	endMessage          = genMessage(endHeader, []byte{})
+)
+
+func uintToBytes(n uint32) []byte {
+	b := make([]byte, 4)
+	binary.BigEndian.PutUint32(b, n)
+	return b
+}
+
+func generatePrelude(msgLen int, headerLen int) []byte {
+	prelude := make([]byte, 0, preludeLen)
+
+	// 4 bytes total byte len
+	prelude = append(prelude, uintToBytes(uint32(msgLen+headerLen+preludeLen+msgCrcLen))...)
+	// 4 bytes headers bytes len
+	prelude = append(prelude, uintToBytes(uint32(headerLen))...)
+	// 4 bytes prelude CRC
+	prelude = append(prelude, uintToBytes(crc32.ChecksumIEEE(prelude))...)
+
+	return prelude
+}
+
+const (
+	maxHeaderSize  = 1024 * 1024
+	maxMessageSize = 5 * 1024 * 1024 * 1024
+)
+
+func genMessage(header, payload []byte) []byte {
+	var msg []byte
+	// below is always true since the size is validated
+	// in the send record
+	if len(header) <= maxHeaderSize && len(payload) <= maxMessageSize {
+		msglen := preludeLen + len(header) + len(payload) + msgCrcLen
+		msg = make([]byte, 0, msglen)
+	}
+
+	msg = append(msg, generatePrelude(len(payload), len(header))...)
+	msg = append(msg, header...)
+	msg = append(msg, payload...)
+	msg = append(msg, uintToBytes(crc32.ChecksumIEEE(msg))...)
+
+	return msg
+}
+
+func genRecordsMessage(payload []byte) []byte {
+	return genMessage(recordsHeader, payload)
+}
+
+type progress struct {
+	XMLName        xml.Name `xml:"Progress"`
+	BytesScanned   int64    `xml:"BytesScanned"`
+	BytesProcessed int64    `xml:"BytesProcessed"`
+	BytesReturned  int64    `xml:"BytesReturned"`
+}
+
+func genProgressMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
+	progress := progress{
+		BytesScanned:   bytesScanned,
+		BytesProcessed: bytesProcessed,
+		BytesReturned:  bytesReturned,
+	}
+
+	xmlData, _ := xml.MarshalIndent(progress, "", "    ")
+	payload := []byte(xml.Header + string(xmlData))
+	return genMessage(progressHeader, payload)
+}
+
+type stats struct {
+	XMLName        xml.Name `xml:"Stats"`
+	BytesScanned   int64    `xml:"BytesScanned"`
+	BytesProcessed int64    `xml:"BytesProcessed"`
+	BytesReturned  int64    `xml:"BytesReturned"`
+}
+
+func genStatsMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
+	stats := stats{
+		BytesScanned:   bytesScanned,
+		BytesProcessed: bytesProcessed,
+		BytesReturned:  bytesReturned,
+	}
+
+	xmlData, _ := xml.MarshalIndent(stats, "", "    ")
+	payload := []byte(xml.Header + string(xmlData))
+	return genMessage(statsHeader, payload)
+}
+
+func genErrorMessage(errorCode, errorMessage string) []byte {
+	return genMessage(generateHeader(
+		":error-code",
+		errorCode,
+		":error-message",
+		errorMessage,
+		":message-type",
+		"error",
+	), []byte{})
+}
+
+// GetProgress is a callback function that periodically retrieves the current
+// values for the following if not nil.  This is used to send Progress
+// messages back to client.
+// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
+// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
+type GetProgress func() (bytesScanned int64, bytesProcessed int64)
+
+type MessageHandler struct {
+	sync.Mutex
+	ctx           context.Context
+	cancel        context.CancelFunc
+	writer        *bufio.Writer
+	data          chan []byte
+	getProgress   GetProgress
+	stopCh        chan bool
+	resetCh       chan bool
+	bytesReturned int64
+}
+
+// NewMessageHandler creates a new MessageHandler instance and starts the event streaming
+func NewMessageHandler(ctx context.Context, w *bufio.Writer, getProgressFunc GetProgress) *MessageHandler {
+	ctx, cancel := context.WithCancel(ctx)
+
+	mh := &MessageHandler{
+		ctx:         ctx,
+		cancel:      cancel,
+		writer:      w,
+		data:        make(chan []byte),
+		getProgress: getProgressFunc,
+		resetCh:     make(chan bool),
+		stopCh:      make(chan bool),
+	}
+
+	go mh.sendBackgroundMessages(mh.resetCh, mh.stopCh)
+	return mh
+}
+
+func (mh *MessageHandler) write(data []byte) error {
+	mh.Lock()
+	defer mh.Unlock()
+
+	mh.stopCh <- true
+	defer func() { mh.resetCh <- true }()
+
+	_, err := mh.writer.Write(data)
+	if err != nil {
+		return err
+	}
+
+	return mh.writer.Flush()
+}
+
+const (
+	continuationInterval = time.Second
+	progressInterval     = time.Minute
+)
+
+func (mh *MessageHandler) sendBackgroundMessages(resetCh, stopCh <-chan bool) {
+	continuationTicker := time.NewTicker(continuationInterval)
+	defer continuationTicker.Stop()
+
+	var progressTicker *time.Ticker
+	var progressTickerChan <-chan time.Time
+	if mh.getProgress != nil {
+		progressTicker = time.NewTicker(progressInterval)
+		progressTickerChan = progressTicker.C
+		defer progressTicker.Stop()
+	}
+
+Loop:
+	for {
+		select {
+		case <-mh.ctx.Done():
+			break Loop
+
+		case <-continuationTicker.C:
+			err := mh.write(continuationMessage)
+			if err != nil {
+				mh.cancel()
+				break Loop
+			}
+
+		case <-resetCh:
+			continuationTicker.Reset(continuationInterval)
+
+		case <-stopCh:
+			continuationTicker.Stop()
+
+		case <-progressTickerChan:
+			var bytesScanned, bytesProcessed int64
+			if mh.getProgress != nil {
+				bytesScanned, bytesProcessed = mh.getProgress()
+			}
+			bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
+			err := mh.write(genProgressMessage(bytesScanned, bytesProcessed, bytesReturned))
+			if err != nil {
+				mh.cancel()
+				break Loop
+			}
+		}
+	}
+}
+
+// SendRecord sends a single Records message
+func (mh *MessageHandler) SendRecord(payload []byte) error {
+	if mh.ctx.Err() != nil {
+		return mh.ctx.Err()
+	}
+
+	if len(payload) > maxMessageSize {
+		return fmt.Errorf("record max size exceeded")
+	}
+
+	err := mh.write(genRecordsMessage(payload))
+	if err != nil {
+		return err
+	}
+
+	atomic.AddInt64(&mh.bytesReturned, int64(len(payload)))
+	return nil
+}
+
+// Finish terminates message stream with Stats and End message
+// generates stats and end message using function args based on:
+// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
+// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
+func (mh *MessageHandler) Finish(bytesScanned, bytesProcessed int64) error {
+	if mh.ctx.Err() != nil {
+		return mh.ctx.Err()
+	}
+
+	bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
+	err := mh.write(genStatsMessage(bytesScanned, bytesProcessed, bytesReturned))
+	if err != nil {
+		return err
+	}
+
+	err = mh.write(endMessage)
+	if err != nil {
+		return err
+	}
+
+	mh.cancel()
+	return nil
+}
+
+// FinishWithError terminates event stream with error
+func (mh *MessageHandler) FinishWithError(errorCode, errorMessage string) error {
+	if mh.ctx.Err() != nil {
+		return mh.ctx.Err()
+	}
+	err := mh.write(genErrorMessage(errorCode, errorMessage))
+	if err != nil {
+		return err
+	}
+
+	mh.cancel()
+	return nil
+}