Merge pull request #384 from versity/luke/posix_test

Luke/posix test

.dockerignore

@@ -0,0 +1,46 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+cmd/versitygw/versitygw
+/versitygw
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# ignore IntelliJ directories
+.idea
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+/versitygw.spec.in
+*.tar
+*.tar.gz
+**/rand.data
+/profile.txt
+
+dist/
+
+# Release config files
+/.github
+
+# Docker configuration files
+*Dockerfile
+/docker-compose.yml
+
+# read files
+/LICENSE
+/NOTICE
+/CODE_OF_CONDUCT.md
+/README.md

.env.dev

@@ -0,0 +1,8 @@
+POSIX_PORT=7071
+PROXY_PORT=7070
+ACCESS_KEY_ID=user
+SECRET_ACCESS_KEY=pass
+IAM_DIR=.
+SETUP_DIR=.
+AZ_ACCOUNT_NAME=devstoreaccount1
+AZ_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Server Version**
+ output of
+```
+./versitygw -version
+uname -a
+```
+
+**Additional context**
+Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      dev-dependencies:
+        patterns:
+          - "*"

.github/workflows/functional.yml

@@ -0,0 +1,30 @@
+name: functional tests
+on: pull_request
+jobs:
+
+  build:
+    name: RunTests
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+
+    - name: Get Dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Build and Run
+      run: |
+        make testbin
+        ./runtests.sh
+
+    - name: Coverage Report
+      run: |
+        go tool covdata percent -i=/tmp/covdata

.github/workflows/go.yml

@@ -7,15 +7,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-
     - name: Verify all files pass gofmt formatting
       run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
 
@@ -24,10 +24,10 @@ jobs:
         go get -v -t -d ./...
 
     - name: Build
-      run: go build -o versitygw cmd/versitygw/*.go
+      run: make
 
     - name: Test
-      run: go test -v -timeout 30s -tags=github ./...
+      run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
 
     - name: Install govulncheck
       run: go install golang.org/x/vuln/cmd/govulncheck@latest

.github/workflows/goreleaser.yml

@@ -0,0 +1,31 @@
+name: goreleaser
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+  # packages: write
+  # issues: write
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/static.yml

@@ -1,23 +1,22 @@
 name: staticcheck
 on: pull_request
 jobs: 
+
   build: 
     name: Check
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: "1.20"
-        id: go
-      - 
-        name: "Set up repo"
-        uses: actions/checkout@v1
-        with: 
-          fetch-depth: 1
-      - 
-        name: "staticcheck"
-        uses: dominikh/staticcheck-action@v1.3.0
-        with: 
-          install-go: false
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+ 
+    - name: "staticcheck"
+      uses: dominikh/staticcheck-action@v1.3.0
+      with: 
+        install-go: false

.github/workflows/system.yml

@@ -0,0 +1,42 @@
+name: system tests
+on: pull_request
+#on:
+#  workflow_dispatch:
+#    inputs:
+#      run_workflow:
+#        description: 'Run command-line tests'
+jobs:
+  build:
+    name: RunTests
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 'stable'
+        id: go
+
+      - name: Get Dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Install BATS
+        run: |
+          git clone https://github.com/bats-core/bats-core.git
+          cd bats-core && ./install.sh $HOME
+
+      - name: Build and Run
+        run: |
+          make testbin
+          export AWS_ACCESS_KEY_ID=user
+          export AWS_SECRET_ACCESS_KEY=pass
+          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity 
+          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
+          export VERSITY_EXE=./versitygw
+          mkdir /tmp/gw
+          VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/s3_bucket_tests.sh
+          VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/posix_tests.sh

.gitignore

@@ -25,6 +25,9 @@ go.work
 # ignore IntelliJ directories
 .idea
 
+# ignore VS code directories
+.vscode
+
 # auto generated VERSION file
 VERSION
 
@@ -32,3 +35,14 @@ VERSION
 /versitygw.spec
 *.tar
 *.tar.gz
+**/rand.data
+/profile.txt
+
+dist/
+
+# secrets file for local github-actions testing
+.secrets
+
+# env files for testing
+.env*
+!.env.default

.goreleaser.yaml

@@ -0,0 +1,55 @@
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - goos:
+      - linux
+      - darwin
+      # windows is untested, we can start doing windows releases
+      # if someone is interested in taking on testing
+      # - windows
+    env:
+      # disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
+      # once we need to enable this, we will need to do per distro releases
+      - CGO_ENABLED=0
+    main: ./cmd/versitygw
+    binary: ./cmd/versitygw
+    id: versitygw
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge '
+
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile

@@ -0,0 +1,24 @@
+FROM golang:1.20-alpine
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY ./ ./
+
+WORKDIR /app/cmd/versitygw
+RUN go build -o versitygw
+
+FROM alpine:latest
+
+# These arguments can be overriden when building the image
+ARG IAM_DIR=/tmp/vgw
+ARG SETUP_DIR=/tmp/vgw
+
+RUN mkdir -p $IAM_DIR
+RUN mkdir -p $SETUP_DIR
+
+COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
+
+ENTRYPOINT [ "/app/versitygw" ]

Dockerfile.dev

@@ -0,0 +1,17 @@
+FROM golang:1.20
+
+WORKDIR /app
+
+COPY go.mod ./
+RUN go mod download
+
+COPY ./ ./
+
+ARG IAM_DIR=/tmp/vgw
+ARG SETUP_DIR=/tmp/vgw
+
+RUN mkdir -p $IAM_DIR
+RUN mkdir -p $SETUP_DIR
+
+RUN go get github.com/githubnemo/CompileDaemon
+RUN go install github.com/githubnemo/CompileDaemon

Makefile

@@ -34,6 +34,9 @@ build: $(BIN)
 $(BIN):
 	$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
 
+testbin:
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
+
 .PHONY: test
 test: 
 	$(GOTEST) ./...
@@ -71,3 +74,23 @@ dist: $(BIN).spec
 	rm -f VERSION
 	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
+
+# Creates and runs S3 gateway instance in a docker container
+.PHONY: up-posix
+up-posix:
+	docker compose --env-file .env.dev up posix
+
+# Creates and runs S3 gateway proxy instance in a docker container
+.PHONY: up-proxy
+up-proxy:
+	docker compose --env-file .env.dev up proxy
+
+# Creates and runs S3 gateway to azurite instance in a docker container
+.PHONY: up-azurite
+up-azurite:
+	docker compose --env-file .env.dev up azurite azuritegw
+
+# Creates and runs both S3 gateway and proxy server instances in docker containers
+.PHONY: up-app
+up-app:
+	docker compose --env-file .env.dev up

README.md

@@ -1,4 +1,4 @@
-# The Versity Gateway: A High-Performance Open Source S3 to File Translation Tool
+# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
 
 <picture>
   <source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
@@ -8,13 +8,20 @@
 
  [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-The Versity Gateway: A High-Performance Open Source S3 to File Translation Tool
+**Current status:** Ready for general testing, Issue reports welcome.
+
+**News:**<br>
+* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
 
-Current status: Alpha, in development not yet suited for production use
 
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
-Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and file-based storage systems. The Versity Gateway bridges the gap between S3-reliant applications and file storage systems, enabling enhanced compatibility and integration with file based systems while offering exceptional scalability.
+* Share filesystem directory via S3 protocol
+* Proxy S3 requests to S3 storage
+* Simple to deploy S3 server with a single command
+* Protocol compatibility in `posix` allows common access to files via posix or S3 
+
+Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
 

auth/acl.go

@@ -17,7 +17,6 @@ package auth
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -42,7 +41,11 @@ type GetBucketAclOutput struct {
 }
 
 type AccessControlList struct {
-	Grants []types.Grant
+	Grants []types.Grant `xml:"Grant"`
+}
+type AccessControlPolicy struct {
+	AccessControlList AccessControlList `xml:"AccessControlList"`
+	Owner             types.Owner
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -80,82 +83,104 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 	}, nil
 }
 
-func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) error {
+func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
+	if input == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
 	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
 	}
 
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
 		acl.ACL = input.ACL
 		acl.Grantees = []Grantee{}
-		return nil
+	} else {
+		grantees := []Grantee{}
+		accs := []string{}
+
+		if input.GrantRead != nil {
+			fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
+
+			if *input.GrantFullControl != "" {
+				fullControlList = splitUnique(*input.GrantFullControl, ",")
+				for _, str := range fullControlList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
+				}
+			}
+			if *input.GrantRead != "" {
+				readList = splitUnique(*input.GrantRead, ",")
+				for _, str := range readList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
+				}
+			}
+			if *input.GrantReadACP != "" {
+				readACPList = splitUnique(*input.GrantReadACP, ",")
+				for _, str := range readACPList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
+				}
+			}
+			if *input.GrantWrite != "" {
+				writeList = splitUnique(*input.GrantWrite, ",")
+				for _, str := range writeList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
+				}
+			}
+			if *input.GrantWriteACP != "" {
+				writeACPList = splitUnique(*input.GrantWriteACP, ",")
+				for _, str := range writeACPList {
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
+				}
+			}
+
+			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
+		} else {
+			cache := make(map[string]bool)
+			for _, grt := range input.AccessControlPolicy.Grants {
+				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
+				if _, ok := cache[*grt.Grantee.ID]; !ok {
+					cache[*grt.Grantee.ID] = true
+					accs = append(accs, *grt.Grantee.ID)
+				}
+			}
+		}
+
+		// Check if the specified accounts exist
+		accList, err := CheckIfAccountsExist(accs, iam)
+		if err != nil {
+			return nil, err
+		}
+		if len(accList) > 0 {
+			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
+		}
+
+		acl.Grantees = grantees
+		acl.ACL = ""
 	}
 
-	grantees := []Grantee{}
-
-	fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
-
-	if *input.GrantFullControl != "" {
-		fullControlList = splitUnique(*input.GrantFullControl, ",")
-		fmt.Println(fullControlList)
-		for _, str := range fullControlList {
-			grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
-		}
-	}
-	if *input.GrantRead != "" {
-		readList = splitUnique(*input.GrantRead, ",")
-		for _, str := range readList {
-			grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
-		}
-	}
-	if *input.GrantReadACP != "" {
-		readACPList = splitUnique(*input.GrantReadACP, ",")
-		for _, str := range readACPList {
-			grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
-		}
-	}
-	if *input.GrantWrite != "" {
-		writeList = splitUnique(*input.GrantWrite, ",")
-		for _, str := range writeList {
-			grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
-		}
-	}
-	if *input.GrantWriteACP != "" {
-		writeACPList = splitUnique(*input.GrantWriteACP, ",")
-		for _, str := range writeACPList {
-			grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
-		}
-	}
-
-	accs := append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
-
-	// Check if the specified accounts exist
-	accList, err := checkIfAccountsExist(accs, iam)
+	result, err := json.Marshal(acl)
 	if err != nil {
-		return err
-	}
-	if len(accList) > 0 {
-		return fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
+		return nil, err
 	}
 
-	acl.Grantees = grantees
-	acl.ACL = ""
-
-	return nil
+	return result, nil
 }
 
-func checkIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
+func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
 	result := []string{}
 
 	for _, acc := range accs {
 		_, err := iam.GetUserAccount(acc)
-		if err != nil && err != ErrNoSuchUser {
+		if err != nil {
+			if err == ErrNoSuchUser {
+				result = append(result, acc)
+				continue
+			}
+			if err == ErrNotSupported {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			}
 			return nil, fmt.Errorf("check user account: %w", err)
 		}
-		if err == nil {
-			result = append(result, acc)
-		}
 	}
 	return result, nil
 }
@@ -175,7 +200,7 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRoot bool) error {
+func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
@@ -214,29 +239,14 @@ func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRo
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
-func IsAdmin(access string, isRoot bool) error {
-	var data IAMConfig
-
+func IsAdmin(acct Account, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
 
-	file, err := os.ReadFile("users.json")
-	if err != nil {
-		return fmt.Errorf("unable to read config file: %w", err)
-	}
-
-	if err := json.Unmarshal(file, &data); err != nil {
-		return err
-	}
-
-	acc, ok := data.AccessAccounts[access]
-	if !ok {
-		return fmt.Errorf("user does not exist")
-	}
-
-	if acc.Role == "admin" {
+	if acct.Role == "admin" {
 		return nil
 	}
-	return fmt.Errorf("only admin users have access to this resource")
+
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }

auth/iam.go

@@ -16,19 +16,88 @@ package auth
 
 import (
 	"errors"
+	"fmt"
+	"time"
 )
 
 // Account is a gateway IAM account
 type Account struct {
-	Secret string `json:"secret"`
-	Role   string `json:"role"`
+	Access    string `json:"access"`
+	Secret    string `json:"secret"`
+	Role      string `json:"role"`
+	UserID    int    `json:"userID"`
+	GroupID   int    `json:"groupID"`
+	ProjectID int    `json:"projectID"`
 }
 
 // IAMService is the interface for all IAM service implementations
+//
+//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
 type IAMService interface {
-	CreateAccount(access string, account Account) error
+	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
 	DeleteUserAccount(access string) error
+	ListUserAccounts() ([]Account, error)
+	Shutdown() error
 }
 
 var ErrNoSuchUser = errors.New("user not found")
+
+type Opts struct {
+	Dir                string
+	LDAPServerURL      string
+	LDAPBindDN         string
+	LDAPPassword       string
+	LDAPQueryBase      string
+	LDAPObjClasses     string
+	LDAPAccessAtr      string
+	LDAPSecretAtr      string
+	LDAPRoleAtr        string
+	S3Access           string
+	S3Secret           string
+	S3Region           string
+	S3Bucket           string
+	S3Endpoint         string
+	S3DisableSSlVerfiy bool
+	S3Debug            bool
+	CacheDisable       bool
+	CacheTTL           int
+	CachePrune         int
+}
+
+func New(o *Opts) (IAMService, error) {
+	var svc IAMService
+	var err error
+
+	switch {
+	case o.Dir != "":
+		svc, err = NewInternal(o.Dir)
+		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
+	case o.LDAPServerURL != "":
+		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
+			o.LDAPObjClasses)
+		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
+	case o.S3Endpoint != "":
+		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
+		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
+			o.S3Endpoint, o.S3Bucket)
+	default:
+		// if no iam options selected, default to the single user mode
+		fmt.Println("No IAM service configured, enabling single account mode")
+		return IAMServiceSingle{}, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if o.CacheDisable {
+		return svc, nil
+	}
+
+	return NewCache(svc,
+		time.Duration(o.CacheTTL)*time.Second,
+		time.Duration(o.CachePrune)*time.Second), nil
+}

auth/iam_cache.go

@@ -0,0 +1,179 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+)
+
+// IAMCache is an in memory cache of the IAM accounts
+// with expiration. This helps to alleviate the load on
+// the real IAM service if the gateway is handling
+// many requests. This forwards account updates to the
+// underlying service, and returns cached results while
+// the in memory account is not expired.
+type IAMCache struct {
+	service  IAMService
+	iamcache *icache
+	cancel   context.CancelFunc
+}
+
+var _ IAMService = &IAMCache{}
+
+type item struct {
+	value Account
+	exp   time.Time
+}
+
+type icache struct {
+	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
+}
+
+func (i *icache) set(k string, v Account) {
+	cpy := v
+	i.Lock()
+	i.items[k] = item{
+		exp:   time.Now().Add(i.expire),
+		value: cpy,
+	}
+	i.Unlock()
+}
+
+func (i *icache) get(k string) (Account, bool) {
+	i.RLock()
+	v, ok := i.items[k]
+	i.RUnlock()
+	if !ok || !v.exp.After(time.Now()) {
+		return Account{}, false
+	}
+	return v.value, true
+}
+
+func (i *icache) Delete(k string) {
+	i.Lock()
+	delete(i.items, k)
+	i.Unlock()
+}
+
+func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+
+		now := time.Now()
+
+		i.Lock()
+		// prune expired entries
+		for k, v := range i.items {
+			if now.After(v.exp) {
+				delete(i.items, k)
+			}
+		}
+		i.Unlock()
+
+		// sleep for the clean interval or context cancelation,
+		// whichever comes first
+		select {
+		case <-ctx.Done():
+		case <-time.After(interval):
+		}
+	}
+}
+
+// NewCache initializes an IAM cache for the provided service. The expireTime
+// is the duration a cache entry can be valid, and the cleanupInterval is
+// how often to scan cache and cleanup expired entries.
+func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
+	i := &IAMCache{
+		service: service,
+		iamcache: &icache{
+			items:  make(map[string]item),
+			expire: expireTime,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go i.iamcache.gcCache(ctx, cleanupInterval)
+	i.cancel = cancel
+
+	return i
+}
+
+// CreateAccount send create to IAM service and creates an account cache entry
+func (c *IAMCache) CreateAccount(account Account) error {
+	err := c.service.CreateAccount(account)
+	if err != nil {
+		return err
+	}
+
+	// we need a copy of account to be able to store beyond the
+	// lifetime of the request, otherwise Fiber will reuse and corrupt
+	// these entries
+	acct := Account{
+		Access: strings.Clone(account.Access),
+		Secret: strings.Clone(account.Secret),
+		Role:   strings.Clone(account.Role),
+	}
+
+	c.iamcache.set(acct.Access, acct)
+	return nil
+}
+
+// GetUserAccount retrieves the cache account if it is in the cache and not
+// expired. Otherwise retrieves from underlying IAM service and caches
+// result for the expire duration.
+func (c *IAMCache) GetUserAccount(access string) (Account, error) {
+	acct, found := c.iamcache.get(access)
+	if found {
+		return acct, nil
+	}
+
+	a, err := c.service.GetUserAccount(access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	c.iamcache.set(access, a)
+	return a, nil
+}
+
+// DeleteUserAccount deletes account from IAM service and cache
+func (c *IAMCache) DeleteUserAccount(access string) error {
+	err := c.service.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.Delete(access)
+	return nil
+}
+
+// ListUserAccounts is a passthrough to the underlying service and
+// does not make use of the cache
+func (c *IAMCache) ListUserAccounts() ([]Account, error) {
+	return c.service.ListUserAccounts()
+}
+
+// Shutdown graceful termination of service
+func (c *IAMCache) Shutdown() error {
+	c.cancel()
+	return nil
+}

auth/iam_internal.go

@@ -16,47 +16,44 @@ package auth
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
-	"hash/crc32"
-	"sync"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+const (
+	iamFile       = "users.json"
+	iamBackupFile = "users.json.backup"
 )
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	storer Storer
-
-	mu     sync.RWMutex
-	accts  IAMConfig
-	serial uint32
+	dir string
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
 type UpdateAcctFunc func([]byte) ([]byte, error)
 
-// Storer is the interface to manage the peristent IAM data for the internal
-// IAM service
-type Storer interface {
-	InitIAM() error
-	GetIAM() ([]byte, error)
-	StoreIAM(UpdateAcctFunc) error
-}
-
-// IAMConfig stores all internal IAM accounts
-type IAMConfig struct {
+// iAMConfig stores all internal IAM accounts
+type iAMConfig struct {
 	AccessAccounts map[string]Account `json:"accessAccounts"`
 }
 
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(s Storer) (*IAMServiceInternal, error) {
+func NewInternal(dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		storer: s,
+		dir: dir,
 	}
 
-	err := i.updateCache()
+	err := i.initIAM()
 	if err != nil {
-		return nil, fmt.Errorf("refresh iam cache: %w", err)
+		return nil, fmt.Errorf("init iam: %w", err)
 	}
 
 	return i, nil
@@ -64,28 +61,20 @@ func NewInternal(s Storer) (*IAMServiceInternal, error) {
 
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
-func (s *IAMServiceInternal) CreateAccount(access string, account Account) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
-		var conf IAMConfig
-
-		if len(data) > 0 {
-			if err := json.Unmarshal(data, &conf); err != nil {
-				return nil, fmt.Errorf("failed to parse iam: %w", err)
-			}
-		} else {
-			conf.AccessAccounts = make(map[string]Account)
+func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
 		}
 
-		_, ok := conf.AccessAccounts[access]
+		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
 			return nil, fmt.Errorf("account already exists")
 		}
-		conf.AccessAccounts[access] = account
+		conf.AccessAccounts[account.Access] = account
 
-		b, err := json.Marshal(s.accts)
+		b, err := json.Marshal(conf)
 		if err != nil {
 			return nil, fmt.Errorf("failed to serialize iam: %w", err)
 		}
@@ -97,25 +86,12 @@ func (s *IAMServiceInternal) CreateAccount(access string, account Account) error
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	data, err := s.storer.GetIAM()
+	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
 	}
 
-	serial := crc32.ChecksumIEEE(data)
-	if serial != s.serial {
-		s.mu.RUnlock()
-		err := s.updateCache()
-		s.mu.RLock()
-		if err != nil {
-			return Account{}, fmt.Errorf("refresh iam cache: %w", err)
-		}
-	}
-
-	acct, ok := s.accts.AccessAccounts[access]
+	acct, ok := conf.AccessAccounts[access]
 	if !ok {
 		return Account{}, ErrNoSuchUser
 	}
@@ -123,52 +99,18 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
-// updateCache must be called with no locks held
-func (s *IAMServiceInternal) updateCache() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	data, err := s.storer.GetIAM()
-	if err != nil {
-		return fmt.Errorf("get iam data: %w", err)
-	}
-
-	serial := crc32.ChecksumIEEE(data)
-
-	if len(data) > 0 {
-		if err := json.Unmarshal(data, &s.accts); err != nil {
-			return fmt.Errorf("failed to parse the config file: %w", err)
-		}
-	} else {
-		s.accts.AccessAccounts = make(map[string]Account)
-	}
-
-	s.serial = serial
-
-	return nil
-}
-
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
-		if len(data) == 0 {
-			// empty config, do nothing
-			return data, nil
-		}
-
-		var conf IAMConfig
-
-		if err := json.Unmarshal(data, &conf); err != nil {
-			return nil, fmt.Errorf("failed to parse iam: %w", err)
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
 		}
 
 		delete(conf.AccessAccounts, access)
 
-		b, err := json.Marshal(s.accts)
+		b, err := json.Marshal(conf)
 		if err != nil {
 			return nil, fmt.Errorf("failed to serialize iam: %w", err)
 		}
@@ -176,3 +118,218 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 		return b, nil
 	})
 }
+
+// ListUserAccounts lists all the user accounts stored.
+func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getIAM()
+	if err != nil {
+		return []Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// Shutdown graceful termination of service
+func (s *IAMServiceInternal) Shutdown() error {
+	return nil
+}
+
+const (
+	iamMode = 0600
+)
+
+func (s *IAMServiceInternal) initIAM() error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	_, err := os.ReadFile(fname)
+	if errors.Is(err, fs.ErrNotExist) {
+		b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
+		if err != nil {
+			return fmt.Errorf("marshal default iam: %w", err)
+		}
+		err = os.WriteFile(fname, b, iamMode)
+		if err != nil {
+			return fmt.Errorf("write default iam: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
+	b, err := s.readIAMData()
+	if err != nil {
+		return iAMConfig{}, err
+	}
+
+	return parseIAM(b)
+}
+
+func parseIAM(b []byte) (iAMConfig, error) {
+	var conf iAMConfig
+	if err := json.Unmarshal(b, &conf); err != nil {
+		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
+	}
+
+	return conf, nil
+}
+
+const (
+	backoff  = 100 * time.Millisecond
+	maxretry = 300
+)
+
+func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
+	// We are going to be racing with other running gateways without any
+	// coordination. So we might find the file does not exist at times.
+	// For this case we need to retry for a while assuming the other gateway
+	// will eventually write the file. If it doesn't after the max retries,
+	// then we will return the error.
+
+	retries := 0
+
+	for {
+		b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+			return nil, fmt.Errorf("read iam file: %w", err)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		return b, nil
+	}
+}
+
+func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
+	// We are going to be racing with other running gateways without any
+	// coordination. So the strategy here is to read the current file data.
+	// If the file doesn't exist, then we assume someone else is currently
+	// updating the file. So we just need to keep retrying. We also need
+	// to make sure the data is consistent within a single update. So racing
+	// writes to a file would possibly leave this in some invalid state.
+	// We can get atomic updates with rename. If we read the data, update
+	// the data, write to a temp file, then rename the tempfile back to the
+	// data file. This should always result in a complete data image.
+
+	// There is at least one unsolved failure mode here.
+	// If a gateway removes the data file and then crashes, all other
+	// gateways will retry forever thinking that the original will eventually
+	// write the file.
+
+	retries := 0
+	fname := filepath.Join(s.dir, iamFile)
+
+	for {
+		b, err := os.ReadFile(fname)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+
+			// we have been unsuccessful trying to read the iam file
+			// so this must be the case where something happened and
+			// the file did not get updated successfully, and probably
+			// isn't going to be. The recovery procedure would be to
+			// copy the backup file into place of the original.
+			return fmt.Errorf("no iam file, needs backup recovery")
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("read iam file: %w", err)
+		}
+
+		// reset retries on successful read
+		retries = 0
+
+		err = os.Remove(fname)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			time.Sleep(backoff)
+			continue
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("remove old iam file: %w", err)
+		}
+
+		// save copy of data
+		datacopy := make([]byte, len(b))
+		copy(datacopy, b)
+
+		// make a backup copy in case we crash before update
+		// this is after remove, so there is a small window something
+		// can go wrong, but the remove should barrier other gateways
+		// from trying to write backup at the same time. Only one
+		// gateway will successfully remove the file.
+		os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
+
+		b, err = update(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return fmt.Errorf("update iam data: %w", err)
+		}
+
+		err = s.writeTempFile(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return err
+		}
+
+		break
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) writeTempFile(b []byte) error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	f, err := os.CreateTemp(s.dir, iamFile)
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	_, err = f.Write(b)
+	if err != nil {
+		return fmt.Errorf("write temp file: %w", err)
+	}
+
+	err = os.Rename(f.Name(), fname)
+	if err != nil {
+		return fmt.Errorf("rename temp file: %w", err)
+	}
+
+	return nil
+}

auth/iam_ldap.go

@@ -0,0 +1,133 @@
+package auth
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-ldap/ldap/v3"
+)
+
+type LdapIAMService struct {
+	conn       *ldap.Conn
+	queryBase  string
+	objClasses []string
+	accessAtr  string
+	secretAtr  string
+	roleAtr    string
+}
+
+var _ IAMService = &LdapIAMService{}
+
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+		return nil, fmt.Errorf("required parameters list not fully provided")
+	}
+	conn, err := ldap.Dial("tcp", url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
+	}
+
+	err = conn.Bind(bindDN, pass)
+	if err != nil {
+		return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
+	}
+	return &LdapIAMService{
+		conn:       conn,
+		queryBase:  queryBase,
+		objClasses: strings.Split(objClasses, ","),
+		accessAtr:  accAtr,
+		secretAtr:  secAtr,
+		roleAtr:    roleAtr,
+	}, nil
+}
+
+func (ld *LdapIAMService) CreateAccount(account Account) error {
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry.Attribute("objectClass", ld.objClasses)
+	userEntry.Attribute(ld.accessAtr, []string{account.Access})
+	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
+	userEntry.Attribute(ld.roleAtr, []string{account.Role})
+
+	err := ld.conn.Add(userEntry)
+	if err != nil {
+		return fmt.Errorf("error adding an entry: %w", err)
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	result, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return Account{}, err
+	}
+
+	entry := result.Entries[0]
+	return Account{
+		Access: entry.GetAttributeValue(ld.accessAtr),
+		Secret: entry.GetAttributeValue(ld.secretAtr),
+		Role:   entry.GetAttributeValue(ld.roleAtr),
+	}, nil
+}
+
+func (ld *LdapIAMService) DeleteUserAccount(access string) error {
+	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+
+	err := ld.conn.Del(delReq)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
+	searchFilter := ""
+	for _, el := range ld.objClasses {
+		searchFilter += fmt.Sprintf("(objectClass=%v)", el)
+	}
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(&%v)", searchFilter),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	resp, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	result := []Account{}
+	for _, el := range resp.Entries {
+		result = append(result, Account{
+			Access: el.GetAttributeValue(ld.accessAtr),
+			Secret: el.GetAttributeValue(ld.secretAtr),
+			Role:   el.GetAttributeValue(ld.roleAtr),
+		})
+	}
+
+	return result, nil
+}
+
+// Shutdown graceful termination of service
+func (ld *LdapIAMService) Shutdown() error {
+	return ld.conn.Close()
+}

auth/iam_s3_object.go

@@ -0,0 +1,263 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+)
+
+// IAMServiceS3 stores user accounts in an S3 object
+// The endpoint, credentials, bucket, and region are provided
+// from cli configuration.
+// The object format and name is the same as the internal IAM service:
+// coming from iAMConfig and iamFile in iam_internal.
+
+type IAMServiceS3 struct {
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
+	sslSkipVerify bool
+	debug         bool
+	client        *s3.Client
+}
+
+var _ IAMService = &IAMServiceS3{}
+
+func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+	if access == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service access key")
+	}
+	if secret == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service secret key")
+	}
+	if region == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service region")
+	}
+	if bucket == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service bucket")
+	}
+	if endpoint == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service endpoint")
+	}
+
+	i := &IAMServiceS3{
+		access:        access,
+		secret:        secret,
+		region:        region,
+		bucket:        bucket,
+		endpoint:      endpoint,
+		sslSkipVerify: sslSkipVerify,
+		debug:         debug,
+	}
+
+	cfg, err := i.getConfig()
+	if err != nil {
+		return nil, fmt.Errorf("init s3 IAM: %v", err)
+	}
+
+	i.client = s3.NewFromConfig(cfg)
+	return i, nil
+}
+
+func (s *IAMServiceS3) CreateAccount(account Account) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[account.Access]
+	if ok {
+		return fmt.Errorf("account already exists")
+	}
+	conf.AccessAccounts[account.Access] = account
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[access]
+	if !ok {
+		return fmt.Errorf("account does not exist")
+	}
+	delete(conf.AccessAccounts, access)
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.region,
+		HostnameImmutable: true,
+	}, nil
+}
+
+func (s *IAMServiceS3) Shutdown() error {
+	return nil
+}
+
+func (s *IAMServiceS3) getConfig() (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.region),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(context.Background(), opts...)
+}
+
+func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
+	obj := iamFile
+
+	out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
+		Bucket: &s.bucket,
+		Key:    &obj,
+	})
+	if err != nil {
+		// if the error is object not exists,
+		// init empty accounts stuct and return that
+		var nsk *types.NoSuchKey
+		if errors.As(err, &nsk) {
+			return iAMConfig{}, nil
+		}
+		var apiErr smithy.APIError
+		if errors.As(err, &apiErr) {
+			if apiErr.ErrorCode() == "NotFound" {
+				return iAMConfig{}, nil
+			}
+		}
+
+		// all other errors, return the error
+		return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
+	}
+
+	defer out.Body.Close()
+
+	b, err := io.ReadAll(out.Body)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
+	}
+
+	conf, err := parseIAM(b)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
+	}
+
+	return conf, nil
+}
+
+func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
+	b, err := json.Marshal(conf)
+	if err != nil {
+		return fmt.Errorf("failed to serialize iam: %w", err)
+	}
+
+	obj := iamFile
+	uploader := manager.NewUploader(s.client)
+	upinfo := &s3.PutObjectInput{
+		Body:   bytes.NewReader(b),
+		Bucket: &s.bucket,
+		Key:    &obj,
+	}
+	_, err = uploader.Upload(context.Background(), upinfo)
+	if err != nil {
+		return fmt.Errorf("store accounts in %v: %w", iamFile, err)
+	}
+
+	return nil
+}

auth/iam_single.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+)
+
+// IAMServiceSingle manages the single tenant (root-only) IAM service
+type IAMServiceSingle struct{}
+
+var _ IAMService = &IAMServiceSingle{}
+
+var ErrNotSupported = errors.New("method is not supported")
+
+// CreateAccount not valid in single tenant mode
+func (IAMServiceSingle) CreateAccount(account Account) error {
+	return ErrNotSupported
+}
+
+// GetUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	return Account{}, ErrNotSupported
+}
+
+// DeleteUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) DeleteUserAccount(access string) error {
+	return ErrNotSupported
+}
+
+// ListUserAccounts no accounts in single tenant mode
+func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
+	return []Account{}, nil
+}
+
+// Shutdown graceful termination of service
+func (IAMServiceSingle) Shutdown() error {
+	return nil
+}

backend/azure/azure.go

@@ -0,0 +1,986 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package azure
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+// When getting container metadata with GetProperties method the sdk returns
+// the first letter capital, when accessing the metadata after listing the containers
+// it returns the first letter lower
+type aclKey string
+
+const aclKeyCapital aclKey = "Acl"
+const aclKeyLower aclKey = "acl"
+
+type Azure struct {
+	backend.BackendUnsupported
+
+	client         *azblob.Client
+	sharedkeyCreds *azblob.SharedKeyCredential
+	defaultCreds   *azidentity.DefaultAzureCredential
+	serviceURL     string
+	sasToken       string
+}
+
+var _ backend.Backend = &Azure{}
+
+func New(accountName, accountKey, serviceURL, sasToken string) (*Azure, error) {
+	url := serviceURL
+	if serviceURL == "" && accountName != "" {
+		// if not otherwise specified, use the typical form:
+		// http(s)://<account>.blob.core.windows.net/
+		url = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName)
+	}
+
+	if sasToken != "" {
+		client, err := azblob.NewClientWithNoCredential(url+"?"+sasToken, nil)
+		if err != nil {
+			return nil, fmt.Errorf("init client: %w", err)
+		}
+		return &Azure{client: client, serviceURL: serviceURL, sasToken: sasToken}, nil
+	}
+
+	if accountName == "" {
+		// if account name not provided, try to get from env var
+		accountName = os.Getenv("AZURE_CLIENT_ID")
+	}
+
+	if accountName == "" || accountKey == "" {
+		cred, err := azidentity.NewDefaultAzureCredential(nil)
+		if err != nil {
+			return nil, fmt.Errorf("init default credentials: %w", err)
+		}
+		client, err := azblob.NewClient(url, cred, nil)
+		if err != nil {
+			return nil, fmt.Errorf("init client: %w", err)
+		}
+		return &Azure{client: client, serviceURL: url, defaultCreds: cred}, nil
+	}
+
+	cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, fmt.Errorf("init credentials: %w", err)
+	}
+
+	client, err := azblob.NewClientWithSharedKeyCredential(url, cred, nil)
+	if err != nil {
+		return nil, fmt.Errorf("init client: %w", err)
+	}
+
+	return &Azure{client: client, serviceURL: url, sharedkeyCreds: cred}, nil
+}
+
+func (az *Azure) Shutdown() {}
+
+func (az *Azure) String() string {
+	return "Azure Blob Gateway"
+}
+
+func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
+	meta := map[string]*string{
+		string(aclKeyCapital): backend.GetStringPtr(string(acl)),
+	}
+	_, err := az.client.CreateContainer(ctx, *input.Bucket, &container.CreateOptions{Metadata: meta})
+	return azureErrToS3Err(err)
+}
+
+func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	pager := az.client.NewListContainersPager(nil)
+
+	var buckets []s3response.ListAllMyBucketsEntry
+	var result s3response.ListAllMyBucketsResult
+
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return result, azureErrToS3Err(err)
+		}
+		for _, v := range resp.ContainerItems {
+			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+				Name: *v.Name,
+				// TODO: using modification date here instead of creation, is that ok?
+				CreationDate: *v.Properties.LastModified,
+			})
+		}
+	}
+
+	result.Buckets.Bucket = buckets
+	result.Owner.ID = owner
+
+	return result, nil
+}
+
+func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	client, err := az.getContainerClient(*input.Bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	return &s3.HeadBucketOutput{}, nil
+}
+
+func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	_, err := az.client.DeleteContainer(ctx, *input.Bucket, nil)
+	return azureErrToS3Err(err)
+}
+
+func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
+	tags, err := parseTags(po.Tagging)
+	if err != nil {
+		return "", err
+	}
+
+	uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, &blockblob.UploadStreamOptions{
+		Metadata: parseMetadata(po.Metadata),
+		Tags:     tags,
+	})
+	if err != nil {
+		return "", azureErrToS3Err(err)
+	}
+
+	return string(*uploadResp.ETag), nil
+}
+
+func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+	var opts *azblob.DownloadStreamOptions
+	if *input.Range != "" {
+		offset, count, err := parseRange(*input.Range)
+		if err != nil {
+			return nil, err
+		}
+		opts = &azblob.DownloadStreamOptions{
+			Range: blob.HTTPRange{
+				Count:  count,
+				Offset: offset,
+			},
+		}
+	}
+	blobDownloadResponse, err := az.client.DownloadStream(ctx, *input.Bucket, *input.Key, opts)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+	defer blobDownloadResponse.Body.Close()
+
+	_, err = io.Copy(writer, blobDownloadResponse.Body)
+	if err != nil {
+		return nil, fmt.Errorf("copy data: %w", err)
+	}
+
+	var tagcount int32
+	if blobDownloadResponse.TagCount != nil {
+		tagcount = int32(*blobDownloadResponse.TagCount)
+	}
+
+	return &s3.GetObjectOutput{
+		AcceptRanges:    input.Range,
+		ContentLength:   blobDownloadResponse.ContentLength,
+		ContentEncoding: blobDownloadResponse.ContentEncoding,
+		ContentType:     blobDownloadResponse.ContentType,
+		ETag:            (*string)(blobDownloadResponse.ETag),
+		LastModified:    blobDownloadResponse.LastModified,
+		Metadata:        parseAzMetadata(blobDownloadResponse.Metadata),
+		TagCount:        &tagcount,
+		ContentRange:    blobDownloadResponse.ContentRange,
+	}, nil
+}
+
+func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	client, err := az.getBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	return &s3.HeadObjectOutput{
+		AcceptRanges:       resp.AcceptRanges,
+		ContentLength:      resp.ContentLength,
+		ContentType:        resp.ContentType,
+		ContentEncoding:    resp.ContentEncoding,
+		ContentLanguage:    resp.ContentLanguage,
+		ContentDisposition: resp.ContentDisposition,
+		ETag:               (*string)(resp.ETag),
+		LastModified:       resp.LastModified,
+		Metadata:           parseAzMetadata(resp.Metadata),
+		Expires:            resp.ExpiresOn,
+	}, nil
+}
+
+func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	pager := az.client.NewListBlobsFlatPager(*input.Bucket, &azblob.ListBlobsFlatOptions{
+		Marker:     input.Marker,
+		MaxResults: input.MaxKeys,
+		Prefix:     input.Prefix,
+	})
+
+	var objects []types.Object
+	var nextMarker *string
+	var isTruncated bool
+	var maxKeys int32 = math.MaxInt32
+
+	if input.MaxKeys != nil {
+		maxKeys = *input.MaxKeys
+	}
+
+Pager:
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+
+		for _, v := range resp.Segment.BlobItems {
+			if nextMarker == nil && *resp.NextMarker != "" {
+				nextMarker = resp.NextMarker
+				isTruncated = true
+			}
+			if len(objects) >= int(maxKeys) {
+				break Pager
+			}
+			objects = append(objects, types.Object{
+				ETag:         (*string)(v.Properties.ETag),
+				Key:          v.Name,
+				LastModified: v.Properties.LastModified,
+				Size:         v.Properties.ContentLength,
+				StorageClass: types.ObjectStorageClass(*v.Properties.AccessTier),
+			})
+		}
+	}
+
+	// TODO: generate common prefixes when appropriate
+
+	return &s3.ListObjectsOutput{
+		Contents:    objects,
+		Marker:      input.Marker,
+		MaxKeys:     input.MaxKeys,
+		Name:        input.Bucket,
+		NextMarker:  nextMarker,
+		Prefix:      input.Prefix,
+		IsTruncated: &isTruncated,
+	}, nil
+}
+
+func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	pager := az.client.NewListBlobsFlatPager(*input.Bucket, &azblob.ListBlobsFlatOptions{
+		Marker:     input.ContinuationToken,
+		MaxResults: input.MaxKeys,
+		Prefix:     input.Prefix,
+	})
+
+	var objects []types.Object
+	var nextMarker *string
+	var isTruncated bool
+	var maxKeys int32 = math.MaxInt32
+
+	if input.MaxKeys != nil {
+		maxKeys = *input.MaxKeys
+	}
+
+Pager:
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+		for _, v := range resp.Segment.BlobItems {
+			if nextMarker == nil && *resp.NextMarker != "" {
+				nextMarker = resp.NextMarker
+				isTruncated = true
+			}
+			if len(objects) >= int(maxKeys) {
+				break Pager
+			}
+			nextMarker = resp.NextMarker
+			objects = append(objects, types.Object{
+				ETag:         (*string)(v.Properties.ETag),
+				Key:          v.Name,
+				LastModified: v.Properties.LastModified,
+				Size:         v.Properties.ContentLength,
+				StorageClass: types.ObjectStorageClass(*v.Properties.AccessTier),
+			})
+		}
+	}
+
+	// TODO: generate common prefixes when appropriate
+
+	return &s3.ListObjectsV2Output{
+		Contents:              objects,
+		ContinuationToken:     input.ContinuationToken,
+		MaxKeys:               input.MaxKeys,
+		Name:                  input.Bucket,
+		NextContinuationToken: nextMarker,
+		Prefix:                input.Prefix,
+		IsTruncated:           &isTruncated,
+	}, nil
+}
+
+func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
+	return azureErrToS3Err(err)
+}
+
+func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	delResult, errs := []types.DeletedObject{}, []types.Error{}
+	for _, obj := range input.Delete.Objects {
+		err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
+			Bucket: input.Bucket,
+			Key:    obj.Key,
+		})
+		if err == nil {
+			delResult = append(delResult, types.DeletedObject{Key: obj.Key})
+		} else {
+			serr, ok := err.(s3err.APIError)
+			if ok {
+				errs = append(errs, types.Error{
+					Key:     obj.Key,
+					Code:    &serr.Code,
+					Message: &serr.Description,
+				})
+			} else {
+				errs = append(errs, types.Error{
+					Key:     obj.Key,
+					Code:    backend.GetStringPtr("InternalError"),
+					Message: backend.GetStringPtr(err.Error()),
+				})
+			}
+		}
+	}
+
+	return s3response.DeleteObjectsResult{
+		Deleted: delResult,
+		Error:   errs,
+	}, nil
+}
+
+func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	containerClient, err := az.getContainerClient(*input.Bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := containerClient.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	dstContainerAcl, err := getAclFromMetadata(res.Metadata, aclKeyCapital)
+	if err != nil {
+		return nil, err
+	}
+
+	err = auth.VerifyACL(*dstContainerAcl, *input.ExpectedBucketOwner, types.PermissionWrite, false)
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource && isMetaSame(res.Metadata, input.Metadata) {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+	}
+
+	tags, err := parseTags(input.Tagging)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := az.getBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.CopyFromURL(ctx, az.serviceURL+"/"+*input.CopySource, &blob.CopyFromURLOptions{
+		BlobTags: tags,
+		Metadata: parseMetadata(input.Metadata),
+	})
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	return &s3.CopyObjectOutput{
+		CopyObjectResult: &types.CopyObjectResult{
+			ETag:         (*string)(resp.ETag),
+			LastModified: resp.LastModified,
+		},
+	}, nil
+}
+
+func (az *Azure) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	client, err := az.getBlobClient(bucket, object)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.SetTags(ctx, tags, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	client, err := az.getBlobClient(bucket, object)
+	if err != nil {
+		return nil, err
+	}
+
+	tags, err := client.GetTags(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	return parseAzTags(tags.BlobTagSet), nil
+}
+
+func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	client, err := az.getBlobClient(bucket, object)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.SetTags(ctx, map[string]string{}, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	// Multipart upload starts with UploadPart action so there is no
+	// correlating function for creating mutlipart uploads.
+	// TODO: since azure only allows for a single multipart upload
+	// for an object name at a time, we need to send an error back to
+	// the client if there is already an outstanding upload in progress
+	// for this object.
+	// Alternatively, is there something we can do with upload ids to
+	// keep concurrent uploads unique still? I haven't found an efficient
+	// way to rename final objects.
+	return &s3.CreateMultipartUploadOutput{
+		Bucket:   input.Bucket,
+		Key:      input.Key,
+		UploadId: input.Key,
+	}, nil
+}
+
+// Each part is translated into an uncommitted block in a newly created blob in staging area
+func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return "", err
+	}
+
+	// TODO: request streamable version of StageBlock()
+	// (*blockblob.Client).StageBlock does not have a streamable
+	// version of this function at this time, so we need to cache
+	// the body in memory to create an io.ReadSeekCloser
+	rdr, err := getReadSeekCloser(input.Body)
+	if err != nil {
+		return "", err
+	}
+
+	// block id serves as etag here
+	etag = blockIDInt32ToBase64(*input.PartNumber)
+	_, err = client.StageBlock(ctx, etag, rdr, nil)
+	if err != nil {
+		return "", parseMpError(err)
+	}
+
+	return etag, nil
+}
+
+func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return s3response.CopyObjectResult{}, nil
+	}
+
+	//TODO: handle block copy by range
+	//TODO: the action returns not implemented on azurite, maybe in production this will work?
+	// UploadId here is the source block id
+	_, err = client.StageBlockFromURL(ctx, *input.UploadId, *input.CopySource, nil)
+	if err != nil {
+		return s3response.CopyObjectResult{}, parseMpError(err)
+	}
+
+	return s3response.CopyObjectResult{}, nil
+}
+
+// Lists all uncommitted parts from the blob
+func (az *Azure) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return s3response.ListPartsResult{}, nil
+	}
+
+	resp, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
+	if err != nil {
+		return s3response.ListPartsResult{}, parseMpError(err)
+	}
+	var partNumberMarker int
+	var nextPartNumberMarker int
+	var maxParts int32 = math.MaxInt32
+	var isTruncated bool
+
+	if *input.PartNumberMarker != "" {
+		partNumberMarker, err = strconv.Atoi(*input.PartNumberMarker)
+		if err != nil {
+			return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker)
+		}
+	}
+	if input.MaxParts != nil {
+		maxParts = *input.MaxParts
+	}
+
+	parts := []s3response.Part{}
+	for _, el := range resp.BlockList.UncommittedBlocks {
+		partNumber, err := decodeBlockId(*el.Name)
+		if err != nil {
+			return s3response.ListPartsResult{}, err
+		}
+		if partNumberMarker != 0 && partNumberMarker < partNumber {
+			continue
+		}
+		if len(parts) >= int(maxParts) {
+			nextPartNumberMarker = partNumber
+			isTruncated = true
+			break
+		}
+		parts = append(parts, s3response.Part{
+			Size:         *el.Size,
+			ETag:         *el.Name,
+			PartNumber:   partNumber,
+			LastModified: time.Now().Format(backend.RFC3339TimeFormat),
+		})
+	}
+	return s3response.ListPartsResult{
+		Bucket:               *input.Bucket,
+		Key:                  *input.Key,
+		Parts:                parts,
+		NextPartNumberMarker: nextPartNumberMarker,
+		PartNumberMarker:     partNumberMarker,
+		IsTruncated:          isTruncated,
+		MaxParts:             int(maxParts),
+	}, nil
+}
+
+// Lists all block blobs, which has uncommitted blocks
+func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	client, err := az.getContainerClient(*input.Bucket)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, err
+	}
+	pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
+		Include: container.ListBlobsInclude{UncommittedBlobs: true},
+		Marker:  input.KeyMarker,
+		Prefix:  input.Prefix,
+	})
+
+	var maxUploads int32
+	if input.MaxUploads != nil {
+		maxUploads = *input.MaxUploads
+	}
+	isTruncated := false
+	nextKeyMarker := ""
+	uploads := []s3response.Upload{}
+	breakFlag := false
+
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return s3response.ListMultipartUploadsResult{}, azureErrToS3Err(err)
+		}
+		for _, el := range resp.Segment.BlobItems {
+			if el.Properties.AccessTier == nil {
+				if len(uploads) >= int(*input.MaxUploads) && maxUploads != 0 {
+					breakFlag = true
+					nextKeyMarker = *el.Name
+					isTruncated = true
+					break
+				}
+				uploads = append(uploads, s3response.Upload{
+					Key:       *el.Name,
+					Initiated: el.Properties.CreationTime.Format(backend.RFC3339TimeFormat),
+				})
+			}
+		}
+		if breakFlag {
+			break
+		}
+	}
+	return s3response.ListMultipartUploadsResult{
+		Uploads:       uploads,
+		Bucket:        *input.Bucket,
+		KeyMarker:     *input.KeyMarker,
+		NextKeyMarker: nextKeyMarker,
+		MaxUploads:    int(maxUploads),
+		Prefix:        *input.Prefix,
+		IsTruncated:   isTruncated,
+		Delimiter:     *input.Delimiter,
+	}, nil
+}
+
+// Deletes the block blob with committed/uncommitted blocks
+func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	// TODO: need to verify this blob has uncommitted blocks?
+	_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
+	if err != nil {
+		return parseMpError(err)
+	}
+	return nil
+}
+
+// Commits all the uncommitted blocks inside the block blob
+// And moves the block blob from staging area into the blobs list
+// It indicates the end of the multipart upload
+func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
+	if err != nil {
+		return nil, err
+	}
+	blockIds := []string{}
+	for _, el := range input.MultipartUpload.Parts {
+		blockIds = append(blockIds, *el.ETag)
+	}
+	resp, err := client.CommitBlockList(ctx, blockIds, nil)
+	if err != nil {
+		return nil, parseMpError(err)
+	}
+
+	return &s3.CompleteMultipartUploadOutput{
+		Bucket: input.Bucket,
+		Key:    input.Key,
+		ETag:   (*string)(resp.ETag),
+	}, nil
+}
+
+func (az *Azure) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+	meta := map[string]*string{
+		string(aclKeyCapital): backend.GetStringPtr(string(data)),
+	}
+	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{
+		Metadata: meta,
+	})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+	return nil
+}
+
+func (az *Azure) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	client, err := az.getContainerClient(*input.Bucket)
+	if err != nil {
+		return nil, err
+	}
+	props, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	aclPtr, ok := props.Metadata[string(aclKeyCapital)]
+	if !ok {
+		return nil, s3err.GetAPIError(s3err.ErrInternalError)
+	}
+
+	return []byte(*aclPtr), nil
+}
+
+func (az *Azure) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+	props, err := client.GetProperties(ctx, nil)
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	acl, err := getAclFromMetadata(props.Metadata, aclKeyCapital)
+	if err != nil {
+		return err
+	}
+
+	acl.Owner = newOwner
+
+	newAcl, err := json.Marshal(acl)
+	if err != nil {
+		return fmt.Errorf("marshal acl: %w", err)
+	}
+
+	err = az.PutBucketAcl(ctx, bucket, newAcl)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// The action actually returns the containers owned by the user, who initialized the gateway
+// TODO: Not sure if there's a way to list all the containers and owners?
+func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
+	pager := az.client.NewListContainersPager(nil)
+
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return buckets, azureErrToS3Err(err)
+		}
+		for _, v := range resp.ContainerItems {
+			acl, err := getAclFromMetadata(v.Metadata, aclKeyLower)
+			if err != nil {
+				return buckets, err
+			}
+
+			buckets = append(buckets, s3response.Bucket{
+				Name:  *v.Name,
+				Owner: acl.Owner,
+			})
+		}
+	}
+	return buckets, nil
+}
+
+func (az *Azure) getContainerURL(cntr string) string {
+	return fmt.Sprintf("%v/%v", az.serviceURL, cntr)
+}
+
+func (az *Azure) getBlobURL(cntr, blb string) string {
+	return fmt.Sprintf("%v/%v", az.getContainerURL(cntr), blb)
+}
+
+func (az *Azure) getBlobClient(cntr, blb string) (*blob.Client, error) {
+	blobURL := az.getBlobURL(cntr, blb)
+	if az.defaultCreds != nil {
+		return blob.NewClient(blobURL, az.defaultCreds, nil)
+	}
+	if az.sasToken != "" {
+		return blob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil)
+	}
+	return blob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil)
+}
+
+func (az *Azure) getContainerClient(cntr string) (*container.Client, error) {
+	containerURL := az.getContainerURL(cntr)
+	if az.defaultCreds != nil {
+		return container.NewClient(containerURL, az.defaultCreds, nil)
+	}
+	if az.sasToken != "" {
+		return container.NewClientWithNoCredential(containerURL+"?"+az.sasToken, nil)
+	}
+	return container.NewClientWithSharedKeyCredential(containerURL, az.sharedkeyCreds, nil)
+}
+
+func (az *Azure) getBlockBlobClient(cntr, blb string) (*blockblob.Client, error) {
+	blobURL := az.getBlobURL(cntr, blb)
+	if az.defaultCreds != nil {
+		return blockblob.NewClient(blobURL, az.defaultCreds, nil)
+	}
+	if az.sasToken != "" {
+		return blockblob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil)
+	}
+	return blockblob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil)
+}
+
+func parseMetadata(m map[string]string) map[string]*string {
+	if m == nil {
+		return nil
+	}
+
+	meta := make(map[string]*string)
+
+	for k, v := range m {
+		val := v
+		meta[k] = &val
+	}
+	return meta
+}
+
+func parseAzMetadata(m map[string]*string) map[string]string {
+	if m == nil {
+		return nil
+	}
+
+	meta := make(map[string]string)
+
+	for k, v := range m {
+		meta[k] = *v
+	}
+	return meta
+}
+
+func parseTags(tagstr *string) (map[string]string, error) {
+	tagsStr := getString(tagstr)
+	tags := make(map[string]string)
+
+	if tagsStr != "" {
+		tagParts := strings.Split(tagsStr, "&")
+		for _, prt := range tagParts {
+			p := strings.Split(prt, "=")
+			if len(p) != 2 {
+				return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+			}
+			tags[p[0]] = p[1]
+		}
+	}
+
+	return tags, nil
+}
+
+func parseAzTags(tagSet []*blob.Tags) map[string]string {
+	tags := map[string]string{}
+	for _, tag := range tagSet {
+		tags[*tag.Key] = *tag.Value
+	}
+
+	return tags
+}
+
+func getString(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
+}
+
+// Converts io.Reader into io.ReadSeekCloser
+func getReadSeekCloser(input io.Reader) (io.ReadSeekCloser, error) {
+	var buffer bytes.Buffer
+	_, err := io.Copy(&buffer, input)
+	if err != nil {
+		return nil, err
+	}
+
+	return streaming.NopCloser(bytes.NewReader(buffer.Bytes())), nil
+}
+
+// Creates a new Base64 encoded block id from a 32 bit integer
+func blockIDInt32ToBase64(blockID int32) string {
+	binaryBlockID := &[4]byte{} // All block IDs are 4 bytes long
+	binary.LittleEndian.PutUint32(binaryBlockID[:], uint32(blockID))
+	return base64.StdEncoding.EncodeToString(binaryBlockID[:])
+}
+
+// Decodes Base64 encoded string to integer
+func decodeBlockId(blockID string) (int, error) {
+	slice, err := base64.StdEncoding.DecodeString(blockID)
+	if err != nil {
+		return 0, nil
+	}
+
+	return int(binary.LittleEndian.Uint32(slice)), nil
+}
+
+func parseRange(rg string) (offset, count int64, err error) {
+	rangeKv := strings.Split(rg, "=")
+
+	if len(rangeKv) < 2 {
+		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) < 1 || len(bRange) > 2 {
+		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+
+	offset, err = strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+
+	if len(bRange) == 1 || bRange[1] == "" {
+		return offset, count, nil
+	}
+
+	count, err = strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+
+	if count < offset {
+		return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+
+	return offset, count - offset + 1, nil
+}
+
+func getAclFromMetadata(meta map[string]*string, key aclKey) (*auth.ACL, error) {
+	aclPtr, ok := meta[string(key)]
+	if !ok {
+		return nil, s3err.GetAPIError(s3err.ErrInternalError)
+	}
+
+	var acl auth.ACL
+	err := json.Unmarshal([]byte(*aclPtr), &acl)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal acl: %w", err)
+	}
+
+	return &acl, nil
+}
+
+func isMetaSame(azMeta map[string]*string, awsMeta map[string]string) bool {
+	if len(azMeta) != len(awsMeta)+1 {
+		return false
+	}
+
+	for key, val := range azMeta {
+		if key == string(aclKeyCapital) || key == string(aclKeyLower) {
+			continue
+		}
+		awsVal, ok := awsMeta[key]
+		if !ok || awsVal != *val {
+			return false
+		}
+	}
+
+	return true
+}

backend/azure/err.go

@@ -0,0 +1,63 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package azure
+
+import (
+	"errors"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/versity/versitygw/s3err"
+)
+
+// Parses azure ResponseError into AWS APIError
+func azureErrToS3Err(apiErr error) error {
+	var azErr *azcore.ResponseError
+	if !errors.As(apiErr, &azErr) {
+		return apiErr
+	}
+
+	return azErrToS3err(azErr)
+}
+
+func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
+	switch azErr.ErrorCode {
+	case "ContainerAlreadyExists":
+		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
+	case "InvalidResourceName", "ContainerNotFound":
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	case "BlobNotFound":
+		return s3err.GetAPIError(s3err.ErrNoSuchKey)
+	case "TagsTooLarge":
+		return s3err.GetAPIError(s3err.ErrInvalidTag)
+	case "Requested Range Not Satisfiable":
+		return s3err.GetAPIError(s3err.ErrInvalidRange)
+	}
+	return s3err.APIError{
+		Code:           azErr.ErrorCode,
+		Description:    azErr.RawResponse.Status,
+		HTTPStatusCode: azErr.StatusCode,
+	}
+}
+
+func parseMpError(mpErr error) error {
+	err := azureErrToS3Err(mpErr)
+
+	serr, ok := err.(s3err.APIError)
+	if !ok || serr.Code != "NoSuchKey" {
+		return mpErr
+	}
+
+	return s3err.GetAPIError(s3err.ErrNoSuchUpload)
+}

backend/backend.go

@@ -15,53 +15,64 @@
 package backend
 
 import (
+	"bufio"
+	"context"
 	"fmt"
 	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
+	"github.com/versity/versitygw/s3select"
 )
 
-//go:generate moq -out backend_moq_test.go . Backend
 //go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
 type Backend interface {
 	fmt.Stringer
 	Shutdown()
 
-	ListBuckets() (s3response.ListAllMyBucketsResult, error)
-	HeadBucket(bucket string) (*s3.HeadBucketOutput, error)
-	GetBucketAcl(bucket string) ([]byte, error)
-	PutBucket(bucket, owner string) error
-	PutBucketAcl(bucket string, data []byte) error
-	DeleteBucket(bucket string) error
+	// bucket operations
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
+	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
+	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
+	PutBucketAcl(_ context.Context, bucket string, data []byte) error
+	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
 
-	CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
-	CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error)
-	AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
-	ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
-	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error)
-	CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error)
-	PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error)
-	UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error)
+	// multipart operations
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
+	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
+	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
 
-	PutObject(*s3.PutObjectInput) (string, error)
-	HeadObject(bucket, object string) (*s3.HeadObjectOutput, error)
-	GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error)
-	GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
-	CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error)
-	ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
-	ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error)
-	DeleteObject(bucket, object string) error
-	DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error
-	PutObjectAcl(*s3.PutObjectAclInput) error
-	RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error
+	// standard object operations
+	PutObject(context.Context, *s3.PutObjectInput) (string, error)
+	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
 
-	GetTags(bucket, object string) (map[string]string, error)
-	SetTags(bucket, object string, tags map[string]string) error
-	RemoveTags(bucket, object string) error
+	// special case object operations
+	RestoreObject(context.Context, *s3.RestoreObjectInput) error
+	SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
+
+	// object tags operations
+	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
+	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
+	DeleteObjectTagging(_ context.Context, bucket, object string) error
+
+	// non AWS actions
+	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
 type BackendUnsupported struct{}
@@ -75,93 +86,112 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets() (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
+func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketAcl(bucket string) ([]byte, error) {
+func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucket(bucket, owner string) error {
+func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(bucket string) error {
+func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) CreateMultipartUpload(input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
+func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) error {
+func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
-	return s3response.ListMultipartUploadsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
-	return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) {
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
 	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) PutObject(*s3.PutObjectInput) (string, error) {
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
 	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObject(bucket, object string) error {
+func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error {
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
 
-func (BackendUnsupported) GetTags(bucket, object string) (map[string]string, error) {
+func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
+	return func(w *bufio.Writer) {
+		var getProgress s3select.GetProgress
+		progress := input.RequestProgress
+		if progress != nil && *progress.Enabled {
+			getProgress = func() (bytesScanned int64, bytesProcessed int64) {
+				return -1, -1
+			}
+		}
+		mh := s3select.NewMessageHandler(ctx, w, getProgress)
+		apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
+		mh.FinishWithError(apiErr.Code, apiErr.Description)
+	}
+}
+
+func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) SetTags(bucket, object string, tags map[string]string) error {
+func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) RemoveTags(bucket, object string) error {
+func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
+	return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}

backend/backend_test.go

@@ -1,222 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package backend
-
-import (
-	"context"
-	"testing"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3response"
-)
-
-func TestBackend_ListBuckets(t *testing.T) {
-	type args struct {
-		ctx context.Context
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "list-Bucket",
-		c: &BackendMock{
-			ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) {
-				return s3response.ListAllMyBucketsResult{
-					Buckets: s3response.ListAllMyBucketsList{
-						Bucket: []s3response.ListAllMyBucketsEntry{
-							{
-								Name: "t1",
-							},
-						},
-					},
-				}, s3err.GetAPIError(0)
-			},
-		},
-		args: args{
-			ctx: context.Background(),
-		},
-		wantErr: false,
-	}, test{
-		name: "list-Bucket-error",
-		c: &BackendMock{
-			ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) {
-				return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx: context.Background(),
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := tt.c.ListBuckets(); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_HeadBucket(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		BucketName string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "head-buckets-error",
-		c: &BackendMock{
-			HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			BucketName: "b1",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := tt.c.HeadBucket(tt.args.BucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.HeadBucket() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_GetBucketAcl(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		bucketName string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "get bucket acl error",
-		c: &BackendMock{
-			GetBucketAclFunc: func(bucket string) ([]byte, error) {
-				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			bucketName: "b1",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := tt.c.GetBucketAcl(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.GetBucketAcl() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_PutBucket(t *testing.T) {
-	type args struct {
-		ctx         context.Context
-		bucketName  string
-		bucketOwner string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "put bucket ",
-		c: &BackendMock{
-			PutBucketFunc: func(bucket, owner string) error {
-				return s3err.GetAPIError(0)
-			},
-		},
-		args: args{
-			ctx:         context.Background(),
-			bucketName:  "b1",
-			bucketOwner: "owner",
-		},
-		wantErr: false,
-	}, test{
-		name: "put bucket error",
-		c: &BackendMock{
-			PutBucketFunc: func(bucket, owner string) error {
-				return s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:         context.Background(),
-			bucketName:  "b2",
-			bucketOwner: "owner",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.c.PutBucket(tt.args.bucketName, tt.args.bucketOwner); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.PutBucket() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-func TestBackend_DeleteBucket(t *testing.T) {
-	type args struct {
-		ctx        context.Context
-		bucketName string
-	}
-	type test struct {
-		name    string
-		c       Backend
-		args    args
-		wantErr bool
-	}
-	var tests []test
-	tests = append(tests, test{
-		name: "Delete Bucket Error",
-		c: &BackendMock{
-			DeleteBucketFunc: func(bucket string) error {
-				return s3err.GetAPIError(s3err.ErrNotImplemented)
-			},
-		},
-		args: args{
-			ctx:        context.Background(),
-			bucketName: "b1",
-		},
-		wantErr: true,
-	})
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.c.DeleteBucket(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
-				t.Errorf("Backend.DeleteBucket() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}

backend/common.go

@@ -17,7 +17,6 @@ package backend
 import (
 	"crypto/md5"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"io/fs"
 	"strconv"
@@ -25,6 +24,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
 
@@ -55,40 +55,51 @@ func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
 
-func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
+var (
+	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
+)
+
+// ParseRange parses input range header and returns startoffset, length, and
+// error. If no endoffset specified, then length is set to -1.
+func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, file.Size(), nil
+		return 0, fi.Size(), nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
 
 	if len(rangeKv) < 2 {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
 	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 2 {
-		return 0, 0, errors.New("invalid range parameter")
+	if len(bRange) < 1 || len(bRange) > 2 {
+		return 0, 0, errInvalidRange
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
 	if err != nil {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
-	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	endOffset := int64(-1)
+	if len(bRange) == 1 || bRange[1] == "" {
+		return startOffset, endOffset, nil
+	}
+
+	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
 	if err != nil {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
 	if endOffset < startOffset {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
-	return int64(startOffset), int64(endOffset - startOffset + 1), nil
+	return startOffset, endOffset - startOffset + 1, nil
 }
 
-func GetMultipartMD5(parts []types.Part) string {
+func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
 		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)

backend/s3proxy/client.go

@@ -0,0 +1,79 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+)
+
+func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
+	cfg, err := s.getConfig(ctx, s.access, s.secret)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg), nil
+}
+
+func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.disableChecksum {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(ctx, opts...)
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}

backend/s3proxy/s3.go

@@ -0,0 +1,545 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+const aclKey string = "versitygwAcl"
+
+type S3Proxy struct {
+	backend.BackendUnsupported
+
+	client *s3.Client
+
+	access          string
+	secret          string
+	endpoint        string
+	awsRegion       string
+	disableChecksum bool
+	sslSkipVerify   bool
+	debug           bool
+}
+
+func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
+	s := &S3Proxy{
+		access:          access,
+		secret:          secret,
+		endpoint:        endpoint,
+		awsRegion:       region,
+		disableChecksum: disableChecksum,
+		sslSkipVerify:   sslSkipVerify,
+		debug:           debug,
+	}
+	client, err := s.getClientWithCtx(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	s.client = client
+	return s, nil
+}
+
+func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, handleError(err)
+	}
+
+	var buckets []s3response.ListAllMyBucketsEntry
+	for _, b := range output.Buckets {
+		buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+			Name:         *b.Name,
+			CreationDate: *b.CreationDate,
+		})
+	}
+
+	return s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID: *output.Owner.ID,
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: buckets,
+		},
+	}, nil
+}
+
+func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	out, err := s.client.HeadBucket(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
+	_, err := s.client.CreateBucket(ctx, input)
+	if err != nil {
+		return handleError(err)
+	}
+
+	var tagSet []types.Tag
+	tagSet = append(tagSet, types.Tag{
+		Key:   backend.GetStringPtr(aclKey),
+		Value: backend.GetStringPtr(base64Encode(acl)),
+	})
+
+	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
+		Bucket: input.Bucket,
+		Tagging: &types.Tagging{
+			TagSet: tagSet,
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	_, err := s.client.DeleteBucket(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	out, err := s.client.CreateMultipartUpload(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	out, err := s.client.CompleteMultipartUpload(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	_, err := s.client.AbortMultipartUpload(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	output, err := s.client.ListMultipartUploads(ctx, input)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, handleError(err)
+	}
+
+	var uploads []s3response.Upload
+	for _, u := range output.Uploads {
+		uploads = append(uploads, s3response.Upload{
+			Key:      *u.Key,
+			UploadID: *u.UploadId,
+			Initiator: s3response.Initiator{
+				ID:          *u.Initiator.ID,
+				DisplayName: *u.Initiator.DisplayName,
+			},
+			Owner: s3response.Owner{
+				ID:          *u.Owner.ID,
+				DisplayName: *u.Owner.DisplayName,
+			},
+			StorageClass: string(u.StorageClass),
+			Initiated:    u.Initiated.Format(backend.RFC3339TimeFormat),
+		})
+	}
+
+	var cps []s3response.CommonPrefix
+	for _, c := range output.CommonPrefixes {
+		cps = append(cps, s3response.CommonPrefix{
+			Prefix: *c.Prefix,
+		})
+	}
+
+	return s3response.ListMultipartUploadsResult{
+		Bucket:             *output.Bucket,
+		KeyMarker:          *output.KeyMarker,
+		UploadIDMarker:     *output.UploadIdMarker,
+		NextKeyMarker:      *output.NextKeyMarker,
+		NextUploadIDMarker: *output.NextUploadIdMarker,
+		Delimiter:          *output.Delimiter,
+		Prefix:             *output.Prefix,
+		EncodingType:       string(output.EncodingType),
+		MaxUploads:         int(*output.MaxUploads),
+		IsTruncated:        *output.IsTruncated,
+		Uploads:            uploads,
+		CommonPrefixes:     cps,
+	}, nil
+}
+
+func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	output, err := s.client.ListParts(ctx, input)
+	if err != nil {
+		return s3response.ListPartsResult{}, handleError(err)
+	}
+
+	var parts []s3response.Part
+	for _, p := range output.Parts {
+		parts = append(parts, s3response.Part{
+			PartNumber:   int(*p.PartNumber),
+			LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
+			ETag:         *p.ETag,
+			Size:         *p.Size,
+		})
+	}
+	pnm, err := strconv.Atoi(*output.PartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse part number marker: %w", err)
+	}
+
+	npmn, err := strconv.Atoi(*output.NextPartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse next part number marker: %w", err)
+	}
+
+	return s3response.ListPartsResult{
+		Bucket:   *output.Bucket,
+		Key:      *output.Key,
+		UploadID: *output.UploadId,
+		Initiator: s3response.Initiator{
+			ID:          *output.Initiator.ID,
+			DisplayName: *output.Initiator.DisplayName,
+		},
+		Owner: s3response.Owner{
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
+		},
+		StorageClass:         string(output.StorageClass),
+		PartNumberMarker:     pnm,
+		NextPartNumberMarker: npmn,
+		MaxParts:             int(*output.MaxParts),
+		IsTruncated:          *output.IsTruncated,
+		Parts:                parts,
+	}, nil
+}
+
+func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
+	if err != nil {
+		return "", handleError(err)
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	output, err := s.client.UploadPartCopy(ctx, input)
+	if err != nil {
+		return s3response.CopyObjectResult{}, handleError(err)
+	}
+
+	return s3response.CopyObjectResult{
+		LastModified: *output.CopyPartResult.LastModified,
+		ETag:         *output.CopyPartResult.ETag,
+	}, nil
+}
+
+func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
+	if err != nil {
+		return "", handleError(err)
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	out, err := s.client.HeadObject(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+	output, err := s.client.GetObject(ctx, input)
+	if err != nil {
+		return nil, handleError(err)
+	}
+	defer output.Body.Close()
+
+	_, err = io.Copy(w, output.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}
+
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	out, err := s.client.GetObjectAttributes(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	out, err := s.client.CopyObject(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	out, err := s.client.ListObjects(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	out, err := s.client.ListObjectsV2(ctx, input)
+	return out, handleError(err)
+}
+
+func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	_, err := s.client.DeleteObject(ctx, input)
+	return handleError(err)
+}
+
+func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	if len(input.Delete.Objects) == 0 {
+		input.Delete.Objects = []types.ObjectIdentifier{}
+	}
+
+	output, err := s.client.DeleteObjects(ctx, input)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, handleError(err)
+	}
+
+	return s3response.DeleteObjectsResult{
+		Deleted: output.Deleted,
+		Error:   output.Errors,
+	}, nil
+}
+
+func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+		Bucket: input.Bucket,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	for _, tag := range tagout.TagSet {
+		if *tag.Key == aclKey {
+			acl, err := base64Decode(*tag.Value)
+			if err != nil {
+				return nil, handleError(err)
+			}
+			return acl, nil
+		}
+	}
+
+	return []byte{}, nil
+}
+
+func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
+	tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+		Bucket: &bucket,
+	})
+	if err != nil {
+		return handleError(err)
+	}
+
+	var found bool
+	for i, tag := range tagout.TagSet {
+		if *tag.Key == aclKey {
+			tagout.TagSet[i] = types.Tag{
+				Key:   backend.GetStringPtr(aclKey),
+				Value: backend.GetStringPtr(base64Encode(data)),
+			}
+			found = true
+			break
+		}
+	}
+	if !found {
+		tagout.TagSet = append(tagout.TagSet, types.Tag{
+			Key:   backend.GetStringPtr(aclKey),
+			Value: backend.GetStringPtr(base64Encode(data)),
+		})
+	}
+
+	_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
+		Bucket: &bucket,
+		Tagging: &types.Tagging{
+			TagSet: tagout.TagSet,
+		},
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	tagging := &types.Tagging{
+		TagSet: []types.Tag{},
+	}
+	for key, val := range tags {
+		tagging.TagSet = append(tagging.TagSet, types.Tag{
+			Key:   &key,
+			Value: &val,
+		})
+	}
+
+	_, err := s.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
+		Bucket:  &bucket,
+		Key:     &object,
+		Tagging: tagging,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	if err != nil {
+		return nil, handleError(err)
+	}
+
+	tags := make(map[string]string)
+	for _, el := range output.TagSet {
+		tags[*el.Key] = *el.Value
+	}
+
+	return tags, nil
+}
+
+func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	return handleError(err)
+}
+
+func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	if resp.StatusCode > 300 {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		return fmt.Errorf(string(body))
+	}
+
+	return nil
+}
+
+func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return []s3response.Bucket{}, err
+	}
+	defer resp.Body.Close()
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return []s3response.Bucket{}, err
+	}
+
+	return buckets, nil
+}
+
+func handleError(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		apiErr := s3err.APIError{
+			Code:        ae.ErrorCode(),
+			Description: ae.ErrorMessage(),
+		}
+		var re *awshttp.ResponseError
+		if errors.As(err, &re) {
+			apiErr.HTTPStatusCode = re.Response.StatusCode
+		}
+		return apiErr
+	}
+	return err
+}
+
+func base64Encode(input []byte) string {
+	return base64.StdEncoding.EncodeToString(input)
+}
+
+func base64Decode(encoded string) ([]byte, error) {
+	decoded, err := base64.StdEncoding.DecodeString(encoded)
+	if err != nil {
+		return nil, err
+	}
+	return decoded, nil
+}

backend/scoutfs/scoutfs.go

@@ -15,6 +15,7 @@
 package scoutfs
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/json"
 	"errors"
@@ -29,7 +30,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
-	"github.com/versity/scoutfs-go"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
@@ -114,7 +114,12 @@ func (*ScoutFS) String() string {
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
-func (s *ScoutFS) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
+func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+	uploadID := *input.UploadId
+	parts := input.MultipartUpload.Parts
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -160,7 +165,9 @@ func (s *ScoutFS) CompleteMultipartUpload(bucket, object, uploadID string, parts
 		if err != nil {
 			etag = ""
 		}
-		parts[i].ETag = &etag
+		if etag != *parts[i].ETag {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
 	}
 
 	// use totalsize=0 because we wont be writing to the file, only moving
@@ -180,7 +187,7 @@ func (s *ScoutFS) CompleteMultipartUpload(bucket, object, uploadID string, parts
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = scoutfs.MoveData(pf, f.f)
+		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
 			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
@@ -347,7 +354,10 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
 	return nil
 }
 
-func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
+func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -381,7 +391,7 @@ func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error
 
 		// Check if there are any offline exents associated with this file.
 		// If so, we will set storage class to glacier.
-		st, err := scoutfs.StatMore(objPath)
+		st, err := statMore(objPath)
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		}
@@ -405,8 +415,10 @@ func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error
 		}
 	}
 
+	contentLength := fi.Size()
+
 	return &s3.HeadObjectOutput{
-		ContentLength:   fi.Size(),
+		ContentLength:   &contentLength,
 		ContentType:     &contentType,
 		ContentEncoding: &contentEncoding,
 		ETag:            &etag,
@@ -417,7 +429,11 @@ func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error
 	}, nil
 }
 
-func (s *ScoutFS) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+	bucket := *input.Bucket
+	object := *input.Key
+	acceptRange := *input.Range
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -440,15 +456,18 @@ func (s *ScoutFS) GetObject(bucket, object, acceptRange string, writer io.Writer
 		return nil, err
 	}
 
+	if length == -1 {
+		length = fi.Size() - startOffset + 1
+	}
+
 	if startOffset+length > fi.Size() {
-		// TODO: is ErrInvalidRequest correct here?
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
-		st, err := scoutfs.StatMore(objPath)
+		st, err := statMore(objPath)
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		}
@@ -490,15 +509,17 @@ func (s *ScoutFS) GetObject(bucket, object, acceptRange string, writer io.Writer
 		return nil, fmt.Errorf("get object tags: %w", err)
 	}
 
+	tagCount := int32(len(tags))
+
 	return &s3.GetObjectOutput{
 		AcceptRanges:    &acceptRange,
-		ContentLength:   length,
+		ContentLength:   &length,
 		ContentEncoding: &contentEncoding,
 		ContentType:     &contentType,
 		ETag:            &etag,
 		LastModified:    backend.GetTimePtr(fi.ModTime()),
 		Metadata:        userMetaData,
-		TagCount:        int32(len(tags)),
+		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
 	}, nil
 }
@@ -524,7 +545,28 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
 	return tags, nil
 }
 
-func (s *ScoutFS) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
+func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	bucket := *input.Bucket
+	prefix := ""
+	if input.Prefix != nil {
+		prefix = *input.Prefix
+	}
+	marker := ""
+	if input.Marker != nil {
+		marker = *input.Marker
+	}
+	delim := ""
+	if input.Delimiter != nil {
+		delim = *input.Delimiter
+	}
+	maxkeys := int32(0)
+	if input.MaxKeys != nil {
+		maxkeys = *input.MaxKeys
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -544,16 +586,37 @@ func (s *ScoutFS) ListObjects(bucket, prefix, marker, delim string, maxkeys int)
 		CommonPrefixes: results.CommonPrefixes,
 		Contents:       results.Objects,
 		Delimiter:      &delim,
-		IsTruncated:    results.Truncated,
+		IsTruncated:    &results.Truncated,
 		Marker:         &marker,
-		MaxKeys:        int32(maxkeys),
+		MaxKeys:        &maxkeys,
 		Name:           &bucket,
 		NextMarker:     &results.NextMarker,
 		Prefix:         &prefix,
 	}, nil
 }
 
-func (s *ScoutFS) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
+func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	bucket := *input.Bucket
+	prefix := ""
+	if input.Prefix != nil {
+		prefix = *input.Prefix
+	}
+	marker := ""
+	if input.ContinuationToken != nil {
+		marker = *input.ContinuationToken
+	}
+	delim := ""
+	if input.Delimiter != nil {
+		delim = *input.Delimiter
+	}
+	maxkeys := int32(0)
+	if input.MaxKeys != nil {
+		maxkeys = *input.MaxKeys
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -563,7 +626,7 @@ func (s *ScoutFS) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys in
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
 		s.fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
@@ -573,9 +636,9 @@ func (s *ScoutFS) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys in
 		CommonPrefixes:        results.CommonPrefixes,
 		Contents:              results.Objects,
 		Delimiter:             &delim,
-		IsTruncated:           results.Truncated,
+		IsTruncated:           &results.Truncated,
 		ContinuationToken:     &marker,
-		MaxKeys:               int32(maxkeys),
+		MaxKeys:               &maxkeys,
 		Name:                  &bucket,
 		NextContinuationToken: &results.NextMarker,
 		Prefix:                &prefix,
@@ -636,7 +699,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if s.glaciermode {
 			// Check if there are any offline exents associated with this file.
 			// If so, we will return the InvalidObjectState error.
-			st, err := scoutfs.StatMore(objPath)
+			st, err := statMore(objPath)
 			if errors.Is(err, fs.ErrNotExist) {
 				return types.Object{}, backend.ErrSkipObj
 			}
@@ -648,11 +711,13 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 			}
 		}
 
+		size := fi.Size()
+
 		return types.Object{
 			ETag:         &etag,
 			Key:          &path,
 			LastModified: backend.GetTimePtr(fi.ModTime()),
-			Size:         fi.Size(),
+			Size:         &size,
 			StorageClass: sc,
 		}, nil
 	}
@@ -660,7 +725,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 
 // RestoreObject will set stage request on file if offline and do nothing if
 // file is online
-func (s *ScoutFS) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
+func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
+	bucket := *input.Bucket
+	object := *input.Key
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)

backend/scoutfs/scoutfs_compat.go

@@ -12,6 +12,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build linux && amd64
+
 package scoutfs
 
 import (
@@ -26,6 +28,7 @@ import (
 
 	"golang.org/x/sys/unix"
 
+	"github.com/versity/scoutfs-go"
 	"github.com/versity/versitygw/backend/posix"
 )
 
@@ -182,3 +185,25 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
+
+func moveData(from *os.File, to *os.File) error {
+	return scoutfs.MoveData(from, to)
+}
+
+func statMore(path string) (stat, error) {
+	st, err := scoutfs.StatMore(path)
+	if err != nil {
+		return stat{}, err
+	}
+	var s stat
+
+	s.Meta_seq = st.Meta_seq
+	s.Data_seq = st.Data_seq
+	s.Data_version = st.Data_version
+	s.Online_blocks = st.Online_blocks
+	s.Offline_blocks = st.Offline_blocks
+	s.Crtime_sec = st.Crtime_sec
+	s.Crtime_nsec = st.Crtime_nsec
+
+	return s, nil
+}

backend/scoutfs/scoutfs_incompat.go

@@ -12,6 +12,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build !(linux && amd64)
+
 package scoutfs
 
 import (
@@ -46,3 +48,11 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 
 func (tmp *tmpfile) cleanup() {
 }
+
+func moveData(from *os.File, to *os.File) error {
+	return errNotSupported
+}
+
+func statMore(path string) (stat, error) {
+	return stat{}, errNotSupported
+}

backend/scoutfs/stat.go

@@ -0,0 +1,25 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package scoutfs
+
+type stat struct {
+	Meta_seq       uint64
+	Data_seq       uint64
+	Data_version   uint64
+	Online_blocks  uint64
+	Offline_blocks uint64
+	Crtime_sec     uint64
+	Crtime_nsec    uint32
+}

backend/walk.go

@@ -38,7 +38,7 @@ var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
 	var objects []types.Object
 
@@ -47,7 +47,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
 		pastMarker = true
 	}
 
-	var pastMax bool
+	pastMax := max == 0
 	var newMarker string
 	var truncated bool
 
@@ -55,30 +55,30 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
 		if err != nil {
 			return err
 		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
 
 		if pastMax {
-			newMarker = path
-			truncated = true
+			if len(objects) != 0 {
+				newMarker = *objects[len(objects)-1].Key
+				truncated = true
+			}
 			return fs.SkipAll
 		}
 
 		if d.IsDir() {
-			// Ignore the root directory
-			if path == "." {
-				return nil
-			}
-
-			if contains(d.Name(), skipdirs) {
-				return fs.SkipDir
-			}
-
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
 			// match this prefix. Make sure to append the / at the end of
 			// directories since this is implied as a directory path name.
 			// If path is a prefix of prefix, then path could still be
-			// building to match. So only skip if path isnt a prefix of prefix
-			// and prefix isnt a prefix of path.
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
 			if prefix != "" &&
 				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
 				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
@@ -106,10 +106,13 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
 		}
 
 		if !pastMarker {
-			if path != marker {
+			if path == marker {
+				pastMarker = true
+				return nil
+			}
+			if path < marker {
 				return nil
 			}
-			pastMarker = true
 		}
 
 		// If object doesn't have prefix, don't include in results.
@@ -129,7 +132,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
 			}
 			objects = append(objects, obj)
 
-			if max > 0 && (len(objects)+len(cpmap)) == max {
+			if max > 0 && (len(objects)+len(cpmap)) == int(max) {
 				pastMax = true
 			}
 
@@ -168,7 +171,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
 				return fmt.Errorf("file to object %q: %w", path, err)
 			}
 			objects = append(objects, obj)
-			if (len(objects) + len(cpmap)) == max {
+			if (len(objects) + len(cpmap)) == int(max) {
 				pastMax = true
 			}
 			return nil
@@ -178,7 +181,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge
 		// These are abstractly a "directory", so need to include the
 		// delimiter at the end.
 		cpmap[prefix+before+delimiter] = struct{}{}
-		if (len(objects) + len(cpmap)) == max {
+		if (len(objects) + len(cpmap)) == int(max) {
 			pastMax = true
 		}
 

backend/walk_test.go

@@ -55,11 +55,13 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
 		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 	}
 
+	size := fi.Size()
+
 	return types.Object{
 		ETag:         &etag,
 		Key:          &path,
 		LastModified: backend.GetTimePtr(fi.ModTime()),
-		Size:         fi.Size(),
+		Size:         &size,
 	}, nil
 }
 

cmd/versitygw/admin.go

@@ -15,32 +15,35 @@
 package main
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"text/tabwriter"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
 )
 
 var (
-	adminAccess string
-	adminSecret string
-	adminRegion string
+	adminAccess   string
+	adminSecret   string
+	adminEndpoint string
 )
 
 func adminCommand() *cli.Command {
 	return &cli.Command{
-		Name:  "admin",
-		Usage: "admin CLI tool",
-		Description: `admin CLI tool for interacting with admin api.
-		Here is the available api list:
-		create-user
-		`,
+		Name:        "admin",
+		Usage:       "admin CLI tool",
+		Description: `Admin CLI tool for interacting with admin APIs.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:   "create-user",
@@ -49,13 +52,13 @@ func adminCommand() *cli.Command {
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:     "access",
-						Usage:    "access value for the new user",
+						Usage:    "access key id for the new user",
 						Required: true,
 						Aliases:  []string{"a"},
 					},
 					&cli.StringFlag{
 						Name:     "secret",
-						Usage:    "secret value for the new user",
+						Usage:    "secret access key for the new user",
 						Required: true,
 						Aliases:  []string{"s"},
 					},
@@ -65,11 +68,20 @@ func adminCommand() *cli.Command {
 						Required: true,
 						Aliases:  []string{"r"},
 					},
-					&cli.StringFlag{
-						Name:    "region",
-						Usage:   "s3 region string for the user",
-						Value:   "us-east-1",
-						Aliases: []string{"rg"},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+					&cli.IntFlag{
+						Name:    "project-id",
+						Usage:   "projectID for the new user",
+						Aliases: []string{"pi"},
 					},
 				},
 			},
@@ -80,62 +92,109 @@ func adminCommand() *cli.Command {
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:     "access",
-						Usage:    "access value for the user to be deleted",
+						Usage:    "access key id of the user to be deleted",
 						Required: true,
 						Aliases:  []string{"a"},
 					},
 				},
 			},
+			{
+				Name:   "list-users",
+				Usage:  "List all the gateway users",
+				Action: listUsers,
+			},
+			{
+				Name:  "change-bucket-owner",
+				Usage: "Changes the bucket owner",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "bucket",
+						Usage:    "the bucket name to change the owner",
+						Required: true,
+						Aliases:  []string{"b"},
+					},
+					&cli.StringFlag{
+						Name:     "owner",
+						Usage:    "the user access key id, who should be the bucket owner",
+						Required: true,
+						Aliases:  []string{"o"},
+					},
+				},
+				Action: changeBucketOwner,
+			},
+			{
+				Name:   "list-buckets",
+				Usage:  "Lists all the gateway buckets and owners.",
+				Action: listBuckets,
+			},
 		},
 		Flags: []cli.Flag{
 			// TODO: create a configuration file for this
 			&cli.StringFlag{
-				Name:        "adminAccess",
-				Usage:       "admin access account",
+				Name:        "access",
+				Usage:       "admin access key id",
 				EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
-				Aliases:     []string{"aa"},
+				Aliases:     []string{"a"},
+				Required:    true,
 				Destination: &adminAccess,
 			},
 			&cli.StringFlag{
-				Name:        "adminSecret",
+				Name:        "secret",
 				Usage:       "admin secret access key",
 				EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
-				Aliases:     []string{"as"},
+				Aliases:     []string{"s"},
+				Required:    true,
 				Destination: &adminSecret,
 			},
 			&cli.StringFlag{
-				Name:        "adminRegion",
-				Usage:       "s3 region string",
-				Value:       "us-east-1",
-				Destination: &adminRegion,
-				Aliases:     []string{"ar"},
+				Name:        "endpoint-url",
+				Usage:       "admin apis endpoint url",
+				EnvVars:     []string{"ADMIN_ENDPOINT_URL"},
+				Aliases:     []string{"er"},
+				Required:    true,
+				Destination: &adminEndpoint,
 			},
 		},
 	}
 }
 
 func createUser(ctx *cli.Context) error {
-	access, secret, role, region := ctx.String("access"), ctx.String("secret"), ctx.String("role"), ctx.String("region")
-	if access == "" || secret == "" || region == "" {
+	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
+	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
+	if access == "" || secret == "" {
 		return fmt.Errorf("invalid input parameters for the new user")
 	}
 	if role != "admin" && role != "user" {
 		return fmt.Errorf("invalid input parameter for role")
 	}
 
-	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:7070/create-user?access=%v&secret=%v&role=%v&region=%v", access, secret, role, region), nil)
+	acc := auth.Account{
+		Access:    access,
+		Secret:    secret,
+		Role:      role,
+		UserID:    userID,
+		GroupID:   groupID,
+		ProjectID: projectID,
+	}
+
+	accJson, err := json.Marshal(acc)
+	if err != nil {
+		return fmt.Errorf("failed to parse user data: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256([]byte{})
+	hashedPayload := sha256.Sum256(accJson)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -151,8 +210,9 @@ func createUser(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
-	fmt.Printf("%s", body)
+	fmt.Printf("%s\n", body)
 
 	return nil
 }
@@ -163,7 +223,7 @@ func deleteUser(ctx *cli.Context) error {
 		return fmt.Errorf("invalid input parameter for the new user")
 	}
 
-	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("http://localhost:7070/delete-user?access=%v", access), nil)
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
@@ -175,7 +235,7 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
@@ -191,8 +251,166 @@ func deleteUser(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
-	fmt.Printf("%s", body)
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
+func listUsers(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
+		return err
+	}
+	fmt.Println(accs)
+
+	printAcctTable(accs)
+
+	return nil
+}
+
+const (
+	// account table formatting
+	minwidth int  = 2   // minimal cell width including any padding
+	tabwidth int  = 0   // width of tab characters (equivalent number of spaces)
+	padding  int  = 2   // padding added to a cell before computing its width
+	padchar  byte = ' ' // ASCII char used for padding
+	flags    uint = 0   // formatting control flags
+)
+
+func printAcctTable(accs []auth.Account) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
+	for _, acc := range accs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func changeBucketOwner(ctx *cli.Context) error {
+	bucket, owner := ctx.String("bucket"), ctx.String("owner")
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Println(string(body))
+
+	return nil
+}
+
+func printBuckets(buckets []s3response.Bucket) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Bucket\tOwner")
+	fmt.Fprintln(w, "-------\t----")
+	for _, acc := range buckets {
+		fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func listBuckets(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf(string(body))
+	}
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return err
+	}
+
+	printBuckets(buckets)
 
 	return nil
 }

cmd/versitygw/azure.go

@@ -0,0 +1,74 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/azure"
+)
+
+var (
+	azAccount, azKey, azServiceURL, azSASToken string
+)
+
+func azureCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "azure",
+		Usage:       "azure blob storage backend",
+		Description: `direct translation from s3 objects to azure blobs`,
+		Action:      runAzure,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "account",
+				Usage:       "azure account name",
+				EnvVars:     []string{"AZ_ACCOUNT_NAME"},
+				Aliases:     []string{"a"},
+				Destination: &azAccount,
+			},
+			&cli.StringFlag{
+				Name:        "access-key",
+				Usage:       "azure account key",
+				EnvVars:     []string{"AZ_ACCESS_KEY"},
+				Aliases:     []string{"k"},
+				Destination: &azKey,
+			},
+			&cli.StringFlag{
+				Name:        "sas-token",
+				Usage:       "azure blob storage SAS token",
+				EnvVars:     []string{"AZ_SAS_TOKEN"},
+				Aliases:     []string{"st"},
+				Destination: &azSASToken,
+			},
+			&cli.StringFlag{
+				Name:        "url",
+				Usage:       "azure service URL",
+				EnvVars:     []string{"AZ_ENDPOINT"},
+				Aliases:     []string{"u"},
+				Destination: &azServiceURL,
+			},
+		},
+	}
+}
+
+func runAzure(ctx *cli.Context) error {
+	be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
+	if err != nil {
+		return fmt.Errorf("init azure: %w", err)
+	}
+
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/gateway_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/versity/versitygw/backend/posix"
+	"github.com/versity/versitygw/integration"
+)
+
+const (
+	tdir = "tempdir"
+)
+
+var (
+	wg sync.WaitGroup
+)
+
+func initEnv(dir string) {
+	// both
+	debug = true
+	region = "us-east-1"
+
+	// server
+	rootUserAccess = "user"
+	rootUserSecret = "pass"
+	iamDir = dir
+	port = "127.0.0.1:7070"
+
+	// client
+	awsID = "user"
+	awsSecret = "pass"
+	endpoint = "http://127.0.0.1:7070"
+}
+
+func initPosix(ctx context.Context) {
+	path, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("get current directory: %v", err)
+	}
+
+	tempdir := filepath.Join(path, tdir)
+	initEnv(tempdir)
+
+	err = os.RemoveAll(tempdir)
+	if err != nil {
+		log.Fatalf("remove temp directory: %v", err)
+	}
+
+	err = os.Mkdir(tempdir, 0755)
+	if err != nil {
+		log.Fatalf("make temp directory: %v", err)
+	}
+
+	be, err := posix.New(tempdir)
+	if err != nil {
+		log.Fatalf("init posix: %v", err)
+	}
+
+	wg.Add(1)
+	go func() {
+		err = runGateway(ctx, be)
+		if err != nil && err != context.Canceled {
+			log.Fatalf("run gateway: %v", err)
+		}
+
+		err := os.RemoveAll(tempdir)
+		if err != nil {
+			log.Fatalf("remove temp directory: %v", err)
+		}
+		wg.Done()
+	}()
+}
+
+func TestIntegration(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	initPosix(ctx)
+
+	opts := []integration.Option{
+		integration.WithAccess(awsID),
+		integration.WithSecret(awsSecret),
+		integration.WithRegion(region),
+		integration.WithEndpoint(endpoint),
+	}
+	if debug {
+		opts = append(opts, integration.WithDebug())
+	}
+
+	s := integration.NewS3Conf(opts...)
+
+	// replace below with desired test
+	err := integration.HeadBucket_non_existing_bucket(s)
+	if err != nil {
+		t.Error(err)
+	}
+
+	cancel()
+	wg.Wait()
+}

cmd/versitygw/main.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"log"
@@ -26,15 +27,34 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api"
 	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3event"
+	"github.com/versity/versitygw/s3log"
 )
 
 var (
-	port              string
-	rootUserAccess    string
-	rootUserSecret    string
-	region            string
-	certFile, keyFile string
-	debug             bool
+	port, admPort                          string
+	rootUserAccess                         string
+	rootUserSecret                         string
+	region                                 string
+	admCertFile, admKeyFile                string
+	certFile, keyFile                      string
+	kafkaURL, kafkaTopic, kafkaKey         string
+	natsURL, natsTopic                     string
+	logWebhookURL                          string
+	accessLog                              string
+	debug                                  bool
+	quiet                                  bool
+	iamDir                                 string
+	ldapURL, ldapBindDN, ldapPassword      string
+	ldapQueryBase, ldapObjClasses          string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	s3IamAccess, s3IamSecret               string
+	s3IamRegion, s3IamBucket               string
+	s3IamEndpoint                          string
+	s3IamSslNoVerify, s3IamDebug           bool
+	iamCacheDisable                        bool
+	iamCacheTTL                            int
+	iamCachePrune                          int
 )
 
 var (
@@ -47,16 +67,27 @@ var (
 )
 
 func main() {
+	setupSignalHandler()
+
 	app := initApp()
 
 	app.Commands = []*cli.Command{
 		posixCommand(),
 		scoutfsCommand(),
+		s3Command(),
+		azureCommand(),
 		adminCommand(),
 		testCommand(),
 	}
 
-	if err := app.Run(os.Args); err != nil {
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		<-sigDone
+		fmt.Fprintf(os.Stderr, "terminating signal caught, shutting down\n")
+		cancel()
+	}()
+
+	if err := app.RunContext(ctx, os.Args); err != nil {
 		log.Fatal(err)
 	}
 }
@@ -126,19 +157,182 @@ func initFlags() []cli.Flag {
 			Usage:       "TLS key file",
 			Destination: &keyFile,
 		},
+		&cli.StringFlag{
+			Name:        "admin-port",
+			Usage:       "gateway admin server listen address <ip>:<port> or :<port>",
+			Destination: &admPort,
+			Aliases:     []string{"ap"},
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert",
+			Usage:       "TLS cert file for admin server",
+			Destination: &admCertFile,
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert-key",
+			Usage:       "TLS key file for admin server",
+			Destination: &admKeyFile,
+		},
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
 			Destination: &debug,
 		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "silence stdout request logging output",
+			Destination: &quiet,
+			Aliases:     []string{"q"},
+		},
+		&cli.StringFlag{
+			Name:        "access-log",
+			Usage:       "enable server access logging to specified file",
+			EnvVars:     []string{"LOGFILE"},
+			Destination: &accessLog,
+		},
+		&cli.StringFlag{
+			Name:        "log-webhook-url",
+			Usage:       "webhook url to send the audit logs",
+			EnvVars:     []string{"WEBHOOK"},
+			Destination: &logWebhookURL,
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-url",
+			Usage:       "kafka server url to send the bucket notifications.",
+			Destination: &kafkaURL,
+			Aliases:     []string{"eku"},
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-topic",
+			Usage:       "kafka server pub-sub topic to send the bucket notifications to",
+			Destination: &kafkaTopic,
+			Aliases:     []string{"ekt"},
+		},
+		&cli.StringFlag{
+			Name:        "event-kafka-key",
+			Usage:       "kafka server put-sub topic key to send the bucket notifications to",
+			Destination: &kafkaKey,
+			Aliases:     []string{"ekk"},
+		},
+		&cli.StringFlag{
+			Name:        "event-nats-url",
+			Usage:       "nats server url to send the bucket notifications",
+			Destination: &natsURL,
+			Aliases:     []string{"enu"},
+		},
+		&cli.StringFlag{
+			Name:        "event-nats-topic",
+			Usage:       "nats server pub-sub topic to send the bucket notifications to",
+			Destination: &natsTopic,
+			Aliases:     []string{"ent"},
+		},
+		&cli.StringFlag{
+			Name:        "iam-dir",
+			Usage:       "if defined, run internal iam service within this directory",
+			Destination: &iamDir,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-url",
+			Usage:       "ldap server url to store iam data",
+			Destination: &ldapURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-dn",
+			Usage:       "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
+			Destination: &ldapBindDN,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-pass",
+			Usage:       "ldap server user password",
+			Destination: &ldapPassword,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-query-base",
+			Usage:       "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
+			Destination: &ldapQueryBase,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-object-classes",
+			Usage:       "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
+			Destination: &ldapObjClasses,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-access-atr",
+			Usage:       "ldap server user access key id attribute name",
+			Destination: &ldapAccessAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-secret-atr",
+			Usage:       "ldap server user secret access key attribute name",
+			Destination: &ldapSecAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-role-atr",
+			Usage:       "ldap server user role attribute name",
+			Destination: &ldapRoleAtr,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-access",
+			Usage:       "s3 IAM access key",
+			Destination: &s3IamAccess,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-secret",
+			Usage:       "s3 IAM secret key",
+			Destination: &s3IamSecret,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-region",
+			Usage:       "s3 IAM region",
+			Destination: &s3IamRegion,
+			Value:       "us-east-1",
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-bucket",
+			Usage:       "s3 IAM bucket",
+			Destination: &s3IamBucket,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-endpoint",
+			Usage:       "s3 IAM endpoint",
+			Destination: &s3IamEndpoint,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-noverify",
+			Usage:       "s3 IAM disable ssl verification",
+			Destination: &s3IamSslNoVerify,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-debug",
+			Usage:       "s3 IAM debug output",
+			Destination: &s3IamDebug,
+		},
+		&cli.BoolFlag{
+			Name:        "iam-cache-disable",
+			Usage:       "disable local iam cache",
+			Destination: &iamCacheDisable,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-ttl",
+			Usage:       "local iam cache entry ttl (seconds)",
+			Value:       120,
+			Destination: &iamCacheTTL,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-prune",
+			Usage:       "local iam cache cleanup interval (seconds)",
+			Value:       3600,
+			Destination: &iamCachePrune,
+		},
 	}
 }
 
-func runGateway(be backend.Backend, s auth.Storer) error {
+func runGateway(ctx context.Context, be backend.Backend) error {
 	app := fiber.New(fiber.Config{
-		AppName:      "versitygw",
-		ServerHeader: "VERSITYGW",
-		BodyLimit:    5 * 1024 * 1024 * 1024,
+		AppName:           "versitygw",
+		ServerHeader:      "VERSITYGW",
+		StreamRequestBody: true,
+		DisableKeepalive:  true,
 	})
 
 	var opts []s3api.Option
@@ -157,28 +351,132 @@ func runGateway(be backend.Backend, s auth.Storer) error {
 		}
 		opts = append(opts, s3api.WithTLS(cert))
 	}
-
 	if debug {
 		opts = append(opts, s3api.WithDebug())
 	}
-
-	err := s.InitIAM()
-	if err != nil {
-		return fmt.Errorf("init iam: %w", err)
+	if admPort == "" {
+		opts = append(opts, s3api.WithAdminServer())
+	}
+	if quiet {
+		opts = append(opts, s3api.WithQuiet())
 	}
 
-	iam, err := auth.NewInternal(s)
+	admApp := fiber.New(fiber.Config{
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+	})
+
+	var admOpts []s3api.AdminOpt
+
+	if admCertFile != "" || admKeyFile != "" {
+		if admCertFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if admKeyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert))
+	}
+
+	iam, err := auth.New(&auth.Opts{
+		Dir:                iamDir,
+		LDAPServerURL:      ldapURL,
+		LDAPBindDN:         ldapBindDN,
+		LDAPPassword:       ldapPassword,
+		LDAPQueryBase:      ldapQueryBase,
+		LDAPObjClasses:     ldapObjClasses,
+		LDAPAccessAtr:      ldapAccessAtr,
+		LDAPSecretAtr:      ldapSecAtr,
+		LDAPRoleAtr:        ldapRoleAtr,
+		S3Access:           s3IamAccess,
+		S3Secret:           s3IamSecret,
+		S3Region:           s3IamRegion,
+		S3Bucket:           s3IamBucket,
+		S3Endpoint:         s3IamEndpoint,
+		S3DisableSSlVerfiy: s3IamSslNoVerify,
+		S3Debug:            s3IamDebug,
+		CacheDisable:       iamCacheDisable,
+		CacheTTL:           iamCacheTTL,
+		CachePrune:         iamCachePrune,
+	})
 	if err != nil {
-		return fmt.Errorf("setup internal iam service: %w", err)
+		return fmt.Errorf("setup iam: %w", err)
+	}
+
+	logger, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:    accessLog,
+		WebhookURL: logWebhookURL,
+	})
+	if err != nil {
+		return fmt.Errorf("setup logger: %w", err)
+	}
+
+	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
+		KafkaURL:      kafkaURL,
+		KafkaTopic:    kafkaTopic,
+		KafkaTopicKey: kafkaKey,
+		NatsURL:       natsURL,
+		NatsTopic:     natsTopic,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to connect to the message broker: %w", err)
 	}
 
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
 		Access: rootUserAccess,
 		Secret: rootUserSecret,
-	}, port, region, iam, opts...)
+	}, port, region, iam, logger, evSender, opts...)
 	if err != nil {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	return srv.Serve()
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
+
+	c := make(chan error, 2)
+	go func() { c <- srv.Serve() }()
+	if admPort != "" {
+		go func() { c <- admSrv.Serve() }()
+	}
+
+	// for/select blocks until shutdown
+Loop:
+	for {
+		select {
+		case <-ctx.Done():
+			err = ctx.Err()
+			break Loop
+		case err = <-c:
+			break Loop
+		case <-sigHup:
+			if logger != nil {
+				err = logger.HangUp()
+				if err != nil {
+					err = fmt.Errorf("HUP logger: %w", err)
+					break Loop
+				}
+			}
+		}
+	}
+	saveErr := err
+
+	be.Shutdown()
+
+	err = iam.Shutdown()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
+	}
+
+	if logger != nil {
+		err := logger.Shutdown()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
+		}
+	}
+
+	return saveErr
 }

cmd/versitygw/posix.go

@@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("init posix: %v", err)
 	}
 
-	return runGateway(be, be)
+	return runGateway(ctx.Context, be)
 }

cmd/versitygw/s3.go

@@ -0,0 +1,99 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/s3proxy"
+)
+
+var (
+	s3proxyAccess          string
+	s3proxySecret          string
+	s3proxyEndpoint        string
+	s3proxyRegion          string
+	s3proxyDisableChecksum bool
+	s3proxySslSkipVerify   bool
+	s3proxyDebug           bool
+)
+
+func s3Command() *cli.Command {
+	return &cli.Command{
+		Name:  "s3",
+		Usage: "s3 storage backend",
+		Description: `This runs the gateway like an s3 proxy redirecting requests
+to an s3 storage backend service.`,
+		Action: runS3,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "access",
+				Usage:       "s3 proxy server access key id",
+				Value:       "",
+				Required:    true,
+				Destination: &s3proxyAccess,
+				Aliases:     []string{"a"},
+			},
+			&cli.StringFlag{
+				Name:        "secret",
+				Usage:       "s3 proxy server secret access key",
+				Value:       "",
+				Required:    true,
+				Destination: &s3proxySecret,
+				Aliases:     []string{"s"},
+			},
+			&cli.StringFlag{
+				Name:        "endpoint",
+				Usage:       "s3 service endpoint, default AWS if not specified",
+				Value:       "",
+				Destination: &s3proxyEndpoint,
+			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "s3 service region, default 'us-east-1' if not specified",
+				Value:       "us-east-1",
+				Destination: &s3proxyRegion,
+			},
+			&cli.BoolFlag{
+				Name:        "disable-checksum",
+				Usage:       "disable gateway to server object checksums",
+				Value:       false,
+				Destination: &s3proxyDisableChecksum,
+			},
+			&cli.BoolFlag{
+				Name:        "ssl-skip-verify",
+				Usage:       "skip ssl cert verification for s3 service",
+				Value:       false,
+				Destination: &s3proxySslSkipVerify,
+			},
+			&cli.BoolFlag{
+				Name:        "debug",
+				Usage:       "output extra debug tracing",
+				Value:       false,
+				Destination: &s3proxyDebug,
+			},
+		},
+	}
+}
+
+func runS3(ctx *cli.Context) error {
+	be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	if err != nil {
+		return fmt.Errorf("init s3 backend: %w", err)
+	}
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/scoutfs.go

@@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}
 
-	return runGateway(be, be)
+	return runGateway(ctx.Context, be)
 }

cmd/versitygw/signal.go

@@ -12,37 +12,33 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package scoutfs
+package main
 
 import (
-	"errors"
 	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
 )
 
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
-	return nil, fmt.Errorf("scoutfs only available on linux")
-}
-
-type tmpfile struct {
-	f *os.File
-}
-
 var (
-	errNotSupported = errors.New("not supported")
+	sigDone = make(chan bool, 1)
+	sigHup  = make(chan bool, 1)
 )
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
-	return nil, errNotSupported
-}
+func setupSignalHandler() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
 
-func (tmp *tmpfile) link() error {
-	return errNotSupported
-}
-
-func (tmp *tmpfile) Write(b []byte) (int, error) {
-	return 0, errNotSupported
-}
-
-func (tmp *tmpfile) cleanup() {
+	go func() {
+		for sig := range sigs {
+			fmt.Fprintf(os.Stderr, "caught signal %v\n", sig)
+			switch sig {
+			case syscall.SIGINT, syscall.SIGTERM:
+				sigDone <- true
+			case syscall.SIGHUP:
+				sigHup <- true
+			}
+		}
+	}()
 }

cmd/versitygw/test.go

@@ -8,9 +8,20 @@ import (
 )
 
 var (
-	awsID     string
-	awsSecret string
-	endpoint  string
+	awsID           string
+	awsSecret       string
+	endpoint        string
+	prefix          string
+	dstBucket       string
+	partSize        int64
+	objSize         int64
+	concurrency     int
+	files           int
+	totalReqs       int
+	upload          bool
+	download        bool
+	pathStyle       bool
+	checksumDisable bool
 )
 
 func testCommand() *cli.Command {
@@ -56,104 +67,176 @@ func initTestFlags() []cli.Flag {
 }
 
 func initTestCommands() []*cli.Command {
-	return []*cli.Command{
-		{
-			Name:  "make-bucket",
-			Usage: "Test bucket creation.",
-			Description: `Calls s3 gateway create-bucket action to create a new bucket,
-		then calls delete-bucket action to delete the bucket.`,
-			Action: getAction(integration.TestMakeBucket),
-		},
-		{
-			Name:  "put-get-object",
-			Usage: "Test put & get object.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object from the bucket, deletes both the object and bucket.`,
-			Action: getAction(integration.TestPutGetObject),
-		},
-		{
-			Name:  "put-get-mp-object",
-			Usage: "Test put & get multipart object.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload,
-			gets the object from the bucket, deletes both the object and bucket.`,
-			Action: getAction(integration.TestPutGetMPObject),
-		},
-		{
-			Name:  "put-dir-object",
-			Usage: "Test put directory object.",
-			Description: `Creates a bucket with s3 gateway action, puts a directory object in it,
-			lists the bucket's objects, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestPutDirObject),
-		},
-		{
-			Name:  "list-objects",
-			Usage: "Test list-objects action.",
-			Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it,
-			lists the bucket's objects, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListObject),
-		},
-		{
-			Name:  "abort-mp",
-			Usage: "Tests abort-multipart-upload action.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			lists the multipart upload, aborts the multipart upload, lists the multipart upload again,
-			deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListAbortMultiPartObject),
-		},
-		{
-			Name:  "list-parts",
-			Usage: "Tests list-parts action.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			lists the upload parts, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListMultiParts),
-		},
-		{
-			Name:  "incorrect-mp",
-			Usage: "Tests incorrect multipart case.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			uploads different parts, completes the multipart upload with incorrect part numbers,
-			calls the head-object action, compares the content length, removes both the object and bucket`,
-			Action: getAction(integration.TestIncorrectMultiParts),
-		},
-		{
-			Name:  "incomplete-mp",
-			Usage: "Tests incomplete multi parts.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			upload a part, lists the parts, checks if the uploaded part is in the list, 
-			removes both the object and the bucket`,
-			Action: getAction(integration.TestIncompleteMultiParts),
-		},
-		{
-			Name:  "incomplete-put-object",
-			Usage: "Tests incomplete put objects case.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object with head-object action, expects the object to be got, 
-			removes both the object and bucket`,
-			Action: getAction(integration.TestIncompletePutObject),
-		},
-		{
-			Name:  "get-range",
-			Usage: "Tests get object by range.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object by specifying the object range, compares the range with the original one,
-			removes both the object and the bucket`,
-			Action: getAction(integration.TestRangeGet),
-		},
-		{
-			Name:  "invalid-mp",
-			Usage: "Tests invalid multi part case.",
-			Description: `Creates a bucket with s3 gateway action, creates a multi part upload,
-			uploads an invalid part, gets the object with head-object action, expects to get error,
-			removes both the object and bucket`,
-			Action: getAction(integration.TestInvalidMultiParts),
-		},
+	return append([]*cli.Command{
 		{
 			Name:        "full-flow",
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
 		},
-	}
+		{
+			Name:   "posix",
+			Usage:  "Tests posix specific features",
+			Action: getAction(integration.TestPosix),
+		},
+		{
+			Name:  "bench",
+			Usage: "Runs download/upload performance test on the gateway",
+			Description: `Uploads/downloads some number(specified by flags) of files with some capacity(bytes).
+			Logs the results to the console`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "files",
+					Usage:       "Number of objects to read/write",
+					Value:       1,
+					Destination: &files,
+				},
+				&cli.Int64Flag{
+					Name:        "objsize",
+					Usage:       "Uploading object size",
+					Value:       0,
+					Destination: &objSize,
+				},
+				&cli.StringFlag{
+					Name:        "prefix",
+					Usage:       "Object name prefix",
+					Destination: &prefix,
+				},
+				&cli.BoolFlag{
+					Name:        "upload",
+					Usage:       "Upload data to the gateway",
+					Value:       false,
+					Destination: &upload,
+				},
+				&cli.BoolFlag{
+					Name:        "download",
+					Usage:       "Download data to the gateway",
+					Value:       false,
+					Destination: &download,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to read/write data",
+					Destination: &dstBucket,
+				},
+				&cli.Int64Flag{
+					Name:        "partSize",
+					Usage:       "Upload/download size per thread",
+					Value:       64 * 1024 * 1024,
+					Destination: &partSize,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "Upload/download threads per object",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "pathStyle",
+					Usage:       "Use Pathstyle bucket addressing",
+					Value:       false,
+					Destination: &pathStyle,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				if upload && download {
+					return fmt.Errorf("must only specify one of upload or download")
+				}
+				if !upload && !download {
+					return fmt.Errorf("must specify one of upload or download")
+				}
+
+				if dstBucket == "" {
+					return fmt.Errorf("must specify bucket")
+				}
+
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+					integration.WithPartSize(partSize),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if pathStyle {
+					opts = append(opts, integration.WithPathStyle())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				if upload {
+					return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+				} else {
+					return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+				}
+			},
+		},
+		{
+			Name:        "throughput",
+			Usage:       "Runs throughput performance test on the gateway",
+			Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "reqs",
+					Usage:       "Total number of requests to send.",
+					Value:       1000,
+					Destination: &totalReqs,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to make the requests",
+					Destination: &dstBucket,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "threads per request",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				if dstBucket == "" {
+					return fmt.Errorf("must specify the destination bucket")
+				}
+
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+			},
+		},
+	}, extractIntTests()...)
 }
 
 type testFunc func(*integration.S3Conf)
@@ -175,6 +258,37 @@ func getAction(tf testFunc) func(*cli.Context) error {
 
 		fmt.Println()
 		fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount)
+		if integration.FailCount > 0 {
+			return fmt.Errorf("test failed with %v errors", integration.FailCount)
+		}
 		return nil
 	}
 }
+
+func extractIntTests() (commands []*cli.Command) {
+	tests := integration.GetIntTests()
+	for key, val := range tests {
+		testKey := key
+		testFunc := val
+		commands = append(commands, &cli.Command{
+			Name:  testKey,
+			Usage: fmt.Sprintf("Runs %v integration test", testKey),
+			Action: func(ctx *cli.Context) error {
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+
+				s := integration.NewS3Conf(opts...)
+				err := testFunc(s)
+				return err
+			},
+		})
+	}
+	return
+}

docker-compose.yml

@@ -0,0 +1,38 @@
+version: "3"
+services:
+  posix:
+    build: 
+      context: .
+      dockerfile: ./Dockerfile.dev
+      args:
+        - IAM_DIR=${IAM_DIR}
+        - SETUP_DIR=${SETUP_DIR}
+    volumes:
+      - ./:/app
+    ports:
+      - "${POSIX_PORT}:${POSIX_PORT}"
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$POSIX_PORT -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR posix $SETUP_DIR"]
+  proxy:
+    build: 
+      context: .
+      dockerfile: ./Dockerfile.dev
+    volumes:
+      - ./:/app
+    ports:
+      - "${PROXY_PORT}:${PROXY_PORT}"
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$PROXY_PORT s3 -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --endpoint http://posix:$POSIX_PORT"]
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    ports:
+      - "10000:10000"
+      - "10001:10001"
+      - "10002:10002"
+  azuritegw:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.dev
+    volumes:
+      - ./:/app
+    ports:
+      - 7070:7070
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url http://azurite:10000/$AZ_ACCOUNT_NAME"]

go.mod

@@ -3,51 +3,66 @@ module github.com/versity/versitygw
 go 1.20
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.18.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0
-	github.com/aws/smithy-go v1.13.5
-	github.com/gofiber/fiber/v2 v2.46.0
-	github.com/google/uuid v1.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1
+	github.com/aws/aws-sdk-go-v2 v1.24.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
+	github.com/aws/smithy-go v1.19.0
+	github.com/go-ldap/ldap/v3 v3.4.6
+	github.com/gofiber/fiber/v2 v2.52.0
+	github.com/google/uuid v1.6.0
+	github.com/nats-io/nats.go v1.32.0
 	github.com/pkg/xattr v0.4.9
-	github.com/urfave/cli/v2 v2.25.6
-	github.com/valyala/fasthttp v1.47.0
+	github.com/segmentio/kafka-go v0.4.47
+	github.com/urfave/cli/v2 v2.27.1
+	github.com/valyala/fasthttp v1.51.0
 	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.9.0
+	golang.org/x/sys v0.16.0
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	golang.org/x/crypto v0.18.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.27
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.26
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.26.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.16
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/klauspost/compress v1.16.6 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
-	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
-	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

go.sum

@@ -1,144 +1,195 @@
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k=
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
-github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
-github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA=
-github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70 h1:4bh28MeeXoBFTjb0JjQ5sVatzlf5xA1DziV8mZed9v4=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70/go.mod h1:9yI5NXzqy2yOiMytv6QLZHvlyHLwYxO9iIq+bZIbrFg=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1 h1:rYYwwsGqbwvGgQHjBkqgDt8MynXk+I8xgS0IEj5gOT0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w=
-github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE=
-github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg=
-github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
-github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
+github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
+github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o=
+github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15 h1:2MUXyGW6dVaQz6aqycpbdLIH1NMcUI6kW6vQ0RabGYg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15/go.mod h1:aHbhbR6WEQgHAiRj41EQ2W47yOYwNtIkWTXmcAtYqj8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10/go.mod h1:FHbKWQtRBYUz4vO5WBWjzMD2by126ny5y/1EoaWoLfI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 h1:L0ai8WICYHozIKK+OtPzVJBugL7culcuM4E4JOpIEm8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10/go.mod h1:byqfyxJBshFk0fF9YmK0M0ugIO8OWjzH2T3bPG4eGuA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 h1:KOxnQeWy5sXyS37fdKEvAsGHOr9fa/qvwxfJurR/BzE=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10/go.mod h1:jMx5INQFYFYB3lQD9W0D8Ohgq6Wnl7NYOJ2TQndbulI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1 h1:5XNlsBsEvBZBMO6p82y+sqpWg8j5aBCe+5C2GBFgqBQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns=
-github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
+github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
+github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
+github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE=
+github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
+github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
-github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
-github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/nats-io/nats.go v1.32.0 h1:Bx9BZS+aXYlxW08k8Gd3yR2s73pV5XSoAQUyp1Kwvp0=
+github.com/nats-io/nats.go v1.32.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
 github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
-github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
-github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
-github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
-github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
+github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
+github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
-github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
-github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
-github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU=
-github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
+github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
-github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
 github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration/bench.go

@@ -0,0 +1,142 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type prefResult struct {
+	elapsed time.Duration
+	size    int64
+	err     error
+}
+
+func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+	if objSize == 0 {
+		return fmt.Errorf("must specify object size for upload")
+	}
+
+	if objSize > (int64(10000) * s.PartSize) {
+		return fmt.Errorf("object size can not exceed 10000 * chunksize")
+	}
+
+	runF("performance test: upload objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			var r io.Reader = NewDataReader(int(objSize), int(s.PartSize))
+
+			start := time.Now()
+			err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = objSize
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed := time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			break
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run upload: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return nil
+}
+
+func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+
+	runF("performance test: download objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			nw := NewNullWriter()
+			start := time.Now()
+			n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = n
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed := time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			break
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run download: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return nil
+}
+
+func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
+	client := s3.NewFromConfig(s.Config())
+	var wg sync.WaitGroup
+	var resErr error
+
+	// Record the start time
+	startTime := time.Now()
+	runF("performance test: measuring request per second")
+
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < totalReqs/s.Concurrency; i++ {
+				_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket})
+				if err != nil && resErr != nil {
+					resErr = err
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	if resErr != nil {
+		failF("performance test failed with error: %w", resErr)
+		return nil
+	}
+	elapsedTime := time.Since(startTime)
+	rps := int(float64(totalReqs) / elapsedTime.Seconds())
+
+	passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
+	return nil
+}

integration/data-io.go

@@ -38,6 +38,26 @@ func (r *RReader) Sum() []byte {
 	return r.hash.Sum(nil)
 }
 
+type ZReader struct {
+	buf      []byte
+	dataleft int
+}
+
+func NewZeroReader(totalsize, bufsize int) *ZReader {
+	b := make([]byte, bufsize)
+	return &ZReader{buf: b, dataleft: totalsize}
+}
+
+func (r *ZReader) Read(p []byte) (int, error) {
+	n := min(len(p), len(r.buf), r.dataleft)
+	r.dataleft -= n
+	err := error(nil)
+	if n == 0 {
+		err = io.EOF
+	}
+	return copy(p, r.buf[:n]), err
+}
+
 func min(values ...int) int {
 	if len(values) == 0 {
 		return 0
@@ -52,3 +72,13 @@ func min(values ...int) int {
 
 	return min
 }
+
+type NW struct{}
+
+func NewNullWriter() NW {
+	return NW{}
+}
+
+func (NW) WriteAt(p []byte, off int64) (n int, err error) {
+	return len(p), nil
+}

integration/group-tests.go

@@ -0,0 +1,355 @@
+package integration
+
+func TestAuthentication(s *S3Conf) {
+	Authentication_empty_auth_header(s)
+	Authentication_invalid_auth_header(s)
+	Authentication_unsupported_signature_version(s)
+	Authentication_malformed_credentials(s)
+	Authentication_malformed_credentials_invalid_parts(s)
+	Authentication_credentials_terminated_string(s)
+	Authentication_credentials_incorrect_service(s)
+	Authentication_credentials_incorrect_region(s)
+	Authentication_credentials_invalid_date(s)
+	Authentication_credentials_future_date(s)
+	Authentication_credentials_past_date(s)
+	Authentication_credentials_non_existing_access_key(s)
+	Authentication_invalid_signed_headers(s)
+	Authentication_missing_date_header(s)
+	Authentication_invalid_date_header(s)
+	Authentication_date_mismatch(s)
+	Authentication_incorrect_payload_hash(s)
+	Authentication_incorrect_md5(s)
+	Authentication_signature_error_incorrect_secret_key(s)
+}
+
+func TestCreateBucket(s *S3Conf) {
+	CreateBucket_invalid_bucket_name(s)
+	CreateBucket_existing_bucket(s)
+	CreateBucket_as_user(s)
+	CreateDeleteBucket_success(s)
+}
+
+func TestHeadBucket(s *S3Conf) {
+	HeadBucket_non_existing_bucket(s)
+	HeadBucket_success(s)
+}
+
+func TestListBuckets(s *S3Conf) {
+	ListBuckets_as_user(s)
+	ListBuckets_as_admin(s)
+	ListBuckets_success(s)
+}
+
+func TestDeleteBucket(s *S3Conf) {
+	DeleteBucket_non_existing_bucket(s)
+	DeleteBucket_non_empty_bucket(s)
+	DeleteBucket_success_status_code(s)
+}
+
+func TestPutObject(s *S3Conf) {
+	PutObject_non_existing_bucket(s)
+	PutObject_special_chars(s)
+	PutObject_invalid_long_tags(s)
+	PutObject_success(s)
+	PutObject_invalid_credentials(s)
+}
+
+func TestHeadObject(s *S3Conf) {
+	HeadObject_non_existing_object(s)
+	HeadObject_success(s)
+}
+
+func TestGetObject(s *S3Conf) {
+	GetObject_non_existing_key(s)
+	GetObject_invalid_ranges(s)
+	GetObject_with_meta(s)
+	GetObject_success(s)
+	GetObject_by_range_success(s)
+}
+
+func TestListObjects(s *S3Conf) {
+	ListObjects_non_existing_bucket(s)
+	ListObjects_with_prefix(s)
+	ListObject_truncated(s)
+	ListObjects_invalid_max_keys(s)
+	ListObjects_max_keys_0(s)
+	ListObjects_delimiter(s)
+	ListObjects_max_keys_none(s)
+	ListObjects_marker_not_from_obj_list(s)
+}
+
+func TestDeleteObject(s *S3Conf) {
+	DeleteObject_non_existing_object(s)
+	DeleteObject_success(s)
+	DeleteObject_success_status_code(s)
+}
+
+func TestDeleteObjects(s *S3Conf) {
+	DeleteObjects_empty_input(s)
+	DeleteObjects_non_existing_objects(s)
+	DeleteObjects_success(s)
+}
+
+func TestCopyObject(s *S3Conf) {
+	CopyObject_non_existing_dst_bucket(s)
+	CopyObject_not_owned_source_bucket(s)
+	CopyObject_copy_to_itself(s)
+	CopyObject_to_itself_with_new_metadata(s)
+	CopyObject_success(s)
+}
+
+func TestPutObjectTagging(s *S3Conf) {
+	PutObjectTagging_non_existing_object(s)
+	PutObjectTagging_long_tags(s)
+	PutObjectTagging_success(s)
+}
+
+func TestGetObjectTagging(s *S3Conf) {
+	GetObjectTagging_non_existing_object(s)
+	GetObjectTagging_success(s)
+}
+
+func TestDeleteObjectTagging(s *S3Conf) {
+	DeleteObjectTagging_non_existing_object(s)
+	DeleteObjectTagging_success_status(s)
+	DeleteObjectTagging_success(s)
+}
+
+func TestCreateMultipartUpload(s *S3Conf) {
+	CreateMultipartUpload_non_existing_bucket(s)
+	CreateMultipartUpload_success(s)
+}
+
+func TestUploadPart(s *S3Conf) {
+	UploadPart_non_existing_bucket(s)
+	UploadPart_invalid_part_number(s)
+	UploadPart_non_existing_key(s)
+	UploadPart_non_existing_mp_upload(s)
+	UploadPart_success(s)
+}
+
+func TestUploadPartCopy(s *S3Conf) {
+	UploadPartCopy_non_existing_bucket(s)
+	UploadPartCopy_incorrect_uploadId(s)
+	UploadPartCopy_incorrect_object_key(s)
+	UploadPartCopy_invalid_part_number(s)
+	UploadPartCopy_invalid_copy_source(s)
+	UploadPartCopy_non_existing_source_bucket(s)
+	UploadPartCopy_non_existing_source_object_key(s)
+	UploadPartCopy_success(s)
+	UploadPartCopy_by_range_invalid_range(s)
+	UploadPartCopy_greater_range_than_obj_size(s)
+	UploadPartCopy_by_range_success(s)
+}
+
+func TestListParts(s *S3Conf) {
+	ListParts_incorrect_uploadId(s)
+	ListParts_incorrect_object_key(s)
+	ListParts_success(s)
+}
+
+func TestListMultipartUploads(s *S3Conf) {
+	ListMultipartUploads_non_existing_bucket(s)
+	ListMultipartUploads_empty_result(s)
+	ListMultipartUploads_invalid_max_uploads(s)
+	ListMultipartUploads_max_uploads(s)
+	ListMultipartUploads_incorrect_next_key_marker(s)
+	ListMultipartUploads_ignore_upload_id_marker(s)
+	ListMultipartUploads_success(s)
+}
+
+func TestAbortMultipartUpload(s *S3Conf) {
+	AbortMultipartUpload_non_existing_bucket(s)
+	AbortMultipartUpload_incorrect_uploadId(s)
+	AbortMultipartUpload_incorrect_object_key(s)
+	AbortMultipartUpload_success(s)
+	AbortMultipartUpload_success_status_code(s)
+}
+
+func TestCompleteMultipartUpload(s *S3Conf) {
+	CompletedMultipartUpload_non_existing_bucket(s)
+	CompleteMultipartUpload_invalid_part_number(s)
+	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_success(s)
+}
+
+func TestPutBucketAcl(s *S3Conf) {
+	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_invalid_acl_canned_and_acp(s)
+	PutBucketAcl_invalid_acl_canned_and_grants(s)
+	PutBucketAcl_invalid_acl_acp_and_grants(s)
+	PutBucketAcl_invalid_owner(s)
+	PutBucketAcl_success_access_denied(s)
+	PutBucketAcl_success_grants(s)
+	PutBucketAcl_success_canned_acl(s)
+	PutBucketAcl_success_acp(s)
+}
+
+func TestGetBucketAcl(s *S3Conf) {
+	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_access_denied(s)
+	GetBucketAcl_success(s)
+}
+
+func TestFullFlow(s *S3Conf) {
+	TestAuthentication(s)
+	TestCreateBucket(s)
+	TestHeadBucket(s)
+	TestListBuckets(s)
+	TestDeleteBucket(s)
+	TestPutObject(s)
+	TestHeadObject(s)
+	TestGetObject(s)
+	TestListObjects(s)
+	TestDeleteObject(s)
+	TestDeleteObjects(s)
+	TestCopyObject(s)
+	TestPutObjectTagging(s)
+	TestDeleteObjectTagging(s)
+	TestCreateMultipartUpload(s)
+	TestUploadPart(s)
+	TestUploadPartCopy(s)
+	TestListParts(s)
+	TestListMultipartUploads(s)
+	TestAbortMultipartUpload(s)
+	TestCompleteMultipartUpload(s)
+	TestPutBucketAcl(s)
+	TestGetBucketAcl(s)
+}
+
+func TestPosix(s *S3Conf) {
+	PutObject_overwrite_dir_obj(s)
+	PutObject_overwrite_file_obj(s)
+	PutObject_dir_obj_with_data(s)
+	CreateMultipartUpload_dir_obj(s)
+}
+
+type IntTests map[string]func(s *S3Conf) error
+
+func GetIntTests() IntTests {
+	return IntTests{
+		"Authentication_empty_auth_header":                    Authentication_empty_auth_header,
+		"Authentication_invalid_auth_header":                  Authentication_invalid_auth_header,
+		"Authentication_unsupported_signature_version":        Authentication_unsupported_signature_version,
+		"Authentication_malformed_credentials":                Authentication_malformed_credentials,
+		"Authentication_malformed_credentials_invalid_parts":  Authentication_malformed_credentials_invalid_parts,
+		"Authentication_credentials_terminated_string":        Authentication_credentials_terminated_string,
+		"Authentication_credentials_incorrect_service":        Authentication_credentials_incorrect_service,
+		"Authentication_credentials_incorrect_region":         Authentication_credentials_incorrect_region,
+		"Authentication_credentials_invalid_date":             Authentication_credentials_invalid_date,
+		"Authentication_credentials_future_date":              Authentication_credentials_future_date,
+		"Authentication_credentials_past_date":                Authentication_credentials_past_date,
+		"Authentication_credentials_non_existing_access_key":  Authentication_credentials_non_existing_access_key,
+		"Authentication_invalid_signed_headers":               Authentication_invalid_signed_headers,
+		"Authentication_missing_date_header":                  Authentication_missing_date_header,
+		"Authentication_invalid_date_header":                  Authentication_invalid_date_header,
+		"Authentication_date_mismatch":                        Authentication_date_mismatch,
+		"Authentication_incorrect_payload_hash":               Authentication_incorrect_payload_hash,
+		"Authentication_incorrect_md5":                        Authentication_incorrect_md5,
+		"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
+		"CreateBucket_invalid_bucket_name":                    CreateBucket_invalid_bucket_name,
+		"CreateBucket_existing_bucket":                        CreateBucket_existing_bucket,
+		"CreateBucket_as_user":                                CreateBucket_as_user,
+		"CreateDeleteBucket_success":                          CreateDeleteBucket_success,
+		"HeadBucket_non_existing_bucket":                      HeadBucket_non_existing_bucket,
+		"HeadBucket_success":                                  HeadBucket_success,
+		"ListBuckets_as_user":                                 ListBuckets_as_user,
+		"ListBuckets_as_admin":                                ListBuckets_as_admin,
+		"ListBuckets_success":                                 ListBuckets_success,
+		"DeleteBucket_non_existing_bucket":                    DeleteBucket_non_existing_bucket,
+		"DeleteBucket_non_empty_bucket":                       DeleteBucket_non_empty_bucket,
+		"DeleteBucket_success_status_code":                    DeleteBucket_success_status_code,
+		"PutObject_non_existing_bucket":                       PutObject_non_existing_bucket,
+		"PutObject_special_chars":                             PutObject_special_chars,
+		"PutObject_invalid_long_tags":                         PutObject_invalid_long_tags,
+		"PutObject_success":                                   PutObject_success,
+		"PutObject_invalid_credentials":                       PutObject_invalid_credentials,
+		"HeadObject_non_existing_object":                      HeadObject_non_existing_object,
+		"HeadObject_success":                                  HeadObject_success,
+		"GetObject_non_existing_key":                          GetObject_non_existing_key,
+		"GetObject_invalid_ranges":                            GetObject_invalid_ranges,
+		"GetObject_with_meta":                                 GetObject_with_meta,
+		"GetObject_success":                                   GetObject_success,
+		"GetObject_by_range_success":                          GetObject_by_range_success,
+		"ListObjects_non_existing_bucket":                     ListObjects_non_existing_bucket,
+		"ListObjects_with_prefix":                             ListObjects_with_prefix,
+		"ListObject_truncated":                                ListObject_truncated,
+		"ListObjects_invalid_max_keys":                        ListObjects_invalid_max_keys,
+		"ListObjects_max_keys_0":                              ListObjects_max_keys_0,
+		"ListObjects_delimiter":                               ListObjects_delimiter,
+		"ListObjects_max_keys_none":                           ListObjects_max_keys_none,
+		"ListObjects_marker_not_from_obj_list":                ListObjects_marker_not_from_obj_list,
+		"DeleteObject_non_existing_object":                    DeleteObject_non_existing_object,
+		"DeleteObject_success":                                DeleteObject_success,
+		"DeleteObject_success_status_code":                    DeleteObject_success_status_code,
+		"DeleteObjects_empty_input":                           DeleteObjects_empty_input,
+		"DeleteObjects_non_existing_objects":                  DeleteObjects_non_existing_objects,
+		"DeleteObjects_success":                               DeleteObjects_success,
+		"CopyObject_non_existing_dst_bucket":                  CopyObject_non_existing_dst_bucket,
+		"CopyObject_not_owned_source_bucket":                  CopyObject_not_owned_source_bucket,
+		"CopyObject_copy_to_itself":                           CopyObject_copy_to_itself,
+		"CopyObject_to_itself_with_new_metadata":              CopyObject_to_itself_with_new_metadata,
+		"CopyObject_success":                                  CopyObject_success,
+		"PutObjectTagging_non_existing_object":                PutObjectTagging_non_existing_object,
+		"PutObjectTagging_long_tags":                          PutObjectTagging_long_tags,
+		"PutObjectTagging_success":                            PutObjectTagging_success,
+		"GetObjectTagging_non_existing_object":                GetObjectTagging_non_existing_object,
+		"GetObjectTagging_success":                            GetObjectTagging_success,
+		"DeleteObjectTagging_non_existing_object":             DeleteObjectTagging_non_existing_object,
+		"DeleteObjectTagging_success_status":                  DeleteObjectTagging_success_status,
+		"DeleteObjectTagging_success":                         DeleteObjectTagging_success,
+		"CreateMultipartUpload_non_existing_bucket":           CreateMultipartUpload_non_existing_bucket,
+		"CreateMultipartUpload_success":                       CreateMultipartUpload_success,
+		"UploadPart_non_existing_bucket":                      UploadPart_non_existing_bucket,
+		"UploadPart_invalid_part_number":                      UploadPart_invalid_part_number,
+		"UploadPart_non_existing_key":                         UploadPart_non_existing_key,
+		"UploadPart_non_existing_mp_upload":                   UploadPart_non_existing_mp_upload,
+		"UploadPart_success":                                  UploadPart_success,
+		"UploadPartCopy_non_existing_bucket":                  UploadPartCopy_non_existing_bucket,
+		"UploadPartCopy_incorrect_uploadId":                   UploadPartCopy_incorrect_uploadId,
+		"UploadPartCopy_incorrect_object_key":                 UploadPartCopy_incorrect_object_key,
+		"UploadPartCopy_invalid_part_number":                  UploadPartCopy_invalid_part_number,
+		"UploadPartCopy_invalid_copy_source":                  UploadPartCopy_invalid_copy_source,
+		"UploadPartCopy_non_existing_source_bucket":           UploadPartCopy_non_existing_source_bucket,
+		"UploadPartCopy_non_existing_source_object_key":       UploadPartCopy_non_existing_source_object_key,
+		"UploadPartCopy_success":                              UploadPartCopy_success,
+		"UploadPartCopy_by_range_invalid_range":               UploadPartCopy_by_range_invalid_range,
+		"UploadPartCopy_greater_range_than_obj_size":          UploadPartCopy_greater_range_than_obj_size,
+		"UploadPartCopy_by_range_success":                     UploadPartCopy_by_range_success,
+		"ListParts_incorrect_uploadId":                        ListParts_incorrect_uploadId,
+		"ListParts_incorrect_object_key":                      ListParts_incorrect_object_key,
+		"ListParts_success":                                   ListParts_success,
+		"ListMultipartUploads_non_existing_bucket":            ListMultipartUploads_non_existing_bucket,
+		"ListMultipartUploads_empty_result":                   ListMultipartUploads_empty_result,
+		"ListMultipartUploads_invalid_max_uploads":            ListMultipartUploads_invalid_max_uploads,
+		"ListMultipartUploads_max_uploads":                    ListMultipartUploads_max_uploads,
+		"ListMultipartUploads_incorrect_next_key_marker":      ListMultipartUploads_incorrect_next_key_marker,
+		"ListMultipartUploads_ignore_upload_id_marker":        ListMultipartUploads_ignore_upload_id_marker,
+		"ListMultipartUploads_success":                        ListMultipartUploads_success,
+		"AbortMultipartUpload_non_existing_bucket":            AbortMultipartUpload_non_existing_bucket,
+		"AbortMultipartUpload_incorrect_uploadId":             AbortMultipartUpload_incorrect_uploadId,
+		"AbortMultipartUpload_incorrect_object_key":           AbortMultipartUpload_incorrect_object_key,
+		"AbortMultipartUpload_success":                        AbortMultipartUpload_success,
+		"AbortMultipartUpload_success_status_code":            AbortMultipartUpload_success_status_code,
+		"CompletedMultipartUpload_non_existing_bucket":        CompletedMultipartUpload_non_existing_bucket,
+		"CompleteMultipartUpload_invalid_part_number":         CompleteMultipartUpload_invalid_part_number,
+		"CompleteMultipartUpload_invalid_ETag":                CompleteMultipartUpload_invalid_ETag,
+		"CompleteMultipartUpload_success":                     CompleteMultipartUpload_success,
+		"PutBucketAcl_non_existing_bucket":                    PutBucketAcl_non_existing_bucket,
+		"PutBucketAcl_invalid_acl_canned_and_acp":             PutBucketAcl_invalid_acl_canned_and_acp,
+		"PutBucketAcl_invalid_acl_canned_and_grants":          PutBucketAcl_invalid_acl_canned_and_grants,
+		"PutBucketAcl_invalid_acl_acp_and_grants":             PutBucketAcl_invalid_acl_acp_and_grants,
+		"PutBucketAcl_invalid_owner":                          PutBucketAcl_invalid_owner,
+		"PutBucketAcl_success_access_denied":                  PutBucketAcl_success_access_denied,
+		"PutBucketAcl_success_grants":                         PutBucketAcl_success_grants,
+		"PutBucketAcl_success_canned_acl":                     PutBucketAcl_success_canned_acl,
+		"PutBucketAcl_success_acp":                            PutBucketAcl_success_acp,
+		"GetBucketAcl_non_existing_bucket":                    GetBucketAcl_non_existing_bucket,
+		"GetBucketAcl_access_denied":                          GetBucketAcl_access_denied,
+		"GetBucketAcl_success":                                GetBucketAcl_success,
+		"PutObject_overwrite_dir_obj":                         PutObject_overwrite_dir_obj,
+		"PutObject_overwrite_file_obj":                        PutObject_overwrite_file_obj,
+		"PutObject_dir_obj_with_data":                         PutObject_dir_obj_with_data,
+		"CreateMultipartUpload_dir_obj":                       CreateMultipartUpload_dir_obj,
+	}
+}

integration/s3conf.go

@@ -2,6 +2,7 @@ package integration
 
 import (
 	"context"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -10,6 +11,8 @@ import (
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go/middleware"
 )
 
@@ -26,10 +29,7 @@ type S3Conf struct {
 }
 
 func NewS3Conf(opts ...Option) *S3Conf {
-	s := &S3Conf{
-		PartSize:    64 * 1024 * 1024, // 64B default chunksize
-		Concurrency: 1,                // 1 default concurrency
-	}
+	s := &S3Conf{}
 
 	for _, opt := range opts {
 		opt(s)
@@ -123,3 +123,31 @@ func (c *S3Conf) Config() aws.Config {
 
 	return cfg
 }
+
+func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
+	uploader := manager.NewUploader(s3.NewFromConfig(c.Config()))
+	uploader.PartSize = c.PartSize
+	uploader.Concurrency = c.Concurrency
+
+	upinfo := &s3.PutObjectInput{
+		Body:   r,
+		Bucket: &bucket,
+		Key:    &object,
+	}
+
+	_, err := uploader.Upload(context.Background(), upinfo)
+	return err
+}
+
+func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
+	downloader := manager.NewDownloader(s3.NewFromConfig(c.Config()))
+	downloader.PartSize = c.PartSize
+	downloader.Concurrency = c.Concurrency
+
+	downinfo := &s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &object,
+	}
+
+	return downloader.Download(context.Background(), w, downinfo)
+}

integration/utils.go

@@ -0,0 +1,553 @@
+package integration
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	rnd "math/rand"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	bcktCount  = 0
+	succUsrCrt = "The user has been created successfully"
+	failUsrCrt = "failed to create a user: update iam data: account already exists"
+)
+
+func getBucketName() string {
+	bcktCount++
+	return fmt.Sprintf("test-bucket-%v", bcktCount)
+}
+
+func setup(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func teardown(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	deleteObject := func(bucket, key, versionId *string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
+			Bucket:    bucket,
+			Key:       key,
+			VersionId: versionId,
+		})
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to delete object %v: %w", *key, err)
+		}
+		return nil
+	}
+
+	in := &s3.ListObjectsV2Input{Bucket: &bucket}
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListObjectsV2(ctx, in)
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to list objects: %w", err)
+		}
+
+		for _, item := range out.Contents {
+			err = deleteObject(&bucket, item.Key, nil)
+			if err != nil {
+				return err
+			}
+		}
+
+		if out.IsTruncated != nil && *out.IsTruncated {
+			in.ContinuationToken = out.ContinuationToken
+		} else {
+			break
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) error {
+	runF(testName)
+	bucketName := getBucketName()
+	err := setup(s, bucketName)
+	if err != nil {
+		failF("%v: failed to create a bucket: %v", testName, err)
+		return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
+	}
+	client := s3.NewFromConfig(s.Config())
+	handlerErr := handler(client, bucketName)
+	if handlerErr != nil {
+		failF("%v: %v", testName, handlerErr)
+	}
+
+	err = teardown(s, bucketName)
+	if err != nil {
+		fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
+		if handlerErr == nil {
+			return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
+		}
+	}
+	if handlerErr == nil {
+		passF(testName)
+	}
+
+	return handlerErr
+}
+
+type authConfig struct {
+	testName string
+	path     string
+	method   string
+	body     []byte
+	service  string
+	date     time.Time
+}
+
+func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
+	runF(cfg.testName)
+	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err)
+		return fmt.Errorf("%v: %w", cfg.testName, err)
+	}
+
+	err = handler(req)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err)
+		return fmt.Errorf("%v: %w", cfg.testName, err)
+	}
+	passF(cfg.testName)
+	return nil
+}
+
+func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
+	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(body)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date)
+	if signErr != nil {
+		return nil, fmt.Errorf("failed to sign the request: %w", signErr)
+	}
+
+	return req, nil
+}
+
+func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var errResp s3err.APIErrorResponse
+	err = xml.Unmarshal(body, &errResp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != apiErr.HTTPStatusCode {
+		return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
+	}
+	if errResp.Code != apiErr.Code {
+		return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
+	}
+	if errResp.Message != apiErr.Description {
+		return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
+	}
+
+	return nil
+}
+
+func checkApiErr(err error, apiErr s3err.APIError) error {
+	if err == nil {
+		return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
+			return nil
+		}
+
+		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+	}
+	return fmt.Errorf("expected aws api error, instead got: %w", err)
+}
+
+func checkSdkApiErr(err error, code string) error {
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() != code {
+			return fmt.Errorf("expected %v, instead got %v", code, ae.ErrorCode())
+		}
+		return nil
+	}
+	return err
+}
+
+func putObjects(client *s3.Client, objs []string, bucket string) error {
+	for _, key := range objs {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Key:    &key,
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
+	data = make([]byte, lgth)
+	rand.Read(data)
+	csum = sha256.Sum256(data)
+	r := bytes.NewReader(data)
+	input.Body = r
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err = client.PutObject(ctx, input)
+	cancel()
+
+	return
+}
+
+func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: &bucket,
+		Key:    &key,
+	})
+	cancel()
+	return out, err
+}
+
+func isEqual(a, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i, d := range a {
+		if d != b[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+	for i, item := range list1 {
+		if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareParts(parts1, parts2 []types.Part) bool {
+	if len(parts1) != len(parts2) {
+		return false
+	}
+
+	for i, prt := range parts1 {
+		if *prt.PartNumber != *parts2[i].PartNumber {
+			return false
+		}
+		if *prt.ETag != *parts2[i].ETag {
+			return false
+		}
+	}
+	return true
+}
+
+func areTagsSame(tags1, tags2 []types.Tag) bool {
+	if len(tags1) != len(tags2) {
+		return false
+	}
+
+	for _, tag := range tags1 {
+		if !containsTag(tag, tags2) {
+			return false
+		}
+	}
+	return true
+}
+
+func containsTag(tag types.Tag, list []types.Tag) bool {
+	for _, item := range list {
+		if *item.Key == *tag.Key && *item.Value == *tag.Value {
+			return true
+		}
+	}
+	return false
+}
+
+func compareGrants(grts1, grts2 []types.Grant) bool {
+	if len(grts1) != len(grts2) {
+		return false
+	}
+
+	for i, grt := range grts1 {
+		if grt.Permission != grts2[i].Permission {
+			return false
+		}
+		if *grt.Grantee.ID != *grts2[i].Grantee.ID {
+			return false
+		}
+	}
+	return true
+}
+
+func execCommand(args ...string) ([]byte, error) {
+	cmd := exec.Command("./versitygw", args...)
+
+	return cmd.CombinedOutput()
+}
+
+func getString(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
+}
+
+func getPtr(str string) *string {
+	return &str
+}
+
+func areMapsSame(mp1, mp2 map[string]string) bool {
+	if len(mp1) != len(mp2) {
+		return false
+	}
+	for key, val := range mp1 {
+		if mp2[key] != val {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[*elem.Name] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[elem.Name]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareObjects(list1 []string, list2 []types.Object) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Prefix]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
+	dr := NewDataReader(size, size)
+	datafile := "rand.data"
+	w, err := os.Create(datafile)
+	if err != nil {
+		return parts, err
+	}
+	defer w.Close()
+
+	_, err = io.Copy(w, dr)
+	if err != nil {
+		return parts, err
+	}
+
+	fileInfo, err := w.Stat()
+	if err != nil {
+		return parts, err
+	}
+
+	partSize := fileInfo.Size() / int64(partCount)
+	var offset int64
+
+	for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
+		partStart := (partNumber - 1) * partSize
+		partEnd := partStart + partSize - 1
+		if partEnd > fileInfo.Size()-1 {
+			partEnd = fileInfo.Size() - 1
+		}
+		partBuffer := make([]byte, partEnd-partStart+1)
+		_, err := w.ReadAt(partBuffer, partStart)
+		if err != nil {
+			return parts, err
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		pn := int32(partNumber)
+		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     &bucket,
+			Key:        &key,
+			UploadId:   &uploadId,
+			Body:       bytes.NewReader(partBuffer),
+			PartNumber: &pn,
+		})
+		cancel()
+		if err != nil {
+			return parts, err
+		}
+		parts = append(parts, types.Part{
+			ETag:       out.ETag,
+			PartNumber: &pn,
+		})
+		offset += partSize
+	}
+
+	return parts, err
+}
+
+type user struct {
+	access string
+	secret string
+	role   string
+}
+
+func createUsers(s *S3Conf, users []user) error {
+	for _, usr := range users {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
+			return fmt.Errorf("failed to create a user account")
+		}
+	}
+	return nil
+}
+
+func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
+	for _, bucket := range buckets {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
+			return fmt.Errorf(string(out))
+		}
+	}
+
+	return nil
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func genRandString(length int) string {
+	source := rnd.NewSource(time.Now().UnixNano())
+	random := rnd.New(source)
+	result := make([]byte, length)
+	for i := range result {
+		result[i] = charset[random.Intn(len(charset))]
+	}
+	return string(result)
+}

runtests.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# make temp dirs
+rm -rf /tmp/gw
+mkdir /tmp/gw
+rm -rf /tmp/covdata
+mkdir /tmp/covdata
+
+# run server in background
+GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
+GW_PID=$!
+
+# wait a second for server to start up
+sleep 1
+
+# check if server is still running
+if ! kill -0 $GW_PID; then
+	echo "server no longer running"
+	exit 1
+fi
+
+# run tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
+	echo "tests failed"
+	kill $GW_PID
+	exit 1
+fi
+
+# kill off server
+kill $GW_PID
+exit 0
+
+# if the above binary was built with -cover enabled (make testbin),
+# then the following can be used for code coverage reports:
+# go tool covdata percent -i=/tmp/covdata
+# go tool covdata textfmt -i=/tmp/covdata -o profile.txt
+# go tool cover -html=profile.txt
+

s3api/admin-router.go

@@ -0,0 +1,43 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+)
+
+type S3AdminRouter struct{}
+
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
+	controller := controllers.NewAdminController(iam, be)
+
+	// CreateUser admin api
+	app.Patch("/create-user", controller.CreateUser)
+
+	// DeleteUsers admin api
+	app.Patch("/delete-user", controller.DeleteUser)
+
+	// ListUsers admin api
+	app.Patch("/list-users", controller.ListUsers)
+
+	// ChangeBucketOwner admin api
+	app.Patch("/change-bucket-owner", controller.ChangeBucketOwner)
+
+	// ListBucketsAndOwners admin api
+	app.Patch("/list-buckets", controller.ListBuckets)
+}

s3api/admin-server.go

@@ -0,0 +1,72 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"crypto/tls"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+type S3AdminServer struct {
+	app     *fiber.App
+	backend backend.Backend
+	router  *S3AdminRouter
+	port    string
+	cert    *tls.Certificate
+}
+
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
+	server := &S3AdminServer{
+		app:     app,
+		backend: be,
+		router:  new(S3AdminRouter),
+		port:    port,
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
+
+	// Logging middlewares
+	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(nil))
+
+	// Authentication middlewares
+	app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(nil))
+	app.Use(middlewares.AclParser(be, nil))
+
+	server.router.Init(app, be, iam)
+
+	return server
+}
+
+type AdminOpt func(s *S3AdminServer)
+
+func WithAdminSrvTLS(cert tls.Certificate) AdminOpt {
+	return func(s *S3AdminServer) { s.cert = &cert }
+}
+
+func (sa *S3AdminServer) Serve() (err error) {
+	if sa.cert != nil {
+		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
+	}
+	return sa.app.Listen(sa.port)
+}

s3api/controllers/admin.go

@@ -15,47 +15,108 @@
 package controllers
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 )
 
 type AdminController struct {
-	IAMService auth.IAMService
+	iam auth.IAMService
+	be  backend.Backend
+}
+
+func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
+	return AdminController{iam: iam, be: be}
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
-	access, secret, role := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role")
-	requesterRole := ctx.Locals("role")
-
-	if requesterRole != "admin" {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
 		return fmt.Errorf("access denied: only admin users have access to this resource")
 	}
+	var usr auth.Account
+	err := json.Unmarshal(ctx.Body(), &usr)
+	if err != nil {
+		return fmt.Errorf("failed to parse request body: %w", err)
+	}
 
-	user := auth.Account{Secret: secret, Role: role}
+	if usr.Role != "user" && usr.Role != "admin" {
+		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
+	}
 
-	err := c.IAMService.CreateAccount(access, user)
+	err = c.iam.CreateAccount(usr)
 	if err != nil {
 		return fmt.Errorf("failed to create a user: %w", err)
 	}
 
-	ctx.SendString("The user has been created successfully")
-	return nil
+	return ctx.SendString("The user has been created successfully")
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
-	requesterRole := ctx.Locals("role")
-	if requesterRole != "admin" {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
 		return fmt.Errorf("access denied: only admin users have access to this resource")
 	}
 
-	err := c.IAMService.DeleteUserAccount(access)
+	err := c.iam.DeleteUserAccount(access)
 	if err != nil {
 		return err
 	}
 
-	ctx.SendString("The user has been created successfully")
-	return nil
+	return ctx.SendString("The user has been deleted successfully")
+}
+
+func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	accs, err := c.iam.ListUserAccounts()
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(accs)
+}
+
+func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	owner := ctx.Query("owner")
+	bucket := ctx.Query("bucket")
+
+	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return fmt.Errorf("user specified as the new bucket owner does not exist")
+	}
+
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
+	if err != nil {
+		return err
+	}
+
+	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+}
+
+func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+
+	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(buckets)
 }

s3api/controllers/admin_test.go

@@ -0,0 +1,481 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
+)
+
+func TestAdminController_CreateUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			CreateAccountFunc: func(account auth.Account) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/create-user", adminController.CreateUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	usr := auth.Account{
+		Access: "access",
+		Secret: "secret",
+		Role:   "invalid role",
+	}
+
+	user, _ := json.Marshal(&usr)
+
+	usr.Role = "admin"
+
+	succUsr, _ := json.Marshal(&usr)
+
+	appErr.Patch("/create-user", adminController.CreateUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-create-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-create-user-invalid-user-role",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-create-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.CreateUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.CreateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_DeleteUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			DeleteUserAccountFunc: func(access string) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/delete-user", adminController.DeleteUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/delete-user", adminController.DeleteUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-delete-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-delete-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.DeleteUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.DeleteUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ListUsers(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, fmt.Errorf("server error")
+			},
+		},
+	}
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/list-users", adminControllerErr.ListUsers)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-users", adminController.ListUsers)
+
+	appSucc := fiber.New()
+
+	appSucc.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appSucc.Patch("/list-users", adminController.ListUsers)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-list-users-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-iam-error",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-success",
+			app:  appSucc,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListUsers() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListUsers() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ChangeBucketOwner(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
+				return nil
+			},
+		},
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerIamErr := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, fmt.Errorf("unknown server error")
+			},
+		},
+	}
+
+	adminControllerIamAccDoesNotExist := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appIamErr := fiber.New()
+
+	appIamErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
+
+	appIamNoSuchUser := fiber.New()
+
+	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Change-bucket-owner-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-check-account-server-error",
+			app:  appIamErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-acc-does-not-exist",
+			app:  appIamNoSuchUser,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 201,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ChangeBucketOwner() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ChangeBucketOwner() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ListBuckets(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
+				return []s3response.Bucket{}, nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/list-buckets", adminController.ListBuckets)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "List-buckets-incorrect-role",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "List-buckets-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListBuckets() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}

s3api/controllers/iam_moq_test.go

@@ -0,0 +1,237 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package controllers
+
+import (
+	"github.com/versity/versitygw/auth"
+	"sync"
+)
+
+// Ensure, that IAMServiceMock does implement auth.IAMService.
+// If this is not the case, regenerate this file with moq.
+var _ auth.IAMService = &IAMServiceMock{}
+
+// IAMServiceMock is a mock implementation of auth.IAMService.
+//
+//	func TestSomethingThatUsesIAMService(t *testing.T) {
+//
+//		// make and configure a mocked auth.IAMService
+//		mockedIAMService := &IAMServiceMock{
+//			CreateAccountFunc: func(account auth.Account) error {
+//				panic("mock out the CreateAccount method")
+//			},
+//			DeleteUserAccountFunc: func(access string) error {
+//				panic("mock out the DeleteUserAccount method")
+//			},
+//			GetUserAccountFunc: func(access string) (auth.Account, error) {
+//				panic("mock out the GetUserAccount method")
+//			},
+//			ListUserAccountsFunc: func() ([]auth.Account, error) {
+//				panic("mock out the ListUserAccounts method")
+//			},
+//			ShutdownFunc: func() error {
+//				panic("mock out the Shutdown method")
+//			},
+//		}
+//
+//		// use mockedIAMService in code that requires auth.IAMService
+//		// and then make assertions.
+//
+//	}
+type IAMServiceMock struct {
+	// CreateAccountFunc mocks the CreateAccount method.
+	CreateAccountFunc func(account auth.Account) error
+
+	// DeleteUserAccountFunc mocks the DeleteUserAccount method.
+	DeleteUserAccountFunc func(access string) error
+
+	// GetUserAccountFunc mocks the GetUserAccount method.
+	GetUserAccountFunc func(access string) (auth.Account, error)
+
+	// ListUserAccountsFunc mocks the ListUserAccounts method.
+	ListUserAccountsFunc func() ([]auth.Account, error)
+
+	// ShutdownFunc mocks the Shutdown method.
+	ShutdownFunc func() error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// CreateAccount holds details about calls to the CreateAccount method.
+		CreateAccount []struct {
+			// Account is the account argument value.
+			Account auth.Account
+		}
+		// DeleteUserAccount holds details about calls to the DeleteUserAccount method.
+		DeleteUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+		}
+		// GetUserAccount holds details about calls to the GetUserAccount method.
+		GetUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+		}
+		// ListUserAccounts holds details about calls to the ListUserAccounts method.
+		ListUserAccounts []struct {
+		}
+		// Shutdown holds details about calls to the Shutdown method.
+		Shutdown []struct {
+		}
+	}
+	lockCreateAccount     sync.RWMutex
+	lockDeleteUserAccount sync.RWMutex
+	lockGetUserAccount    sync.RWMutex
+	lockListUserAccounts  sync.RWMutex
+	lockShutdown          sync.RWMutex
+}
+
+// CreateAccount calls CreateAccountFunc.
+func (mock *IAMServiceMock) CreateAccount(account auth.Account) error {
+	if mock.CreateAccountFunc == nil {
+		panic("IAMServiceMock.CreateAccountFunc: method is nil but IAMService.CreateAccount was just called")
+	}
+	callInfo := struct {
+		Account auth.Account
+	}{
+		Account: account,
+	}
+	mock.lockCreateAccount.Lock()
+	mock.calls.CreateAccount = append(mock.calls.CreateAccount, callInfo)
+	mock.lockCreateAccount.Unlock()
+	return mock.CreateAccountFunc(account)
+}
+
+// CreateAccountCalls gets all the calls that were made to CreateAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.CreateAccountCalls())
+func (mock *IAMServiceMock) CreateAccountCalls() []struct {
+	Account auth.Account
+} {
+	var calls []struct {
+		Account auth.Account
+	}
+	mock.lockCreateAccount.RLock()
+	calls = mock.calls.CreateAccount
+	mock.lockCreateAccount.RUnlock()
+	return calls
+}
+
+// DeleteUserAccount calls DeleteUserAccountFunc.
+func (mock *IAMServiceMock) DeleteUserAccount(access string) error {
+	if mock.DeleteUserAccountFunc == nil {
+		panic("IAMServiceMock.DeleteUserAccountFunc: method is nil but IAMService.DeleteUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+	}{
+		Access: access,
+	}
+	mock.lockDeleteUserAccount.Lock()
+	mock.calls.DeleteUserAccount = append(mock.calls.DeleteUserAccount, callInfo)
+	mock.lockDeleteUserAccount.Unlock()
+	return mock.DeleteUserAccountFunc(access)
+}
+
+// DeleteUserAccountCalls gets all the calls that were made to DeleteUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.DeleteUserAccountCalls())
+func (mock *IAMServiceMock) DeleteUserAccountCalls() []struct {
+	Access string
+} {
+	var calls []struct {
+		Access string
+	}
+	mock.lockDeleteUserAccount.RLock()
+	calls = mock.calls.DeleteUserAccount
+	mock.lockDeleteUserAccount.RUnlock()
+	return calls
+}
+
+// GetUserAccount calls GetUserAccountFunc.
+func (mock *IAMServiceMock) GetUserAccount(access string) (auth.Account, error) {
+	if mock.GetUserAccountFunc == nil {
+		panic("IAMServiceMock.GetUserAccountFunc: method is nil but IAMService.GetUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+	}{
+		Access: access,
+	}
+	mock.lockGetUserAccount.Lock()
+	mock.calls.GetUserAccount = append(mock.calls.GetUserAccount, callInfo)
+	mock.lockGetUserAccount.Unlock()
+	return mock.GetUserAccountFunc(access)
+}
+
+// GetUserAccountCalls gets all the calls that were made to GetUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.GetUserAccountCalls())
+func (mock *IAMServiceMock) GetUserAccountCalls() []struct {
+	Access string
+} {
+	var calls []struct {
+		Access string
+	}
+	mock.lockGetUserAccount.RLock()
+	calls = mock.calls.GetUserAccount
+	mock.lockGetUserAccount.RUnlock()
+	return calls
+}
+
+// ListUserAccounts calls ListUserAccountsFunc.
+func (mock *IAMServiceMock) ListUserAccounts() ([]auth.Account, error) {
+	if mock.ListUserAccountsFunc == nil {
+		panic("IAMServiceMock.ListUserAccountsFunc: method is nil but IAMService.ListUserAccounts was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockListUserAccounts.Lock()
+	mock.calls.ListUserAccounts = append(mock.calls.ListUserAccounts, callInfo)
+	mock.lockListUserAccounts.Unlock()
+	return mock.ListUserAccountsFunc()
+}
+
+// ListUserAccountsCalls gets all the calls that were made to ListUserAccounts.
+// Check the length with:
+//
+//	len(mockedIAMService.ListUserAccountsCalls())
+func (mock *IAMServiceMock) ListUserAccountsCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockListUserAccounts.RLock()
+	calls = mock.calls.ListUserAccounts
+	mock.lockListUserAccounts.RUnlock()
+	return calls
+}
+
+// Shutdown calls ShutdownFunc.
+func (mock *IAMServiceMock) Shutdown() error {
+	if mock.ShutdownFunc == nil {
+		panic("IAMServiceMock.ShutdownFunc: method is nil but IAMService.Shutdown was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockShutdown.Lock()
+	mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo)
+	mock.lockShutdown.Unlock()
+	return mock.ShutdownFunc()
+}
+
+// ShutdownCalls gets all the calls that were made to Shutdown.
+// Check the length with:
+//
+//	len(mockedIAMService.ShutdownCalls())
+func (mock *IAMServiceMock) ShutdownCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockShutdown.RLock()
+	calls = mock.calls.Shutdown
+	mock.lockShutdown.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -0,0 +1,61 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3log"
+)
+
+func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
+		path := ctx.Path()
+		pathParts := strings.Split(path, "/")
+		bucket := pathParts[1]
+		if path == "/" && ctx.Method() == http.MethodGet {
+			return ctx.Next()
+		}
+		if ctx.Method() == http.MethodPatch {
+			return ctx.Next()
+		}
+		if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
+			if err := auth.IsAdmin(acct, isRoot); err != nil {
+				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
+			}
+			return ctx.Next()
+		}
+		//TODO: provide correct action names for the logger, after implementing DetectAction middleware
+		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
+		parsedAcl, err := auth.ParseACL(data)
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
+		ctx.Locals("parsedAcl", parsedAcl)
+		return ctx.Next()
+	}
+}

s3api/middlewares/authentication.go

@@ -17,18 +17,18 @@ package middlewares
 import (
 	"crypto/sha256"
 	"encoding/hex"
-	"os"
-	"strings"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/aws/smithy-go/logging"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3api/controllers"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
 )
 
 const (
@@ -40,105 +40,102 @@ type RootUserConfig struct {
 	Secret string
 }
 
-func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, debug bool) fiber.Handler {
+func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
+		ctx.Locals("region", region)
+		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
 		if authorization == "" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty))
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
 		}
 
-		// Check the signature version
-		authParts := strings.Split(authorization, " ")
-		if len(authParts) < 4 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
-		}
-		if authParts[0] != "AWS4-HMAC-SHA256" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported))
+		authData, err := utils.ParseAuthorization(authorization)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
 		}
 
-		credKv := strings.Split(authParts[1], "=")
-		if len(credKv) != 2 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
-		}
-		creds := strings.Split(credKv[1], "/")
-		if len(creds) < 4 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
+		if authData.Algorithm != "AWS4-HMAC-SHA256" {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
 		}
 
-		signHdrKv := strings.Split(authParts[2][:len(authParts[2])-1], "=")
-		if len(signHdrKv) != 2 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
+		if authData.Region != region {
+			return sendResponse(ctx, s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
+				HTTPStatusCode: http.StatusForbidden,
+			}, logger)
 		}
-		signedHdrs := strings.Split(signHdrKv[1], ";")
 
-		account, err := acct.getAccount(creds[0])
+		ctx.Locals("isRoot", authData.Access == root.Access)
+
+		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID))
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
 		}
 		if err != nil {
-			return controllers.SendResponse(ctx, err)
+			return sendResponse(ctx, err, logger)
 		}
+		ctx.Locals("account", account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
 		if date == "" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader))
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
 		}
 
 		// Parse the date and check the date validity
 		tdate, err := time.Parse(iso8601Format, date)
 		if err != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate))
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
 		}
 
-		// Calculate the hash of the request payload
-		hashedPayload := sha256.Sum256(ctx.Body())
-		hexPayload := hex.EncodeToString(hashedPayload[:])
-
-		hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
-
-		// Compare the calculated hash with the hash provided
-		if hashPayloadHeader != hexPayload {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch))
+		if date[:8] != authData.Date {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
 		}
 
-		// Create a new http request instance from fasthttp request
-		req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
+		// Validate the dates difference
+		err = validateDate(tdate)
 		if err != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
+			return sendResponse(ctx, err, logger)
 		}
 
-		signer := v4.NewSigner()
+		if utils.IsBigDataAction(ctx) {
+			// for streaming PUT actions, authorization is deferred
+			// until end of stream due to need to get length and
+			// checksum of the stream to validate authorization
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
+			})
+			return ctx.Next()
+		}
 
-		signErr := signer.SignHTTP(req.Context(), aws.Credentials{
-			AccessKeyID:     creds[0],
-			SecretAccessKey: account.Secret,
-		}, req, hexPayload, creds[3], region, tdate, func(options *v4.SignerOptions) {
-			if debug {
-				options.LogSigning = true
-				options.Logger = logging.NewStandardLogger(os.Stderr)
+		hashPayload := ctx.Get("X-Amz-Content-Sha256")
+		if !utils.IsSpecialPayload(hashPayload) {
+			// Calculate the hash of the request payload
+			hashedPayload := sha256.Sum256(ctx.Body())
+			hexPayload := hex.EncodeToString(hashedPayload[:])
+
+			// Compare the calculated hash with the hash provided
+			if hashPayload != hexPayload {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
 			}
-		})
-		if signErr != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
 		}
 
-		parts := strings.Split(req.Header.Get("Authorization"), " ")
-		if len(parts) < 4 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
-		}
-		calculatedSign := strings.Split(parts[3], "=")[1]
-		expectedSign := strings.Split(authParts[3], "=")[1]
-
-		if expectedSign != calculatedSign {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch))
+		var contentLength int64
+		contentLengthStr := ctx.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+			if err != nil {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
+			}
 		}
 
-		ctx.Locals("role", account.Role)
-		ctx.Locals("access", creds[0])
-		ctx.Locals("isRoot", creds[0] == root.Access)
+		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
 
 		return ctx.Next()
 	}
@@ -152,6 +149,7 @@ type accounts struct {
 func (a accounts) getAccount(access string) (auth.Account, error) {
 	if access == a.root.Access {
 		return auth.Account{
+			Access: a.root.Access,
 			Secret: a.root.Secret,
 			Role:   "admin",
 		}, nil
@@ -159,3 +157,30 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
 
 	return a.iam.GetUserAccount(access)
 }
+
+func validateDate(date time.Time) error {
+	now := time.Now().UTC()
+	diff := date.Unix() - now.Unix()
+
+	// Checks the dates difference to be less than a minute
+	if diff > 60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+	if diff < -60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+
+	return nil
+}
+
+func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
+	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+}

s3api/middlewares/body-reader.go

@@ -0,0 +1,31 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
+	r, ok := ctx.Locals("body-reader").(io.Reader)
+	if !ok {
+		r = ctx.Request().BodyStream()
+	}
+
+	r = wr(r)
+	ctx.Locals("body-reader", r)
+}

s3api/middlewares/chunk.go

@@ -0,0 +1,61 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3log"
+)
+
+// ProcessChunkedBody initializes the chunked upload stream if the
+// request appears to be a chunked upload
+func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
+		if decodedLength == "" {
+			return ctx.Next()
+		}
+		// TODO: validate content length
+
+		authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		acct := ctx.Locals("account").(auth.Account)
+		amzdate := ctx.Get("X-Amz-Date")
+		date, _ := time.Parse(iso8601Format, amzdate)
+
+		if utils.IsBigDataAction(ctx) {
+			var err error
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				var cr *utils.ChunkReader
+				cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
+				return cr
+			})
+			if err != nil {
+				return sendResponse(ctx, err, logger)
+			}
+			return ctx.Next()
+		}
+
+		return ctx.Next()
+	}
+}

s3api/middlewares/md5.go

@@ -16,28 +16,41 @@ package middlewares
 
 import (
 	"crypto/md5"
-	"encoding/base64"
+	"io"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
 )
 
-func VerifyMD5Body() fiber.Handler {
+func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		incomingSum := ctx.Get("Content-Md5")
 		if incomingSum == "" {
 			return ctx.Next()
 		}
 
+		if utils.IsBigDataAction(ctx) {
+			var err error
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				r, err = utils.NewHashReader(r, incomingSum, utils.HashTypeMd5)
+				return r
+			})
+			if err != nil {
+				return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+			}
+			return ctx.Next()
+		}
+
 		sum := md5.Sum(ctx.Body())
-		calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
+		calculatedSum := utils.Md5SumString(sum[:])
 
 		if incomingSum != calculatedSum {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest))
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
 		}
 
 		return ctx.Next()
-
 	}
 }

s3api/middlewares/url-decoder.go

@@ -0,0 +1,36 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/url"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		reqURL := ctx.Request().URI().String()
+		decoded, err := url.Parse(reqURL)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+		}
+		ctx.Path(decoded.Path)
+		return ctx.Next()
+	}
+}

s3api/router.go

@@ -19,23 +19,40 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3event"
+	"github.com/versity/versitygw/s3log"
 )
 
-type S3ApiRouter struct{}
+type S3ApiRouter struct {
+	WithAdmSrv bool
+}
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
-	s3ApiController := controllers.New(be)
-	adminController := controllers.AdminController{IAMService: iam}
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
+	s3ApiController := controllers.New(be, iam, logger, evs)
 
-	// TODO: think of better routing system
-	app.Post("/create-user", adminController.CreateUser)
+	if sa.WithAdmSrv {
+		adminController := controllers.NewAdminController(iam, be)
+
+		// CreateUser admin api
+		app.Patch("/create-user", adminController.CreateUser)
+
+		// DeleteUsers admin api
+		app.Patch("/delete-user", adminController.DeleteUser)
+
+		// ListUsers admin api
+		app.Patch("/list-users", adminController.ListUsers)
+
+		// ChangeBucketOwner admin api
+		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+		// ListBucketsAndOwners admin api
+		app.Patch("/list-buckets", adminController.ListBuckets)
+	}
 
-	// Admin Delete api
-	app.Delete("/delete-user", adminController.DeleteUser)
 	// ListBuckets action
 	app.Get("/", s3ApiController.ListBuckets)
 
-	// PutBucket action
+	// CreateBucket action
 	// PutBucketAcl action
 	app.Put("/:bucket", s3ApiController.PutBucketActions)
 
@@ -44,6 +61,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 
 	// HeadBucket
 	app.Head("/:bucket", s3ApiController.HeadBucket)
+
 	// GetBucketAcl action
 	// ListMultipartUploads action
 	// ListObjects action
@@ -52,22 +70,34 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 
 	// HeadObject action
 	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
+
 	// GetObjectAcl action
 	// GetObject action
 	// ListObjectParts action
+	// GetObjectTagging action
+	// ListParts action
+	// GetObjectAttributes action
 	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
+
 	// DeleteObject action
 	// AbortMultipartUpload action
+	// DeleteObjectTagging action
 	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
+
 	// DeleteObjects action
 	app.Post("/:bucket", s3ApiController.DeleteObjects)
+
 	// CompleteMultipartUpload action
 	// CreateMultipartUpload
 	// RestoreObject action
+	// SelectObjectContent action
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
+
 	// CopyObject action
 	// PutObject action
 	// UploadPart action
 	// UploadPartCopy action
+	// PutObjectTagging action
+	// PutObjectAcl action
 	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
 }

s3api/router_test.go

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
 		})
 	}
 }

s3api/server.go

@@ -22,6 +22,8 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/middlewares"
+	"github.com/versity/versitygw/s3event"
+	"github.com/versity/versitygw/s3log"
 )
 
 type S3ApiServer struct {
@@ -30,10 +32,11 @@ type S3ApiServer struct {
 	router  *S3ApiRouter
 	port    string
 	cert    *tls.Certificate
+	quiet   bool
 	debug   bool
 }
 
-func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) {
+func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
 	server := &S3ApiServer{
 		app:     app,
 		backend: be,
@@ -46,14 +49,19 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 	}
 
 	// Logging middlewares
-	app.Use(logger.New())
+	if !server.quiet {
+		app.Use(logger.New())
+	}
+	app.Use(middlewares.DecodeURL(l))
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyV4Signature(root, iam, region, server.debug))
-	app.Use(middlewares.VerifyMD5Body())
+	app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
+	app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
+	app.Use(middlewares.VerifyMD5Body(l))
+	app.Use(middlewares.AclParser(be, l))
 
-	server.router.Init(app, be, iam)
+	server.router.Init(app, be, iam, l, evs)
 
 	return server, nil
 }
@@ -66,11 +74,21 @@ func WithTLS(cert tls.Certificate) Option {
 	return func(s *S3ApiServer) { s.cert = &cert }
 }
 
+// WithAdminServer runs admin endpoints with the gateway in the same network
+func WithAdminServer() Option {
+	return func(s *S3ApiServer) { s.router.WithAdmSrv = true }
+}
+
 // WithDebug sets debug output
 func WithDebug() Option {
 	return func(s *S3ApiServer) { s.debug = true }
 }
 
+// WithQuiet silences default logging output
+func WithQuiet() Option {
+	return func(s *S3ApiServer) { s.quiet = true }
+}
+
 func (sa *S3ApiServer) Serve() (err error) {
 	if sa.cert != nil {
 		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

s3api/server_test.go

@@ -15,6 +15,7 @@
 package s3api
 
 import (
+	"crypto/tls"
 	"reflect"
 	"testing"
 
@@ -63,7 +64,7 @@ func TestNew(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
-				tt.args.port, "us-east-1", &auth.IAMServiceInternal{})
+				tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -82,15 +83,26 @@ func TestS3ApiServer_Serve(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			name:    "Return error when serving S3 api server with invalid address",
+			name:    "Serve-invalid-address",
 			wantErr: true,
 			sa: &S3ApiServer{
 				app:     fiber.New(),
 				backend: backend.BackendUnsupported{},
-				port:    "Wrong address",
+				port:    "Invalid address",
 				router:  &S3ApiRouter{},
 			},
 		},
+		{
+			name:    "Serve-invalid-address-with-certificate",
+			wantErr: true,
+			sa: &S3ApiServer{
+				app:     fiber.New(),
+				backend: backend.BackendUnsupported{},
+				port:    "Invalid address",
+				router:  &S3ApiRouter{},
+				cert:    &tls.Certificate{},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

s3api/utils/auth-reader.go

@@ -0,0 +1,275 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go/logging"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	iso8601Format = "20060102T150405Z"
+	yyyymmdd      = "20060102"
+)
+
+// AuthReader is an io.Reader that validates the request authorization
+// once the underlying reader returns io.EOF.  This is needed for streaming
+// data requests where the data size and checksum are not known until
+// the data is completely read.
+type AuthReader struct {
+	ctx    *fiber.Ctx
+	auth   AuthData
+	secret string
+	size   int
+	r      *HashReader
+	debug  bool
+}
+
+// NewAuthReader initializes an io.Reader that will verify the request
+// v4 auth when the underlying reader returns io.EOF. This postpones the
+// authorization check until the reader is consumed. So it is important that
+// the consumer of this reader checks for the auth errors while reading.
+func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *AuthReader {
+	var hr *HashReader
+	hashPayload := ctx.Get("X-Amz-Content-Sha256")
+	if !IsSpecialPayload(hashPayload) {
+		hr, _ = NewHashReader(r, "", HashTypeSha256)
+	} else {
+		hr, _ = NewHashReader(r, "", HashTypeNone)
+	}
+
+	return &AuthReader{
+		ctx:    ctx,
+		r:      hr,
+		auth:   auth,
+		secret: secret,
+		debug:  debug,
+	}
+}
+
+// Read allows *AuthReader to be used as an io.Reader
+func (ar *AuthReader) Read(p []byte) (int, error) {
+	n, err := ar.r.Read(p)
+	ar.size += n
+
+	if errors.Is(err, io.EOF) {
+		verr := ar.validateSignature()
+		if verr != nil {
+			return n, verr
+		}
+	}
+
+	return n, err
+}
+
+func (ar *AuthReader) validateSignature() error {
+	date := ar.ctx.Get("X-Amz-Date")
+	if date == "" {
+		return s3err.GetAPIError(s3err.ErrMissingDateHeader)
+	}
+
+	hashPayload := ar.ctx.Get("X-Amz-Content-Sha256")
+	if !IsSpecialPayload(hashPayload) {
+		hexPayload := ar.r.Sum()
+
+		// Compare the calculated hash with the hash provided
+		if hashPayload != hexPayload {
+			return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+		}
+	}
+
+	// Parse the date and check the date validity
+	tdate, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrMalformedDate)
+	}
+
+	return CheckValidSignature(ar.ctx, ar.auth, ar.secret, hashPayload, tdate, int64(ar.size), ar.debug)
+}
+
+const (
+	service = "s3"
+)
+
+// CheckValidSignature validates the ctx v4 auth signature
+func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64, debug bool) error {
+	signedHdrs := strings.Split(auth.SignedHeaders, ";")
+
+	// Create a new http request instance from fasthttp request
+	req, err := createHttpRequestFromCtx(ctx, signedHdrs, contentLen)
+	if err != nil {
+		return fmt.Errorf("create http request from context: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{
+		AccessKeyID:     auth.Access,
+		SecretAccessKey: secret,
+	}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
+		options.DisableURIPathEscaping = true
+		if debug {
+			options.LogSigning = true
+			options.Logger = logging.NewStandardLogger(os.Stderr)
+		}
+	})
+	if signErr != nil {
+		return fmt.Errorf("sign generated http request: %w", err)
+	}
+
+	genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
+	if err != nil {
+		return err
+	}
+
+	if auth.Signature != genAuth.Signature {
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+
+	return nil
+}
+
+// AuthData is the parsed authorization data from the header
+type AuthData struct {
+	Algorithm     string
+	Access        string
+	Region        string
+	SignedHeaders string
+	Signature     string
+	Date          string
+}
+
+// ParseAuthorization returns the parsed fields for the aws v4 auth header
+// example authorization string from aws docs:
+// Authorization: AWS4-HMAC-SHA256
+// Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,
+// SignedHeaders=host;range;x-amz-date,
+// Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
+func ParseAuthorization(authorization string) (AuthData, error) {
+	a := AuthData{}
+
+	// authorization must start with:
+	// Authorization: <ALGORITHM>
+	// followed by key=value pairs separated by ","
+	authParts := strings.SplitN(authorization, " ", 2)
+	for i, el := range authParts {
+		if strings.Contains(el, " ") {
+			authParts[i] = removeSpace(el)
+		}
+	}
+
+	if len(authParts) < 2 {
+		return a, s3err.GetAPIError(s3err.ErrMissingFields)
+	}
+
+	algo := authParts[0]
+
+	kvData := authParts[1]
+	kvPairs := strings.Split(kvData, ",")
+	// we are expecting at least Credential, SignedHeaders, and Signature
+	// key value pairs here
+	if len(kvPairs) < 3 {
+		return a, s3err.GetAPIError(s3err.ErrMissingFields)
+	}
+
+	var access, region, signedHeaders, signature, date string
+
+	for _, kv := range kvPairs {
+		keyValue := strings.Split(kv, "=")
+		if len(keyValue) != 2 {
+			switch {
+			case strings.HasPrefix(kv, "Credential"):
+				return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+			case strings.HasPrefix(kv, "SignedHeaders"):
+				return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+			}
+			return a, s3err.GetAPIError(s3err.ErrMissingFields)
+		}
+		key := strings.TrimSpace(keyValue[0])
+		value := strings.TrimSpace(keyValue[1])
+
+		switch key {
+		case "Credential":
+			creds := strings.Split(value, "/")
+			if len(creds) != 5 {
+				return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+			}
+			if creds[3] != "s3" {
+				return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
+			}
+			if creds[4] != "aws4_request" {
+				return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
+			}
+			_, err := time.Parse(yyyymmdd, creds[1])
+			if err != nil {
+				return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
+			}
+			access = creds[0]
+			date = creds[1]
+			region = creds[2]
+		case "SignedHeaders":
+			signedHeaders = value
+		case "Signature":
+			signature = value
+		}
+	}
+
+	return AuthData{
+		Algorithm:     algo,
+		Access:        access,
+		Region:        region,
+		SignedHeaders: signedHeaders,
+		Signature:     signature,
+		Date:          date,
+	}, nil
+}
+
+func removeSpace(str string) string {
+	var b strings.Builder
+	b.Grow(len(str))
+	for _, ch := range str {
+		if !unicode.IsSpace(ch) {
+			b.WriteRune(ch)
+		}
+	}
+	return b.String()
+}
+
+var (
+	specialValues = map[string]bool{
+		"UNSIGNED-PAYLOAD":                                 true,
+		"STREAMING-UNSIGNED-PAYLOAD-TRAILER":               true,
+		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD":               true,
+		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER":       true,
+		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD":         true,
+		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
+	}
+)
+
+// IsSpecialPayload checks for streaming/unsigned authorization types
+func IsSpecialPayload(str string) bool {
+	return specialValues[str]
+}

s3api/utils/auth_test.go

@@ -0,0 +1,40 @@
+package utils
+
+import (
+	"testing"
+)
+
+func TestAuthParse(t *testing.T) {
+	vectors := []struct {
+		name    string // name of test string
+		authstr string // Authorization string
+		algo    string
+		sig     string
+	}{{
+		name:    "restic",
+		authstr: "AWS4-HMAC-SHA256 Credential=user/20240116/us-east-1/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
+		algo:    "AWS4-HMAC-SHA256",
+		sig:     "d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
+	},
+		{
+			name:    "aws eaxample",
+			authstr: "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-date, Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
+			algo:    "AWS4-HMAC-SHA256",
+			sig:     "fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
+		}}
+
+	for _, v := range vectors {
+		t.Run(v.name, func(t *testing.T) {
+			data, err := ParseAuthorization(v.authstr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if data.Algorithm != v.algo {
+				t.Errorf("algo got %v, expected %v", data.Algorithm, v.algo)
+			}
+			if data.Signature != v.sig {
+				t.Errorf("signature got %v, expected %v", data.Signature, v.sig)
+			}
+		})
+	}
+}

s3api/utils/chunk-reader.go

@@ -0,0 +1,269 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"strconv"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+// chunked uploads described in:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+
+const (
+	chunkHdrStr       = ";chunk-signature="
+	chunkHdrDelim     = "\r\n"
+	zeroLenSig        = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	awsV4             = "AWS4"
+	awsS3Service      = "s3"
+	awsV4Request      = "aws4_request"
+	streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
+)
+
+// ChunkReader reads from chunked upload request body, and returns
+// object data stream
+type ChunkReader struct {
+	r                io.Reader
+	signingKey       []byte
+	prevSig          string
+	parsedSig        string
+	currentChunkSize int64
+	chunkDataLeft    int64
+	trailerExpected  int
+	stash            []byte
+	chunkHash        hash.Hash
+	strToSignPrefix  string
+	skipcheck        bool
+}
+
+// NewChunkReader reads from request body io.Reader and parses out the
+// chunk metadata in stream. The headers are validated for proper signatures.
+// Reading from the chunk reader will read only the object data stream
+// without the chunk headers/trailers.
+func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (*ChunkReader, error) {
+	return &ChunkReader{
+		r:          r,
+		signingKey: getSigningKey(secret, region, date),
+		// the authdata.Signature is validated in the auth-reader,
+		// so we can use that here without any other checks
+		prevSig:         authdata.Signature,
+		chunkHash:       sha256.New(),
+		strToSignPrefix: getStringToSignPrefix(date, region),
+	}, nil
+}
+
+// Read satisfies the io.Reader for this type
+func (cr *ChunkReader) Read(p []byte) (int, error) {
+	n, err := cr.r.Read(p)
+	if err != nil && err != io.EOF {
+		return n, err
+	}
+
+	if cr.chunkDataLeft < int64(n) {
+		chunkSize := cr.chunkDataLeft
+		if chunkSize > 0 {
+			cr.chunkHash.Write(p[:chunkSize])
+		}
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		n += int(chunkSize)
+		return n, err
+	}
+
+	cr.chunkDataLeft -= int64(n)
+	cr.chunkHash.Write(p[:n])
+	return n, err
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// This part is the same for all chunks,
+// only the previous signature and hash of current chunk changes
+func getStringToSignPrefix(date time.Time, region string) string {
+	credentialScope := fmt.Sprintf("%s/%s/%s/%s",
+		date.Format("20060102"),
+		region,
+		awsS3Service,
+		awsV4Request)
+
+	return fmt.Sprintf("%s\n%s\n%s",
+		streamPayloadAlgo,
+		date.Format("20060102T150405Z"),
+		credentialScope)
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+// signature For each chunk, you calculate the signature using the following
+// string to sign. For the first chunk, you use the seed-signature as the
+// previous signature.
+func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
+	return fmt.Sprintf("%s\n%s\n%s\n%s",
+		prefix,
+		prevSig,
+		zeroLenSig,
+		hex.EncodeToString(chunkHash))
+}
+
+// The provided p should have all of the previous chunk data and trailer
+// consumed already. The positioning here is expected that p[0] starts the
+// new chunk size with the ";chunk-signature=" following. The only exception
+// is if we started consuming the trailer, but hit the end of the read buffer.
+// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
+// finish consuming the final trailer bytes.
+// This parses the chunk metadata in situ without allocating an extra buffer.
+// It will just read and validate the chunk metadata and then move the
+// following chunk data to overwrite the metadata in the provided buffer.
+func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
+	n := len(p)
+
+	if !cr.skipcheck && cr.parsedSig != "" {
+		chunkhash := cr.chunkHash.Sum(nil)
+		cr.chunkHash.Reset()
+
+		sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
+		cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
+
+		if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
+			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+		}
+	}
+
+	if cr.trailerExpected != 0 {
+		if len(p) < len(chunkHdrDelim) {
+			// This is the special case where we need to consume the
+			// trailer, but instead hit the end of the buffer. The
+			// subsequent call will finish consuming the trailer.
+			cr.chunkDataLeft = 0
+			cr.trailerExpected -= len(p)
+			cr.skipcheck = true
+			return 0, nil
+		}
+		// move data up to remove trailer
+		copy(p, p[cr.trailerExpected:])
+		n -= cr.trailerExpected
+	}
+
+	cr.skipcheck = false
+
+	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
+	cr.currentChunkSize = chunkSize
+	cr.parsedSig = sig
+	if err == errskipHeader {
+		cr.chunkDataLeft = 0
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	if chunkSize == 0 {
+		return 0, io.EOF
+	}
+
+	cr.trailerExpected = len(chunkHdrDelim)
+
+	// move data up to remove chunk header
+	copy(p, p[bufOffset:n])
+	n -= bufOffset
+
+	// if remaining buffer larger than chunk data,
+	// parse next header in buffer
+	if int64(n) > chunkSize {
+		cr.chunkDataLeft = 0
+		cr.chunkHash.Write(p[:chunkSize])
+		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
+		return n + int(chunkSize), err
+	} else {
+		cr.chunkDataLeft = chunkSize - int64(n)
+		cr.chunkHash.Write(p[:n])
+	}
+
+	return n, nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+// Task 3: Calculate Signature
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
+func getSigningKey(secret, region string, date time.Time) []byte {
+	dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
+	dateRegionKey := hmac256(dateKey, []byte(region))
+	dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
+	signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
+	return signingKey
+}
+
+func hmac256(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+var (
+	errInvalidChunkFormat = errors.New("invalid chunk header format")
+	errskipHeader         = errors.New("skip to next header")
+)
+
+const (
+	maxHeaderSize = 1024
+)
+
+// Theis returns the chunk payload size, signature, data start offset, and
+// error if any. See the AWS documentation for the chunk header format. The
+// header[0] byte is expected to be the first byte of the chunk size here.
+func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
+	if cr.stash != nil {
+		tmp := make([]byte, maxHeaderSize)
+		copy(tmp, cr.stash)
+		copy(tmp[len(cr.stash):], header)
+		header = tmp
+		cr.stash = nil
+	}
+
+	semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
+	if semicolonIndex == -1 {
+		cr.stash = make([]byte, len(header))
+		copy(cr.stash, header)
+		cr.trailerExpected = 0
+		return 0, "", 0, errskipHeader
+	}
+
+	sigIndex := semicolonIndex + len(chunkHdrStr)
+	sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
+	if sigEndIndex == -1 {
+		cr.stash = make([]byte, len(header))
+		copy(cr.stash, header)
+		cr.trailerExpected = 0
+		return 0, "", 0, errskipHeader
+	}
+
+	chunkSizeBytes := header[:semicolonIndex]
+	chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
+	if err != nil {
+		return 0, "", 0, errInvalidChunkFormat
+	}
+
+	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
+	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
+
+	return chunkSize, signature, dataStartOffset, nil
+}

s3api/utils/csum-reader.go

@@ -0,0 +1,130 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"hash"
+	"io"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+// HashType identifies the checksum algorithm to be used
+type HashType string
+
+const (
+	// HashTypeMd5 generates MD5 checksum for the data stream
+	HashTypeMd5 = "md5"
+	// HashTypeSha256 generates SHA256 checksum for the data stream
+	HashTypeSha256 = "sha256"
+	// HashTypeNone is a no-op checksum for the data stream
+	HashTypeNone = "none"
+)
+
+// HashReader is an io.Reader that calculates the checksum
+// as the data is read
+type HashReader struct {
+	hashType HashType
+	hash     hash.Hash
+	r        io.Reader
+	sum      string
+}
+
+var (
+	errInvalidHashType = errors.New("unsupported or invalid checksum type")
+)
+
+// NewHashReader intializes an io.Reader from an underlying io.Reader that
+// calculates the checksum while the reader is being read from. If the
+// sum provided is not "", the reader will return an error when the underlying
+// reader returns io.EOF if the checksum does not match the provided expected
+// checksum.  If the provided sum is "", then the Sum() method can still
+// be used to get the current checksum for the data read so far.
+func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
+	var hash hash.Hash
+	switch ht {
+	case HashTypeMd5:
+		hash = md5.New()
+	case HashTypeSha256:
+		hash = sha256.New()
+	case HashTypeNone:
+		hash = noop{}
+	default:
+		return nil, errInvalidHashType
+	}
+
+	return &HashReader{
+		hash:     hash,
+		r:        r,
+		sum:      expectedSum,
+		hashType: ht,
+	}, nil
+}
+
+// Read allows *HashReader to be used as an io.Reader
+func (hr *HashReader) Read(p []byte) (int, error) {
+	n, readerr := hr.r.Read(p)
+	_, err := hr.hash.Write(p[:n])
+	if err != nil {
+		return n, err
+	}
+	if errors.Is(readerr, io.EOF) && hr.sum != "" {
+		switch hr.hashType {
+		case HashTypeMd5:
+			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
+			}
+		case HashTypeSha256:
+			sum := hex.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+			}
+		default:
+			return n, errInvalidHashType
+		}
+	}
+	return n, readerr
+}
+
+// Sum returns the checksum hash of the data read so far
+func (hr *HashReader) Sum() string {
+	switch hr.hashType {
+	case HashTypeMd5:
+		return Md5SumString(hr.hash.Sum(nil))
+	case HashTypeSha256:
+		return hex.EncodeToString(hr.hash.Sum(nil))
+	default:
+		return ""
+	}
+}
+
+// Md5SumString converts the hash bytes to the string checksum value
+func Md5SumString(b []byte) string {
+	return base64.StdEncoding.EncodeToString(b)
+}
+
+type noop struct{}
+
+func (n noop) Write(p []byte) (int, error) { return 0, nil }
+func (n noop) Sum(b []byte) []byte         { return []byte{} }
+func (n noop) Reset()                      {}
+func (n noop) Size() int                   { return 0 }
+func (n noop) BlockSize() int              { return 1 }

s3api/utils/utils.go

@@ -18,30 +18,48 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
+	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	bucketNameRegexp   = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]+[a-z0-9]$`)
+	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
-	headers.VisitAll(func(key, value []byte) {
-		if strings.HasPrefix(string(key), "X-Amz-Meta-") {
-			trimmedKey := strings.TrimPrefix(string(key), "X-Amz-Meta-")
+	headers.DisableNormalizing()
+	headers.VisitAllInOrder(func(key, value []byte) {
+		hKey := string(key)
+		if strings.HasPrefix(strings.ToLower(hKey), "x-amz-meta-") {
+			trimmedKey := hKey[11:]
 			headerValue := string(value)
 			metadata[trimmedKey] = headerValue
 		}
 	})
+	headers.EnableNormalizing()
 
 	return
 }
 
-func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
+func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
 	req := ctx.Request()
+	var body io.Reader
+	if IsBigDataAction(ctx) {
+		body = req.BodyStream()
+	} else {
+		body = bytes.NewReader(req.Body())
+	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
+	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -58,6 +76,8 @@ func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Reques
 	// If content length is non 0, then the header will be included
 	if !includeHeader("Content-Length", signedHdrs) {
 		httpReq.ContentLength = 0
+	} else {
+		httpReq.ContentLength = contentLength
 	}
 
 	// Set the Host header
@@ -67,9 +87,22 @@ func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Reques
 }
 
 func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
+	ctx.Response().Header.DisableNormalizing()
 	for key, val := range meta {
-		ctx.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
+		ctx.Response().Header.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
 	}
+	ctx.Response().Header.EnableNormalizing()
+}
+
+func ParseUint(str string) (int32, error) {
+	if str == "" {
+		return 1000, nil
+	}
+	num, err := strconv.ParseUint(str, 10, 16)
+	if err != nil {
+		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+	}
+	return int32(num), nil
 }
 
 type CustomHeader struct {
@@ -83,6 +116,22 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
 	}
 }
 
+func IsValidBucketName(bucket string) bool {
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	// Checks to contain only digits, lowercase letters, dot, hyphen.
+	// Checks to start and end with only digits and lowercase letters.
+	if !bucketNameRegexp.MatchString(bucket) {
+		return false
+	}
+	// Checks not to be a valid IP address
+	if bucketNameIpRegexp.MatchString(bucket) {
+		return false
+	}
+	return true
+}
+
 func includeHeader(hdr string, signedHdrs []string) bool {
 	for _, shdr := range signedHdrs {
 		if strings.EqualFold(hdr, shdr) {
@@ -91,3 +140,12 @@ func includeHeader(hdr string, signedHdrs []string) bool {
 	}
 	return false
 }
+
+func IsBigDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}

s3api/utils/utils_test.go

@@ -55,7 +55,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := CreateHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"})
+			got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -79,13 +79,6 @@ func TestGetUserMetaData(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 	req := ctx.Request()
 
-	// Case 2
-	ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
-	req2 := ctx2.Request()
-
-	req2.Header.Add("X-Amz-Meta-Name", "Nick")
-	req2.Header.Add("X-Amz-Meta-Age", "27")
-
 	tests := []struct {
 		name         string
 		args         args
@@ -98,16 +91,6 @@ func TestGetUserMetaData(t *testing.T) {
 			},
 			wantMetadata: map[string]string{},
 		},
-		{
-			name: "Success-non-empty-response",
-			args: args{
-				headers: &req2.Header,
-			},
-			wantMetadata: map[string]string{
-				"Age":  "27",
-				"Name": "Nick",
-			},
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -117,3 +100,164 @@ func TestGetUserMetaData(t *testing.T) {
 		})
 	}
 }
+
+func Test_includeHeader(t *testing.T) {
+	type args struct {
+		hdr        string
+		signedHdrs []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "include-header-falsy-case",
+			args: args{
+				hdr:        "Content-Type",
+				signedHdrs: []string{"X-Amz-Acl", "Content-Encoding"},
+			},
+			want: false,
+		},
+		{
+			name: "include-header-falsy-case",
+			args: args{
+				hdr:        "Content-Type",
+				signedHdrs: []string{"X-Amz-Acl", "Content-Type"},
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := includeHeader(tt.args.hdr, tt.args.signedHdrs); got != tt.want {
+				t.Errorf("includeHeader() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsValidBucketName(t *testing.T) {
+	type args struct {
+		bucket string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "IsValidBucketName-short-name",
+			args: args{
+				bucket: "a",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-start-with-hyphen",
+			args: args{
+				bucket: "-bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-start-with-dot",
+			args: args{
+				bucket: ".bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-contain-invalid-character",
+			args: args{
+				bucket: "my@bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-end-with-hyphen",
+			args: args{
+				bucket: "bucket-",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-end-with-dot",
+			args: args{
+				bucket: "bucket.",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-valid-bucket-name",
+			args: args{
+				bucket: "my-bucket",
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidBucketName(tt.args.bucket); got != tt.want {
+				t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseUint(t *testing.T) {
+	type args struct {
+		str string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    int32
+		wantErr bool
+	}{
+		{
+			name: "Parse-uint-empty-string",
+			args: args{
+				str: "",
+			},
+			want:    1000,
+			wantErr: false,
+		},
+		{
+			name: "Parse-uint-invalid-number-string",
+			args: args{
+				str: "bla",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-invalid-negative-number",
+			args: args{
+				str: "-5",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-success",
+			args: args{
+				str: "23",
+			},
+			want:    23,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseUint(tt.args.str)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseMaxKeys() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ParseMaxKeys() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -97,6 +97,9 @@ const (
 	ErrNegativeExpires
 	ErrMaximumExpires
 	ErrSignatureDoesNotMatch
+	ErrSignatureDateDoesNotMatch
+	ErrSignatureTerminationStr
+	ErrSignatureIncorrService
 	ErrContentSHA256Mismatch
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
@@ -106,9 +109,13 @@ const (
 	ErrNotImplemented
 	ErrPreconditionFailed
 	ErrInvalidObjectState
+	ErrInvalidRange
+	ErrInvalidURI
 
+	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
+	ErrDirectoryObjectContainsData
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -187,13 +194,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "We encountered an internal error, please try again.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
-
 	ErrInvalidPart: {
 		Code:           "InvalidPart",
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -284,7 +289,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Signature header missing Signature field.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
 		Description:    "There were headers present in the request which were not signed",
@@ -320,25 +324,36 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
 		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
 	ErrRequestNotReadyYet: {
 		Code:           "AccessDenied",
 		Description:    "Request is not valid yet",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
 	ErrSignatureDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
 		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
+	ErrSignatureDateDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureTerminationStr: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureIncorrService: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped to correct service: s3",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrContentSHA256Mismatch: {
 		Code:           "XAmzContentSHA256Mismatch",
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
@@ -374,6 +389,16 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The operation is not valid for the current state of the object",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidRange: {
+		Code:           "InvalidRange",
+		Description:    "The requested range is not valid for the request. Try another range.",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
+	ErrInvalidURI: {
+		Code:           "InvalidURI",
+		Description:    "The specified URI couldn't be parsed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrExistingObjectIsDirectory: {
 		Code:           "ExistingObjectIsDirectory",
 		Description:    "Existing Object is a directory.",
@@ -384,6 +409,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Object parent already exists as a file.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrDirectoryObjectContainsData: {
+		Code:           "DirectoryObjectContainsData",
+		Description:    "Directory object contains data payload.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3event/event.go

@@ -0,0 +1,130 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"fmt"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type S3EventSender interface {
+	SendEvent(ctx *fiber.Ctx, meta EventMeta)
+}
+
+type EventMeta struct {
+	BucketOwner string
+	EventName   EventType
+	ObjectSize  int64
+	ObjectETag  *string
+	VersionId   *string
+}
+
+type EventFields struct {
+	Records []EventSchema
+}
+
+type EventType string
+
+const (
+	EventObjectPut               EventType = "s3:ObjectCreated:Put"
+	EventObjectCopy              EventType = "s3:ObjectCreated:Copy"
+	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
+	EventObjectDelete            EventType = "s3:ObjectRemoved:Delete"
+	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
+	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
+	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
+	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
+	// Not supported
+	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
+)
+
+type EventSchema struct {
+	EventVersion      string                `json:"eventVersion"`
+	EventSource       string                `json:"eventSource"`
+	AwsRegion         string                `json:"awsRegion"`
+	EventTime         string                `json:"eventTime"`
+	EventName         EventType             `json:"eventName"`
+	UserIdentity      EventUserIdentity     `json:"userIdentity"`
+	RequestParameters EventRequestParams    `json:"requestParameters"`
+	ResponseElements  EventResponseElements `json:"responseElements"`
+	S3                EventS3Data           `json:"s3"`
+	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
+}
+
+type EventUserIdentity struct {
+	PrincipalId string `json:"PrincipalId"`
+}
+
+type EventRequestParams struct {
+	SourceIPAddress string `json:"sourceIPAddress"`
+}
+
+type EventResponseElements struct {
+	RequestId string `json:"x-amz-request-id"`
+	HostId    string `json:"x-amz-id-2"`
+}
+
+type EventS3Data struct {
+	S3SchemaVersion string            `json:"s3SchemaVersion"`
+	ConfigurationId string            `json:"configurationId"`
+	Bucket          EventS3BucketData `json:"bucket"`
+	Object          EventObjectData   `json:"object"`
+}
+
+type EventGlacierData struct {
+	RestoreEventData EventRestoreData `json:"restoreEventData"`
+}
+
+type EventRestoreData struct {
+	LifecycleRestorationExpiryTime string `json:"lifecycleRestorationExpiryTime"`
+	LifecycleRestoreStorageClass   string `json:"lifecycleRestoreStorageClass"`
+}
+
+type EventS3BucketData struct {
+	Name          string            `json:"name"`
+	OwnerIdentity EventUserIdentity `json:"ownerIdentity"`
+	Arn           string            `json:"arn"`
+}
+
+type EventObjectData struct {
+	Key       string  `json:"key"`
+	Size      int64   `json:"size"`
+	ETag      *string `json:"eTag"`
+	VersionId *string `json:"versionId"`
+	Sequencer string  `json:"sequencer"`
+}
+
+type EventConfig struct {
+	KafkaURL      string
+	KafkaTopic    string
+	KafkaTopicKey string
+	NatsURL       string
+	NatsTopic     string
+}
+
+func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
+	if cfg.KafkaURL != "" && cfg.NatsURL != "" {
+		return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
+	}
+	if cfg.NatsURL != "" {
+		return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
+	}
+	if cfg.KafkaURL != "" {
+		return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
+	}
+	return nil, nil
+}

s3event/kafka.go

@@ -0,0 +1,153 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/segmentio/kafka-go"
+)
+
+var sequencer = 0
+
+type Kafka struct {
+	key    string
+	writer *kafka.Writer
+	mu     sync.Mutex
+}
+
+func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
+	if topic == "" {
+		return nil, fmt.Errorf("kafka message topic should be specified")
+	}
+
+	w := kafka.NewWriter(kafka.WriterConfig{
+		Brokers:      []string{url},
+		Topic:        topic,
+		Balancer:     &kafka.LeastBytes{},
+		BatchTimeout: 5 * time.Millisecond,
+	})
+
+	msg := map[string]string{
+		"Service": "S3",
+		"Event":   "s3:TestEvent",
+		"Time":    time.Now().Format(time.RFC3339),
+		"Bucket":  "Test-Bucket",
+	}
+
+	msgJSON, err := json.Marshal(msg)
+	if err != nil {
+		return nil, err
+	}
+
+	message := kafka.Message{
+		Key:   []byte(key),
+		Value: msgJSON,
+	}
+
+	ctx := context.Background()
+
+	err = w.WriteMessages(ctx, message)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Kafka{
+		key:    key,
+		writer: w,
+	}, nil
+}
+
+func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+
+	schema := EventSchema{
+		EventVersion: "2.2",
+		EventSource:  "aws:s3",
+		AwsRegion:    ctx.Locals("region").(string),
+		EventTime:    time.Now().Format(time.RFC3339),
+		EventName:    meta.EventName,
+		UserIdentity: EventUserIdentity{
+			PrincipalId: ctx.Locals("access").(string),
+		},
+		RequestParameters: EventRequestParams{
+			SourceIPAddress: ctx.IP(),
+		},
+		ResponseElements: EventResponseElements{
+			RequestId: ctx.Get("X-Amz-Request-Id"),
+			HostId:    ctx.Get("X-Amx-Id-2"),
+		},
+		S3: EventS3Data{
+			S3SchemaVersion: "1.0",
+			// This field will come up after implementing per bucket notifications
+			ConfigurationId: "kafka-global",
+			Bucket: EventS3BucketData{
+				Name: bucket,
+				OwnerIdentity: EventUserIdentity{
+					PrincipalId: ctx.Locals("access").(string),
+				},
+				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+			},
+			Object: EventObjectData{
+				Key:       object,
+				Size:      meta.ObjectSize,
+				ETag:      meta.ObjectETag,
+				VersionId: meta.VersionId,
+				Sequencer: genSequencer(),
+			},
+		},
+		GlacierEventData: EventGlacierData{
+			// Not supported
+			RestoreEventData: EventRestoreData{},
+		},
+	}
+
+	ks.send([]EventSchema{schema})
+}
+
+func (ks *Kafka) send(evnt []EventSchema) {
+	msg, err := json.Marshal(evnt)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
+		return
+	}
+
+	message := kafka.Message{
+		Key:   []byte(ks.key),
+		Value: msg,
+	}
+
+	ctx := context.Background()
+	err = ks.writer.WriteMessages(ctx, message)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
+	}
+}
+
+func genSequencer() string {
+	sequencer = sequencer + 1
+	return fmt.Sprintf("%X", sequencer)
+}

s3event/nats.go

@@ -0,0 +1,112 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/nats-io/nats.go"
+)
+
+type NatsEventSender struct {
+	topic  string
+	client *nats.Conn
+	mu     sync.Mutex
+}
+
+func InitNatsEventService(url, topic string) (S3EventSender, error) {
+	if topic == "" {
+		return nil, fmt.Errorf("nats message topic should be specified")
+	}
+
+	client, err := nats.Connect(url)
+	if err != nil {
+		return nil, err
+	}
+
+	return &NatsEventSender{
+		topic:  topic,
+		client: client,
+	}, nil
+}
+
+func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
+	ns.mu.Lock()
+	defer ns.mu.Unlock()
+
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+
+	schema := EventSchema{
+		EventVersion: "2.2",
+		EventSource:  "aws:s3",
+		AwsRegion:    ctx.Locals("region").(string),
+		EventTime:    time.Now().Format(time.RFC3339),
+		EventName:    meta.EventName,
+		UserIdentity: EventUserIdentity{
+			PrincipalId: ctx.Locals("access").(string),
+		},
+		RequestParameters: EventRequestParams{
+			SourceIPAddress: ctx.IP(),
+		},
+		ResponseElements: EventResponseElements{
+			RequestId: ctx.Get("X-Amz-Request-Id"),
+			HostId:    ctx.Get("X-Amx-Id-2"),
+		},
+		S3: EventS3Data{
+			S3SchemaVersion: "1.0",
+			// This field will come up after implementing per bucket notifications
+			ConfigurationId: "nats-global",
+			Bucket: EventS3BucketData{
+				Name: bucket,
+				OwnerIdentity: EventUserIdentity{
+					PrincipalId: ctx.Locals("access").(string),
+				},
+				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+			},
+			Object: EventObjectData{
+				Key:       object,
+				Size:      meta.ObjectSize,
+				ETag:      meta.ObjectETag,
+				VersionId: meta.VersionId,
+				Sequencer: genSequencer(),
+			},
+		},
+		GlacierEventData: EventGlacierData{
+			// Not supported
+			RestoreEventData: EventRestoreData{},
+		},
+	}
+
+	ns.send([]EventSchema{schema})
+}
+
+func (ns *NatsEventSender) send(evnt []EventSchema) {
+	msg, err := json.Marshal(evnt)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
+	}
+
+	err = ns.client.Publish(ns.topic, msg)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
+	}
+}

s3log/audit-logger.go

@@ -0,0 +1,112 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"crypto/tls"
+	"encoding/hex"
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type AuditLogger interface {
+	Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
+	HangUp() error
+	Shutdown() error
+}
+
+type LogMeta struct {
+	BucketOwner string
+	ObjectSize  int64
+	Action      string
+}
+
+type LogConfig struct {
+	LogFile    string
+	WebhookURL string
+}
+
+type LogFields struct {
+	BucketOwner        string
+	Bucket             string
+	Time               time.Time
+	RemoteIP           string
+	Requester          string
+	RequestID          string
+	Operation          string
+	Key                string
+	RequestURI         string
+	HttpStatus         int
+	ErrorCode          string
+	BytesSent          int
+	ObjectSize         int64
+	TotalTime          int64
+	TurnAroundTime     int64
+	Referer            string
+	UserAgent          string
+	VersionID          string
+	HostID             string
+	SignatureVersion   string
+	CipherSuite        string
+	AuthenticationType string
+	HostHeader         string
+	TLSVersion         string
+	AccessPointARN     string
+	AclRequired        string
+}
+
+func InitLogger(cfg *LogConfig) (AuditLogger, error) {
+	if cfg.WebhookURL != "" && cfg.LogFile != "" {
+		return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
+	}
+	if cfg.WebhookURL != "" {
+		return InitWebhookLogger(cfg.WebhookURL)
+	}
+	if cfg.LogFile != "" {
+		return InitFileLogger(cfg.LogFile)
+	}
+
+	return nil, nil
+}
+
+func genID() string {
+	src := rand.New(rand.NewSource(time.Now().UnixNano()))
+	b := make([]byte, 8)
+
+	if _, err := src.Read(b); err != nil {
+		panic(err)
+	}
+
+	return strings.ToUpper(hex.EncodeToString(b))
+}
+
+func getTLSVersionName(version uint16) string {
+	switch version {
+	case tls.VersionTLS10:
+		return "TLSv1.0"
+	case tls.VersionTLS11:
+		return "TLSv1.1"
+	case tls.VersionTLS12:
+		return "TLSv1.2"
+	case tls.VersionTLS13:
+		return "TLSv1.3"
+	default:
+		return ""
+	}
+}

s3log/file.go

@@ -0,0 +1,230 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"crypto/tls"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	logFileMode = 0600
+	timeFormat  = "02/January/2006:15:04:05 -0700"
+)
+
+// FileLogger is a local file audit log
+type FileLogger struct {
+	logfile string
+	f       *os.File
+	gotErr  bool
+	mu      sync.Mutex
+}
+
+var _ AuditLogger = &FileLogger{}
+
+// InitFileLogger initializes audit logs to local file
+func InitFileLogger(logname string) (AuditLogger, error) {
+	f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("open log: %w", err)
+	}
+
+	f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return &FileLogger{logfile: logname, f: f}, nil
+}
+
+// Log sends log message to file logger
+func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if f.gotErr {
+		return
+	}
+
+	lf := LogFields{}
+
+	access := "-"
+	reqURI := ctx.OriginalURL()
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+	errorCode := ""
+	httpStatus := 200
+	startTime := ctx.Locals("startTime").(time.Time)
+	tlsConnState := ctx.Context().TLSConnectionState()
+	if tlsConnState != nil {
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
+	}
+
+	if err != nil {
+		serr, ok := err.(s3err.APIError)
+		if ok {
+			errorCode = serr.Code
+			httpStatus = serr.HTTPStatusCode
+		} else {
+			errorCode = err.Error()
+			httpStatus = 500
+		}
+	}
+
+	switch ctx.Locals("access").(type) {
+	case string:
+		access = ctx.Locals("access").(string)
+	}
+
+	lf.BucketOwner = meta.BucketOwner
+	lf.Bucket = bucket
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.Key = object
+	lf.RequestURI = reqURI
+	lf.HttpStatus = httpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.ObjectSize = meta.ObjectSize
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.VersionID = ctx.Query("versionId")
+	lf.HostID = ctx.Get("X-Amz-Id-2")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
+	lf.AclRequired = "Yes"
+
+	f.writeLog(lf)
+}
+
+func (f *FileLogger) writeLog(lf LogFields) {
+	if lf.BucketOwner == "" {
+		lf.BucketOwner = "-"
+	}
+	if lf.Bucket == "" {
+		lf.Bucket = "-"
+	}
+	if lf.RemoteIP == "" {
+		lf.RemoteIP = "-"
+	}
+	if lf.Requester == "" {
+		lf.Requester = "-"
+	}
+	if lf.Operation == "" {
+		lf.Operation = "-"
+	}
+	if lf.Key == "" {
+		lf.Key = "-"
+	}
+	if lf.RequestURI == "" {
+		lf.RequestURI = "-"
+	}
+	if lf.ErrorCode == "" {
+		lf.ErrorCode = "-"
+	}
+	if lf.Referer == "" {
+		lf.Referer = "-"
+	}
+	if lf.UserAgent == "" {
+		lf.UserAgent = "-"
+	}
+	if lf.VersionID == "" {
+		lf.VersionID = "-"
+	}
+	if lf.HostID == "" {
+		lf.HostID = "-"
+	}
+	if lf.CipherSuite == "" {
+		lf.CipherSuite = "-"
+	}
+	if lf.HostHeader == "" {
+		lf.HostHeader = "-"
+	}
+	if lf.TLSVersion == "" {
+		lf.TLSVersion = "-"
+	}
+
+	log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
+		lf.BucketOwner,
+		lf.Bucket,
+		fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
+		lf.RemoteIP,
+		lf.Requester,
+		lf.RequestID,
+		lf.Operation,
+		lf.Key,
+		lf.RequestURI,
+		lf.HttpStatus,
+		lf.ErrorCode,
+		lf.BytesSent,
+		lf.ObjectSize,
+		lf.TotalTime,
+		lf.TurnAroundTime,
+		lf.Referer,
+		lf.UserAgent,
+		lf.VersionID,
+		lf.HostID,
+		lf.SignatureVersion,
+		lf.CipherSuite,
+		lf.AuthenticationType,
+		lf.HostHeader,
+		lf.TLSVersion,
+		lf.AccessPointARN,
+		lf.AclRequired,
+	)
+
+	_, err := f.f.WriteString(log)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
+		// TODO: do we need to terminate on log error?
+		// set err for now so that we don't spew errors
+		f.gotErr = true
+	}
+}
+
+// HangUp closes current logfile handle and opens a new one
+// typically needed for log rotations
+func (f *FileLogger) HangUp() error {
+	err := f.f.Close()
+	if err != nil {
+		return fmt.Errorf("close log: %w", err)
+	}
+
+	f.f, err = os.OpenFile(f.logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("open log: %w", err)
+	}
+
+	f.f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return nil
+}
+
+// Shutdown closes logfile handle
+func (f *FileLogger) Shutdown() error {
+	return f.f.Close()
+}

s3log/webhook.go

@@ -0,0 +1,156 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3log
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+// WebhookLogger is a webhook URL audit log
+type WebhookLogger struct {
+	mu  sync.Mutex
+	url string
+}
+
+var _ AuditLogger = &WebhookLogger{}
+
+// InitWebhookLogger initializes audit logs to webhook URL
+func InitWebhookLogger(url string) (AuditLogger, error) {
+	client := &http.Client{
+		Timeout: 3 * time.Second,
+	}
+	_, err := client.Post(url, "application/json", nil)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			return nil, fmt.Errorf("unreachable webhook url: %w", err)
+		}
+	}
+	return &WebhookLogger{
+		url: url,
+	}, nil
+}
+
+// Log sends log message to webhook
+func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
+	wl.mu.Lock()
+	defer wl.mu.Unlock()
+
+	lf := LogFields{}
+
+	access := "-"
+	reqURI := ctx.OriginalURL()
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+	errorCode := ""
+	httpStatus := 200
+	startTime := ctx.Locals("startTime").(time.Time)
+	tlsConnState := ctx.Context().TLSConnectionState()
+	if tlsConnState != nil {
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
+	}
+
+	if err != nil {
+		serr, ok := err.(s3err.APIError)
+		if ok {
+			errorCode = serr.Code
+			httpStatus = serr.HTTPStatusCode
+		} else {
+			errorCode = err.Error()
+			httpStatus = 500
+		}
+	}
+
+	switch ctx.Locals("access").(type) {
+	case string:
+		access = ctx.Locals("access").(string)
+	}
+
+	lf.BucketOwner = meta.BucketOwner
+	lf.Bucket = bucket
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.Key = object
+	lf.RequestURI = reqURI
+	lf.HttpStatus = httpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.ObjectSize = meta.ObjectSize
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.VersionID = ctx.Query("versionId")
+	lf.HostID = ctx.Get("X-Amz-Id-2")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
+	lf.AclRequired = "Yes"
+
+	wl.sendLog(lf)
+}
+
+func (wl *WebhookLogger) sendLog(lf LogFields) {
+	jsonLog, err := json.Marshal(lf)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to parse the log data: %v\n", err.Error())
+	}
+
+	req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	go makeRequest(req)
+}
+
+func makeRequest(req *http.Request) {
+	client := &http.Client{
+		Timeout: 1 * time.Second,
+	}
+	_, err := client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			fmt.Fprintf(os.Stderr, "error sending webhook log: %v\n", err)
+		}
+	}
+}
+
+// HangUp does nothing for webhooks
+func (wl *WebhookLogger) HangUp() error {
+	return nil
+}
+
+// Shutdown does nothing for webhooks
+func (wl *WebhookLogger) Shutdown() error {
+	return nil
+}

s3response/s3response.go

@@ -16,6 +16,8 @@ package s3response
 
 import (
 	"encoding/xml"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 // Part describes part metadata.
@@ -27,7 +29,7 @@ type Part struct {
 }
 
 // ListPartsResponse - s3 api list parts response.
-type ListPartsResponse struct {
+type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
 	Bucket   string
@@ -50,7 +52,7 @@ type ListPartsResponse struct {
 }
 
 // ListMultipartUploadsResponse - s3 api list multipart uploads response.
-type ListMultipartUploadsResponse struct {
+type ListMultipartUploadsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
 
 	Bucket             string
@@ -71,7 +73,7 @@ type ListMultipartUploadsResponse struct {
 	CommonPrefixes []CommonPrefix
 }
 
-// Upload desribes in progress multipart upload
+// Upload describes in progress multipart upload
 type Upload struct {
 	Key          string
 	UploadID     string `xml:"UploadId"`
@@ -94,3 +96,46 @@ type Owner struct {
 	ID          string
 	DisplayName string
 }
+
+type Tag struct {
+	Key   string `xml:"Key"`
+	Value string `xml:"Value"`
+}
+
+type TagSet struct {
+	Tags []Tag `xml:"Tag"`
+}
+
+type Tagging struct {
+	TagSet TagSet `xml:"TagSet"`
+}
+
+type DeleteObjects struct {
+	Objects []types.ObjectIdentifier `xml:"Object"`
+}
+
+type DeleteObjectsResult struct {
+	Deleted []types.DeletedObject
+	Error   []types.Error
+}
+type SelectObjectContentPayload struct {
+	Expression          *string
+	ExpressionType      types.ExpressionType
+	RequestProgress     *types.RequestProgress
+	InputSerialization  *types.InputSerialization
+	OutputSerialization *types.OutputSerialization
+	ScanRange           *types.ScanRange
+}
+
+type SelectObjectContentResult struct {
+	Records  *types.RecordsEvent
+	Stats    *types.StatsEvent
+	Progress *types.ProgressEvent
+	Cont     *types.ContinuationEvent
+	End      *types.EndEvent
+}
+
+type Bucket struct {
+	Name  string `json:"name"`
+	Owner string `json:"owner"`
+}

s3select/message-handler.go

@@ -0,0 +1,358 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3select
+
+import (
+	"bufio"
+	"context"
+	"encoding/binary"
+	"encoding/xml"
+	"fmt"
+	"hash/crc32"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// Protocol definition for messages can be found here:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
+
+var (
+	// From ptotocol def:
+	// Enum indicating the header value type.
+	// For Amazon S3 Select, this is always 7.
+	headerValueType = byte(7)
+)
+
+func intToTwoBytes(i int) []byte {
+	return []byte{byte(i >> 8), byte(i)}
+}
+
+func generateHeader(messages ...string) []byte {
+	var header []byte
+
+	for i, message := range messages {
+		if i%2 == 1 {
+			header = append(header, headerValueType)
+			header = append(header, intToTwoBytes(len(message))...)
+		} else {
+			header = append(header, byte(len(message)))
+		}
+		header = append(header, message...)
+	}
+
+	return header
+}
+
+func generateOctetHeader(message string) []byte {
+	return generateHeader(
+		":message-type",
+		"event",
+		":content-type",
+		"application/octet-stream",
+		":event-type",
+		message)
+}
+
+func generateTextHeader(message string) []byte {
+	return generateHeader(
+		":message-type",
+		"event",
+		":content-type",
+		"text/xml",
+		":event-type",
+		message)
+}
+
+func generateNoContentHeader(message string) []byte {
+	return generateHeader(
+		":message-type",
+		"event",
+		":event-type",
+		message)
+}
+
+const (
+	// 4 bytes total byte len +
+	// 4 bytes headers bytes len +
+	// 4 bytes prelude CRC
+	preludeLen = 12
+	// CRC is uint32
+	msgCrcLen = 4
+)
+
+var (
+	recordsHeader       = generateOctetHeader("Records")
+	continuationHeader  = generateNoContentHeader("Cont")
+	continuationMessage = genMessage(continuationHeader, []byte{})
+	progressHeader      = generateTextHeader("Progress")
+	statsHeader         = generateTextHeader("Stats")
+	endHeader           = generateNoContentHeader("End")
+	endMessage          = genMessage(endHeader, []byte{})
+)
+
+func uintToBytes(n uint32) []byte {
+	b := make([]byte, 4)
+	binary.BigEndian.PutUint32(b, n)
+	return b
+}
+
+func generatePrelude(msgLen int, headerLen int) []byte {
+	prelude := make([]byte, 0, preludeLen)
+
+	// 4 bytes total byte len
+	prelude = append(prelude, uintToBytes(uint32(msgLen+headerLen+preludeLen+msgCrcLen))...)
+	// 4 bytes headers bytes len
+	prelude = append(prelude, uintToBytes(uint32(headerLen))...)
+	// 4 bytes prelude CRC
+	prelude = append(prelude, uintToBytes(crc32.ChecksumIEEE(prelude))...)
+
+	return prelude
+}
+
+const (
+	maxHeaderSize  = 1024 * 1024
+	maxMessageSize = 5 * 1024 * 1024 * 1024
+)
+
+func genMessage(header, payload []byte) []byte {
+	var msg []byte
+	// below is always true since the size is validated
+	// in the send record
+	if len(header) <= maxHeaderSize && len(payload) <= maxMessageSize {
+		msglen := preludeLen + len(header) + len(payload) + msgCrcLen
+		msg = make([]byte, 0, msglen)
+	}
+
+	msg = append(msg, generatePrelude(len(payload), len(header))...)
+	msg = append(msg, header...)
+	msg = append(msg, payload...)
+	msg = append(msg, uintToBytes(crc32.ChecksumIEEE(msg))...)
+
+	return msg
+}
+
+func genRecordsMessage(payload []byte) []byte {
+	return genMessage(recordsHeader, payload)
+}
+
+type progress struct {
+	XMLName        xml.Name `xml:"Progress"`
+	BytesScanned   int64    `xml:"BytesScanned"`
+	BytesProcessed int64    `xml:"BytesProcessed"`
+	BytesReturned  int64    `xml:"BytesReturned"`
+}
+
+func genProgressMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
+	progress := progress{
+		BytesScanned:   bytesScanned,
+		BytesProcessed: bytesProcessed,
+		BytesReturned:  bytesReturned,
+	}
+
+	xmlData, _ := xml.MarshalIndent(progress, "", "    ")
+	payload := []byte(xml.Header + string(xmlData))
+	return genMessage(progressHeader, payload)
+}
+
+type stats struct {
+	XMLName        xml.Name `xml:"Stats"`
+	BytesScanned   int64    `xml:"BytesScanned"`
+	BytesProcessed int64    `xml:"BytesProcessed"`
+	BytesReturned  int64    `xml:"BytesReturned"`
+}
+
+func genStatsMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
+	stats := stats{
+		BytesScanned:   bytesScanned,
+		BytesProcessed: bytesProcessed,
+		BytesReturned:  bytesReturned,
+	}
+
+	xmlData, _ := xml.MarshalIndent(stats, "", "    ")
+	payload := []byte(xml.Header + string(xmlData))
+	return genMessage(statsHeader, payload)
+}
+
+func genErrorMessage(errorCode, errorMessage string) []byte {
+	return genMessage(generateHeader(
+		":error-code",
+		errorCode,
+		":error-message",
+		errorMessage,
+		":message-type",
+		"error",
+	), []byte{})
+}
+
+// GetProgress is a callback function that periodically retrieves the current
+// values for the following if not nil.  This is used to send Progress
+// messages back to client.
+// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
+// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
+type GetProgress func() (bytesScanned int64, bytesProcessed int64)
+
+type MessageHandler struct {
+	sync.Mutex
+	ctx           context.Context
+	cancel        context.CancelFunc
+	writer        *bufio.Writer
+	data          chan []byte
+	getProgress   GetProgress
+	stopCh        chan bool
+	resetCh       chan bool
+	bytesReturned int64
+}
+
+// NewMessageHandler creates a new MessageHandler instance and starts the event streaming
+func NewMessageHandler(ctx context.Context, w *bufio.Writer, getProgressFunc GetProgress) *MessageHandler {
+	ctx, cancel := context.WithCancel(ctx)
+
+	mh := &MessageHandler{
+		ctx:         ctx,
+		cancel:      cancel,
+		writer:      w,
+		data:        make(chan []byte),
+		getProgress: getProgressFunc,
+		resetCh:     make(chan bool),
+		stopCh:      make(chan bool),
+	}
+
+	go mh.sendBackgroundMessages(mh.resetCh, mh.stopCh)
+	return mh
+}
+
+func (mh *MessageHandler) write(data []byte) error {
+	mh.Lock()
+	defer mh.Unlock()
+
+	mh.stopCh <- true
+	defer func() { mh.resetCh <- true }()
+
+	_, err := mh.writer.Write(data)
+	if err != nil {
+		return err
+	}
+
+	return mh.writer.Flush()
+}
+
+const (
+	continuationInterval = time.Second
+	progressInterval     = time.Minute
+)
+
+func (mh *MessageHandler) sendBackgroundMessages(resetCh, stopCh <-chan bool) {
+	continuationTicker := time.NewTicker(continuationInterval)
+	defer continuationTicker.Stop()
+
+	var progressTicker *time.Ticker
+	var progressTickerChan <-chan time.Time
+	if mh.getProgress != nil {
+		progressTicker = time.NewTicker(progressInterval)
+		progressTickerChan = progressTicker.C
+		defer progressTicker.Stop()
+	}
+
+Loop:
+	for {
+		select {
+		case <-mh.ctx.Done():
+			break Loop
+
+		case <-continuationTicker.C:
+			err := mh.write(continuationMessage)
+			if err != nil {
+				mh.cancel()
+				break Loop
+			}
+
+		case <-resetCh:
+			continuationTicker.Reset(continuationInterval)
+
+		case <-stopCh:
+			continuationTicker.Stop()
+
+		case <-progressTickerChan:
+			var bytesScanned, bytesProcessed int64
+			if mh.getProgress != nil {
+				bytesScanned, bytesProcessed = mh.getProgress()
+			}
+			bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
+			err := mh.write(genProgressMessage(bytesScanned, bytesProcessed, bytesReturned))
+			if err != nil {
+				mh.cancel()
+				break Loop
+			}
+		}
+	}
+}
+
+// SendRecord sends a single Records message
+func (mh *MessageHandler) SendRecord(payload []byte) error {
+	if mh.ctx.Err() != nil {
+		return mh.ctx.Err()
+	}
+
+	if len(payload) > maxMessageSize {
+		return fmt.Errorf("record max size exceeded")
+	}
+
+	err := mh.write(genRecordsMessage(payload))
+	if err != nil {
+		return err
+	}
+
+	atomic.AddInt64(&mh.bytesReturned, int64(len(payload)))
+	return nil
+}
+
+// Finish terminates message stream with Stats and End message
+// generates stats and end message using function args based on:
+// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
+// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
+func (mh *MessageHandler) Finish(bytesScanned, bytesProcessed int64) error {
+	if mh.ctx.Err() != nil {
+		return mh.ctx.Err()
+	}
+
+	bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
+	err := mh.write(genStatsMessage(bytesScanned, bytesProcessed, bytesReturned))
+	if err != nil {
+		return err
+	}
+
+	err = mh.write(endMessage)
+	if err != nil {
+		return err
+	}
+
+	mh.cancel()
+	return nil
+}
+
+// FinishWithError terminates event stream with error
+func (mh *MessageHandler) FinishWithError(errorCode, errorMessage string) error {
+	if mh.ctx.Err() != nil {
+		return mh.ctx.Err()
+	}
+	err := mh.write(genErrorMessage(errorCode, errorMessage))
+	if err != nil {
+		return err
+	}
+
+	mh.cancel()
+	return nil
+}

tests/.env.default

@@ -0,0 +1,6 @@
+AWS_REGION=us-west-2
+AWS_PROFILE=versity
+VERSITY_EXE=./versitygw
+BACKEND=posix
+LOCAL_FOLDER=/tmp/gw
+AWS_ENDPOINT_URL=http://127.0.0.1:7070

tests/.env.versitygw

@@ -0,0 +1,6 @@
+AWS_REGION=us-east-1
+AWS_PROFILE=versity
+VERSITY_EXE=./versitygw
+BACKEND=posix
+LOCAL_FOLDER=/tmp/gw
+AWS_ENDPOINT_URL=http://127.0.0.1:7070

tests/README.md

@@ -0,0 +1,13 @@
+# Command-Line Tests
+
+Instructions:
+1. Build the `versitygw` binary.
+2. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values above to the profile.
+3. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
+4. In the root repo folder, run with `VERSITYGW_TEST_ENV=<env file> tests/s3_bucket_tests.sh`.
+5. If running/testing the GitHub workflow locally, create a `.secrets` file, and set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` parameters here to the values of your AWS S3 IAM account.
+```
+AWS_ACCESS_KEY_ID=<key_id>
+AWS_SECRET_ACCESS_KEY=<secret_key>
+```
+6. To run the workflow locally, install **act** and run with `act -W .github/workflows/system.yml`.

tests/posix_tests.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bats
+
+source ./tests/tests.sh
+
+# check if object exists both on S3 and locally
+# param:  object path
+# 0 for yes, 1 for no, 2 for error
+object_exists_remote_and_local() {
+  if [ $# -ne 1 ]; then
+    echo "object existence check requires single name parameter"
+    return 2
+  fi
+  object_exists "$1" || local exist_result=$?
+  if [[ $exist_result -eq 2 ]]; then
+    echo "Error checking if object exists"
+    return 2
+  fi
+  if [[ $exist_result -eq 1 ]]; then
+    echo "Error:  object doesn't exist remotely"
+    return 1
+  fi
+  if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then
+    echo "Error:  object doesn't exist locally"
+    return 1
+  fi
+  return 0
+}
+
+# check if object doesn't exist both on S3 and locally
+# param:  object path
+# return 0 for doesn't exist, 1 for still exists, 2 for error
+object_not_exists_remote_and_local() {
+  if [ $# -ne 1 ]; then
+    echo "object non-existence check requires single name parameter"
+    return 2
+  fi
+  object_exists "$1" || local exist_result=$?
+  if [[ $exist_result -eq 2 ]]; then
+    echo "Error checking if object doesn't exist"
+    return 2
+  fi
+  if [[ $exist_result -eq 0 ]]; then
+    echo "Error:  object exists remotely"
+    return 1
+  fi
+  if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then
+    echo "Error:  object exists locally"
+    return 1
+  fi
+  return 0
+}
+
+# check if a bucket doesn't exist both on S3 and on gateway
+# param: bucket name
+# return:  0 for doesn't exist, 1 for does, 2 for error
+bucket_not_exists_remote_and_local() {
+  if [ $# -ne 1 ]; then
+    echo "bucket existence check requires single name parameter"
+    return 2
+  fi
+  bucket_exists "$1" || local exist_result=$?
+  if [[ $exist_result -eq 2 ]]; then
+    echo "Error checking if bucket exists"
+    return 2
+  fi
+  if [[ $exist_result -eq 0 ]]; then
+    echo "Error:  bucket exists remotely"
+    return 1
+  fi
+  if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then
+    echo "Error:  bucket exists locally"
+    return 1
+  fi
+  return 0
+}
+
+# check if a bucket exists both on S3 and on gateway
+# param: bucket name
+# return:  0 for yes, 1 for no, 2 for error
+bucket_exists_remote_and_local() {
+  if [ $# -ne 1 ]; then
+    echo "bucket existence check requires single name parameter"
+    return 2
+  fi
+  bucket_exists "$1" || local exist_result=$?
+  if [[ $exist_result -eq 2 ]]; then
+    echo "Error checking if bucket exists"
+    return 2
+  fi
+  if [[ $exist_result -eq 1 ]]; then
+    echo "Error:  bucket doesn't exist remotely"
+    return 1
+  fi
+  if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then
+    echo "Error:  bucket doesn't exist locally"
+    return 1
+  fi
+  return 0
+}
+
+# test that changes to local folders and files are reflected on S3
+@test test_local_creation_deletion {
+
+  local bucket_name="versity-gwtest-put-object-test"
+  local object_name="test-object"
+
+  bucket_exists_remote_and_local $bucket_name || local bucket_exists=$?
+  if [[ $bucket_exists -eq 2 ]]; then
+    fail "Bucket existence check error"
+  fi
+  local object="$bucket_name"/"$object_name"
+  if [[ $bucket_exists -eq 0 ]]; then
+    object_exists_remote_and_local "$object" || local object_exists=$?
+    if [[ $object_exists -eq 2 ]]; then
+      fail "Object existence check error"
+    fi
+    if [[ $object_exists -eq 0 ]]; then
+      delete_object "$object" || local delete_object=$?
+      [[ $delete_object -eq 0 ]] || fail "Failed to delete object"
+    fi
+    delete_bucket $bucket_name || local delete_bucket=$?
+    [[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
+  fi
+  mkdir "$LOCAL_FOLDER"/$bucket_name
+  touch "$LOCAL_FOLDER"/$object
+  bucket_exists_remote_and_local $bucket_name || local bucket_exists_two=$?
+  [[ $bucket_exists_two -eq 0 ]] || fail "Failed bucket existence check"
+  object_exists_remote_and_local $object || local object_exists_two=$?
+  [[ $object_exists_two -eq 0 ]] || fail "Failed object existence check"
+  rm "$LOCAL_FOLDER"/$object
+  sleep 1
+  object_not_exists_remote_and_local $object || local object_deleted=$?
+  [[ $object_deleted -eq 0 ]] || fail "Failed object deletion check"
+  rmdir "$LOCAL_FOLDER"/$bucket_name
+  sleep 1
+  bucket_not_exists_remote_and_local $bucket_name || local bucket_deleted=$?
+  [[ $bucket_deleted -eq 0 ]] || fail "Failed bucket deletion check"
+}
+

tests/s3_bucket_tests.sh

@@ -0,0 +1,353 @@
+#!/usr/bin/env bats
+
+source ./tests/tests.sh
+
+# create an AWS bucket
+# param:  bucket name
+# return 0 for success, 1 for failure
+create_bucket() {
+  if [ $# -ne 1 ]; then
+    echo "create bucket missing bucket name"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "error creating bucket: $error"
+    return 1
+  fi
+  return 0
+}
+
+# delete an AWS bucket
+# param:  bucket name
+# return 0 for success, 1 for failure
+delete_bucket() {
+  if [ $# -ne 1 ]; then
+    echo "delete bucket missing bucket name"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?"
+  if [ $exit_code -ne 0 ]; then
+    if [[ "$error" == *"The specified bucket does not exist"* ]]; then
+      return 0
+    else
+      echo "error deleting bucket: $error"
+      return 1
+    fi
+  fi
+  return 0
+}
+
+# check if bucket exists
+# param:  bucket name
+# return 0 for true, 1 for false, 2 for error
+bucket_exists() {
+  if [ $# -ne 1 ]; then
+    echo "bucket exists check missing bucket name"
+    return 2
+  fi
+  local exit_code=0
+  local error
+  error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
+  echo "Exit code: $exit_code, error: $error"
+  if [ $exit_code -ne 0 ]; then
+    if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
+      return 1
+    else
+      echo "error checking if bucket exists: $error"
+      return 2
+    fi
+  fi
+  return 0
+}
+
+# create bucket if it doesn't exist
+# param:  bucket name
+# return 0 for success, 1 for failure
+check_and_create_bucket() {
+  if [ $# -ne 1 ]; then
+    echo "bucket creation function requires bucket name"
+    return 1
+  fi
+  local exists_result
+  bucket_exists "$1" || exists_result=$?
+  if [[ $exists_result -eq 2 ]]; then
+    echo "Bucket existence check error"
+    return 1
+  fi
+  local create_result
+  if [[ $exists_result -eq 1 ]]; then
+    create_bucket "$1" || create_result=$?
+    if [[ $create_result -ne 0 ]]; then
+      echo "Error creating bucket"
+      return 1
+    fi
+  fi
+  return 0
+}
+
+# check if object exists on S3 via gateway
+# param:  object path
+# return 0 for true, 1 for false, 2 for error
+object_exists() {
+  if [ $# -ne 1 ]; then
+    echo "object exists check missing object name"
+    return 2
+  fi
+  local exit_code=0
+  local error
+  error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
+  if [ $exit_code -ne 0 ]; then
+    if [[ "$error" == "" ]]; then
+      return 1
+    else
+      echo "error checking if object exists: $error"
+      return 2
+    fi
+  fi
+  return 0
+}
+
+# add object to versitygw
+# params:  source file, destination copy location
+# return 0 for success, 1 for failure
+put_object() {
+  if [ $# -ne 2 ]; then
+    echo "put object command requires source, destination"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  error=$(aws s3 cp "$1" s3://"$2" 2>&1) || exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "error copying object to bucket: $error"
+    return 1
+  fi
+  return 0
+}
+
+# add object to versitygw if it doesn't exist
+# params:  source file, destination copy location
+# return 0 for success or already exists, 1 for failure
+check_and_put_object() {
+  if [ $# -ne 2 ]; then
+    echo "check and put object function requires source, destination"
+    return 1
+  fi
+  object_exists "$2" || local exists_result=$?
+  if [ $exists_result -eq 2 ]; then
+    echo "error checking if object exists"
+    return 1
+  fi
+  if [ $exists_result -eq 1 ]; then
+    put_object "$1" "$2" || local put_result=$?
+    if [ $put_result -ne 0 ]; then
+      echo "error adding object"
+      return 1
+    fi
+  fi
+  return 0
+}
+
+# delete object from versitygw
+# param:  object location
+# return 0 for success, 1 for failure
+delete_object() {
+  if [ $# -ne 1 ]; then
+    echo "delete object command requires object parameter"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  error=$(aws s3 rm s3://"$1" 2>&1) || exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "error deleting object: $error"
+    return 1
+  fi
+  return 0
+}
+
+# list buckets on versitygw
+# no params
+# export bucket_array (bucket names) on success, return 1 for failure
+list_buckets() {
+  local exit_code=0
+  local output
+  output=$(aws s3 ls 2>&1) || exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "error listing buckets: $output"
+    return 1
+  fi
+
+  bucket_array=()
+  while IFS= read -r line; do
+    bucket_name=$(echo "$line" | awk '{print $NF}')
+    bucket_array+=("$bucket_name")
+  done <<< "$output"
+
+  export bucket_array
+}
+
+# list objects on versitygw, in bucket or folder
+# param:  path of bucket or folder
+# export object_array (object names) on success, return 1 for failure
+list_objects() {
+  if [ $# -ne 1 ]; then
+    echo "list objects command requires bucket or folder"
+    return 1
+  fi
+  local exit_code=0
+  local output
+  output=$(aws s3 ls s3://"$1" 2>&1) || exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "error listing objects: $output"
+    return 1
+  fi
+
+  object_array=()
+  while IFS= read -r line; do
+    object_name=$(echo "$line" | awk '{print $NF}')
+    object_array+=("$object_name")
+  done <<< "$output"
+
+  export object_array
+}
+
+# test creation and deletion of bucket on versitygw
+@test "create_delete_bucket_test" {
+
+  local bucket_name="versity-gwtest-create-delete-bucket-test"
+
+  bucket_exists $bucket_name || local exists=$?
+  if [[ $exists -eq 2 ]]; then
+    fail "Bucket existence check error"
+  fi
+  if [[ $exists -eq 0 ]]; then
+    delete_bucket $bucket_name || local delete_result=$?
+    [[ $delete_result -eq 0 ]] || fail "Failed to delete bucket"
+    bucket_exists $bucket_name || local exists_two=$?
+    [[ $exists_two -eq 1 ]] || fail "Failed bucket deletion"
+  fi
+  create_bucket $bucket_name || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Failed to create bucket"
+  bucket_exists $bucket_name || local exists_three=$?
+  [[ $exists_three -eq 0 ]] || fail "Failed bucket existence check"
+  delete_bucket $bucket_name || local delete_result_two=$?
+  [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
+}
+
+# test adding and removing an object on versitygw
+@test "put_object_test" {
+
+  local bucket_name="versity-gwtest-put-object-test"
+  local object_name="test-object"
+
+  bucket_exists $bucket_name || local bucket_exists=$?
+  if [[ $bucket_exists -eq 2 ]]; then
+    fail "Bucket existence check error"
+  fi
+  local object="$bucket_name"/"$object_name"
+  if [[ $bucket_exists -eq 0 ]]; then
+    object_exists "$object" || local object_exists=$?
+    if [[ $object_exists -eq 2 ]]; then
+      fail "Object existence check error"
+    fi
+    if [[ $object_exists -eq 0 ]]; then
+      delete_object "$object" || local delete_object=$?
+      [[ $delete_object -eq 0 ]] || fail "Failed to delete object"
+    fi
+    delete_bucket $bucket_name || local delete_bucket=$?
+    [[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
+  fi
+  touch "$object_name"
+  create_bucket $bucket_name || local create_bucket=$?
+  [[ $create_bucket -eq 0 ]] || fail "Failed to create bucket"
+  put_object "$object_name" "$object" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  object_exists "$object" || local object_exists_two=$?
+  [[ $object_exists_two -eq 0 ]] || fail "Object not added to bucket"
+  delete_object "$object" || local delete_object_two=$?
+  [[ $delete_object_two -eq 0 ]] || fail "Failed to delete object"
+  delete_bucket $bucket_name || local delete_bucket=$?
+  [[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
+  rm "$object_name"
+}
+
+# test listing buckets on versitygw
+@test "test_list_buckets" {
+
+  bucket_name_one="versity-gwtest-list-one"
+  bucket_name_two="versity-gwtest-list-two"
+
+  bucket_exists $bucket_name_one || local exists=$?
+  if [[ $exists -eq 2 ]]; then
+    fail "Bucket existence check error"
+  fi
+  if [[ $exists -eq 1 ]]; then
+    create_bucket $bucket_name_one || local bucket_create_one=$?
+    [[ $bucket_create_one -eq 0 ]] || fail "Failed to create bucket"
+  fi
+  bucket_exists $bucket_name_two || local exists_two=$?
+  if [[ $exists_two -eq 2 ]]; then
+    fail "Bucket existence check error"
+  fi
+  if [[ $exists_two -eq 1 ]]; then
+    create_bucket $bucket_name_two || local bucket_create_two=$?
+    [[ $bucket_create_two -eq 0 ]] || fail "Failed to create bucket"
+  fi
+  list_buckets
+  local bucket_one_found=false
+  local bucket_two_found=false
+  for bucket in "${bucket_array[@]}"; do
+    if [ "$bucket" == $bucket_name_one ]; then
+      bucket_one_found=true
+    elif [ "$bucket" == $bucket_name_two ]; then
+      bucket_two_found=true
+    fi
+    if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then
+      return
+    fi
+  done
+  fail "$bucket_name_one and/or $bucket_name_two not listed (all buckets: ${bucket_array[*]})"
+  delete_bucket $bucket_name_one || local deleted_one=$?
+  [[ $deleted_one -eq 0 ]] || fail "Failed to delete bucket one"
+  delete_bucket $bucket_name_two || local deleted_two=$?
+  [[ $deleted_two -eq 0 ]] || fail "Failed to delete bucket one"
+}
+
+# test listing a bucket's objects on versitygw
+@test test_list_objects {
+
+  bucket_name="versity-gwtest-list-object"
+  object_one="test-file-one"
+  object_two="test-file-two"
+
+  touch $object_one $object_two
+  check_and_create_bucket $bucket_name || local result_one=$?
+  [[ result_one -eq 0 ]] || fail "Error creating bucket"
+  put_object $object_one "$bucket_name"/"$object_one"  || local result_two=$?
+  [[ result_two -eq 0 ]] || fail "Error adding object one"
+  put_object $object_two "$bucket_name"/"$object_two" || local result_three=$?
+  [[ result_three -eq 0 ]] || fail "Error adding object two"
+  list_objects $bucket_name
+  local object_one_found=false
+  local object_two_found=false
+  for object in "${object_array[@]}"; do
+    if [ "$object" == $object_one ]; then
+      object_one_found=true
+    elif [ "$object" == $object_two ]; then
+      object_two_found=true
+    fi
+  done
+  if [ $object_one_found != true ] || [ $object_two_found != true ]; then
+    fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
+  fi
+  delete_object "$bucket_name"/"$object_one"
+  delete_object "$bucket_name"/"$object_two"
+  delete_bucket $bucket_name
+  rm $object_one $object_two
+}

tests/tests.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bats
+
+setup() {
+
+  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r .secrets ]; then
+    source .secrets
+  else
+    echo "Warning: no secrets file found"
+  fi
+  if [ -z "$VERSITYGW_TEST_ENV" ]; then
+    if [ -r tests/.env ]; then
+      source tests/.env
+    else
+      echo "Warning: no .env file found in tests folder"
+    fi
+  else
+    echo "$VERSITYGW_TEST_ENV"
+    source $VERSITYGW_TEST_ENV
+  fi
+
+  if [ -z "$AWS_ACCESS_KEY_ID" ]; then
+    echo "No AWS access key set"
+    return 1
+  elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+    echo "No AWS secret access key set"
+    return 1
+  elif [ -z "$VERSITY_EXE" ]; then
+    echo "No versity executable location set"
+    return 1
+  elif [ -z "$BACKEND" ]; then
+    echo "No backend parameter set (options: 'posix')"
+    return 1
+  elif [ -z "$AWS_REGION" ]; then
+    echo "No AWS region set"
+    return 1
+  elif [ -z "$AWS_PROFILE" ]; then
+    echo "No AWS profile set"
+    return 1
+  elif [ -z "$LOCAL_FOLDER" ]; then
+    echo "No local storage folder set"
+    return 1
+  elif [ -z "$AWS_ENDPOINT_URL" ]; then
+    echo "No AWS endpoint URL set"
+    return 1
+  fi
+
+  ROOT_ACCESS_KEY="$AWS_ACCESS_KEY_ID" ROOT_SECRET_KEY="$AWS_SECRET_ACCESS_KEY" "$VERSITY_EXE" "$BACKEND" "$LOCAL_FOLDER" &
+
+  export AWS_REGION
+  export AWS_PROFILE
+  export AWS_ENDPOINT_URL
+  export LOCAL_FOLDER
+
+  versitygw_pid=$!
+  export versitygw_pid
+}
+
+fail() {
+  echo "$1"
+  return 1
+}
+
+teardown() {
+  if [ -n "$versitygw_pid" ]; then
+    if ps -p "$versitygw_pid" > /dev/null; then
+      kill "$versitygw_pid"
+      wait "$versitygw_pid" || true
+    else
+      echo "Process with PID $versitygw_pid does not exist."
+    fi
+  else
+    echo "versitygw_pid is not set or empty."
+  fi
+}