Merge pull request #78 from versity/ben/cleanup_base

Ben/cleanup base

.github/workflows/go.yml

@@ -0,0 +1,38 @@
+name: general
+on: pull_request
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: "1.20"
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+
+    - name: Verify all files pass gofmt formatting
+      run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Build
+      run: go build -o versitygw cmd/versitygw/*.go
+
+    - name: Test
+      run: go test -v -timeout 30s -tags=github ./...
+
+    - name: Install govulncheck
+      run: go install golang.org/x/vuln/cmd/govulncheck@latest
+      shell: bash
+
+    - name: Run govulncheck
+      run: govulncheck ./...
+      shell: bash

.github/workflows/static.yml

@@ -0,0 +1,23 @@
+name: staticcheck
+on: pull_request
+jobs: 
+  build: 
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.20"
+        id: go
+      - 
+        name: "Set up repo"
+        uses: actions/checkout@v1
+        with: 
+          fetch-depth: 1
+      - 
+        name: "staticcheck"
+        uses: dominikh/staticcheck-action@v1.3.0
+        with: 
+          install-go: false

.gitignore

@@ -7,6 +7,8 @@
 *.dll
 *.so
 *.dylib
+cmd/versitygw/versitygw
+/versitygw
 
 # Test binary, built with `go test -c`
 *.test
@@ -19,3 +21,14 @@
 
 # Go workspace file
 go.work
+
+# ignore IntelliJ directories
+.idea
+
+# auto generated VERSION file
+VERSION
+
+# build output
+/versitygw.spec
+*.tar
+*.tar.gz

Makefile

@@ -0,0 +1,73 @@
+# Copyright 2023 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Go parameters
+GOCMD=go
+GOBUILD=$(GOCMD) build
+GOCLEAN=$(GOCMD) clean
+GOTEST=$(GOCMD) test
+
+BIN=versitygw
+
+VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
+BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
+TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
+
+LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
+
+all: build
+
+build: $(BIN)
+
+.PHONY: $(BIN)
+$(BIN):
+	$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
+
+.PHONY: test
+test: 
+	$(GOTEST) ./...
+
+.PHONY: check
+check:
+# note this requires staticcheck be in your PATH:
+# export PATH=$PATH:~/go/bin
+# go install honnef.co/go/tools/cmd/staticcheck@latest
+	staticcheck ./...
+	golint ./...
+	gofmt -s -l .
+
+.PHONY: clean
+clean: 
+	$(GOCLEAN)
+
+.PHONY: cleanall
+cleanall: clean
+	rm -f $(BIN)
+	rm -f versitygw-*.tar
+	rm -f versitygw-*.tar.gz
+	rm -f versitygw.spec
+
+%.spec: %.spec.in
+	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
+	mv $@+ $@
+
+TARFILE = $(BIN)-$(VERSION).tar
+
+dist: $(BIN).spec
+	echo $(VERSION) >VERSION
+	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
+	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
+	rm -f VERSION
+	rm -f $(BIN).spec
+	gzip -f $(TARFILE)

NOTICE

@@ -0,0 +1,2 @@
+versitygw - Versity S3 Gateway
+Copyright 2023 Versity Software

README.md

@@ -0,0 +1,64 @@
+# The Versity Gateway: A High-Performance Open Source S3 to File Translation Tool
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
+  <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
+</picture>
+
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
+
+The Versity Gateway: A High-Performance Open Source S3 to File Translation Tool
+
+Current status: Alpha, in development not yet suited for production use
+
+See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
+
+Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and file-based storage systems. The Versity Gateway bridges the gap between S3-reliant applications and file storage systems, enabling enhanced compatibility and integration with file based systems while offering exceptional scalability.
+
+The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
+
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
+
+The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
+
+The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework.  This framework is actively developed with a focus on performance.  S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3. 
+
+## Getting Started
+See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
+
+### Run the gateway with posix backend:
+
+```
+mkdir /tmp/vgw
+ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
+```
+This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
+
+To get the usage output, run the following:
+
+```
+./versitygw --help
+```
+
+The command format is
+
+```
+versitygw [global options] command [command options] [arguments...]
+```
+The global options are specified before the backend type and the backend options are specified after.
+
+***
+
+#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
+
+### Contact 
+info@versity.com <br />
++1 844 726 8826
+
+### @versitysoftware 
+[![linkedin](https://github.com/versity/versitygw/blob/assets/assets/linkedin.jpg)](https://www.linkedin.com/company/versity/) &nbsp; 
+[![twitter](https://github.com/versity/versitygw/blob/assets/assets/twitter.jpg)](https://twitter.com/VersitySoftware) &nbsp;
+[![facebook](https://github.com/versity/versitygw/blob/assets/assets/facebook.jpg)](https://www.facebook.com/versitysoftware) &nbsp;
+[![instagram](https://github.com/versity/versitygw/blob/assets/assets/instagram.jpg)](https://www.instagram.com/versitysoftware/) &nbsp;
+

backend/auth/iam.go

@@ -0,0 +1,137 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+type Account struct {
+	Secret string `json:"secret"`
+	Role   string `json:"role"`
+	Region string `json:"region"`
+}
+
+type IAMConfig struct {
+	AccessAccounts map[string]Account `json:"accessAccounts"`
+}
+
+type AccountsCache struct {
+	mu       sync.Mutex
+	Accounts map[string]Account
+}
+
+func (c *AccountsCache) getAccount(access string) *Account {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	acc, ok := c.Accounts[access]
+	if !ok {
+		return nil
+	}
+
+	return &acc
+}
+
+func (c *AccountsCache) updateAccounts() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	var data IAMConfig
+
+	file, err := os.ReadFile("users.json")
+	if err != nil {
+		return fmt.Errorf("error reading config file: %w", err)
+	}
+
+	if err := json.Unmarshal(file, &data); err != nil {
+		return fmt.Errorf("error parsing the data: %w", err)
+	}
+
+	c.Accounts = data.AccessAccounts
+
+	return nil
+}
+
+type IAMService interface {
+	GetIAMConfig() (*IAMConfig, error)
+	CreateAccount(access string, account *Account) error
+	GetUserAccount(access string) *Account
+}
+
+type IAMServiceUnsupported struct {
+	accCache *AccountsCache
+}
+
+var _ IAMService = &IAMServiceUnsupported{}
+
+func New() IAMService {
+	return &IAMServiceUnsupported{accCache: &AccountsCache{Accounts: map[string]Account{}}}
+}
+
+func (IAMServiceUnsupported) GetIAMConfig() (*IAMConfig, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (s IAMServiceUnsupported) CreateAccount(access string, account *Account) error {
+	var data IAMConfig
+
+	file, err := os.ReadFile("users.json")
+	if err != nil {
+		data = IAMConfig{AccessAccounts: map[string]Account{
+			access: *account,
+		}}
+	} else {
+		if err := json.Unmarshal(file, &data); err != nil {
+			return err
+		}
+
+		_, ok := data.AccessAccounts[access]
+		if ok {
+			return fmt.Errorf("user with the given access already exists")
+		}
+
+		data.AccessAccounts[access] = *account
+	}
+
+	updatedJSON, err := json.MarshalIndent(data, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	if err := os.WriteFile("users.json", updatedJSON, 0644); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s IAMServiceUnsupported) GetUserAccount(access string) *Account {
+	acc := s.accCache.getAccount(access)
+	if acc == nil {
+		err := s.accCache.updateAccounts()
+		if err != nil {
+			return nil
+		}
+
+		return s.accCache.getAccount(access)
+	}
+
+	return acc
+}

backend/backend.go

@@ -0,0 +1,171 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+//go:generate moq -out backend_moq_test.go . Backend
+//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
+type Backend interface {
+	fmt.Stringer
+	Shutdown()
+
+	ListBuckets() (*s3.ListBucketsOutput, error)
+	HeadBucket(bucket string) (*s3.HeadBucketOutput, error)
+	GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, error)
+	PutBucket(bucket string) error
+	PutBucketAcl(*s3.PutBucketAclInput) error
+	DeleteBucket(bucket string) error
+
+	CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error)
+	AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
+	ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
+	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error)
+	CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error)
+	PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error)
+
+	PutObject(*s3.PutObjectInput) (string, error)
+	HeadObject(bucket, object string) (*s3.HeadObjectOutput, error)
+	GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
+	GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
+	CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error)
+	ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
+	ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error)
+	DeleteObject(bucket, object string) error
+	DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error
+	PutObjectAcl(*s3.PutObjectAclInput) error
+	RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error
+	UploadPart(bucket, object, uploadId string, Body io.ReadSeeker) (*s3.UploadPartOutput, error)
+	UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error)
+
+	GetTags(bucket, object string) (map[string]string, error)
+	SetTags(bucket, object string, tags map[string]string) error
+	RemoveTags(bucket, object string) error
+}
+
+type BackendUnsupported struct{}
+
+var _ Backend = &BackendUnsupported{}
+
+func New() Backend {
+	return &BackendUnsupported{}
+}
+func (BackendUnsupported) Shutdown() {}
+func (BackendUnsupported) String() string {
+	return "Unsupported"
+}
+func (BackendUnsupported) ListBuckets() (*s3.ListBucketsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketAcl(*s3.PutBucketAclInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPart(bucket, object, uploadId string, Body io.ReadSeeker) (*s3.UploadPartOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucket(bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucket(bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) CreateMultipartUpload(input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
+	return s3response.ListMultipartUploadsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
+	return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) PutObject(*s3.PutObjectInput) (string, error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(bucket, object string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) GetTags(bucket, object string) (map[string]string, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) SetTags(bucket, object string, tags map[string]string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) RemoveTags(bucket, object string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}

backend/backend_test.go

@@ -0,0 +1,218 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+)
+
+func TestBackend_ListBuckets(t *testing.T) {
+	type args struct {
+		ctx context.Context
+	}
+	type test struct {
+		name    string
+		c       Backend
+		args    args
+		wantErr bool
+	}
+	var tests []test
+	tests = append(tests, test{
+		name: "list-Bucket",
+		c: &BackendMock{
+			ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
+				return &s3.ListBucketsOutput{
+					Buckets: []types.Bucket{
+						{
+							Name: aws.String("t1"),
+						},
+					},
+				}, s3err.GetAPIError(0)
+			},
+		},
+		args: args{
+			ctx: context.Background(),
+		},
+		wantErr: false,
+	}, test{
+		name: "list-Bucket-error",
+		c: &BackendMock{
+			ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			},
+		},
+		args: args{
+			ctx: context.Background(),
+		},
+		wantErr: true,
+	})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if _, err := tt.c.ListBuckets(); (err.(s3err.APIError).Code != "") != tt.wantErr {
+				t.Errorf("Backend.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+func TestBackend_HeadBucket(t *testing.T) {
+	type args struct {
+		ctx        context.Context
+		BucketName string
+	}
+	type test struct {
+		name    string
+		c       Backend
+		args    args
+		wantErr bool
+	}
+	var tests []test
+	tests = append(tests, test{
+		name: "head-buckets-error",
+		c: &BackendMock{
+			HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			},
+		},
+		args: args{
+			ctx:        context.Background(),
+			BucketName: "b1",
+		},
+		wantErr: true,
+	})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if _, err := tt.c.HeadBucket(tt.args.BucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
+				t.Errorf("Backend.HeadBucket() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+func TestBackend_GetBucketAcl(t *testing.T) {
+	type args struct {
+		ctx        context.Context
+		bucketName string
+	}
+	type test struct {
+		name    string
+		c       Backend
+		args    args
+		wantErr bool
+	}
+	var tests []test
+	tests = append(tests, test{
+		name: "get bucket acl error",
+		c: &BackendMock{
+			GetBucketAclFunc: func(bucket string) (*s3.GetBucketAclOutput, error) {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			},
+		},
+		args: args{
+			ctx:        context.Background(),
+			bucketName: "b1",
+		},
+		wantErr: true,
+	})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if _, err := tt.c.GetBucketAcl(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
+				t.Errorf("Backend.GetBucketAcl() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+func TestBackend_PutBucket(t *testing.T) {
+	type args struct {
+		ctx        context.Context
+		bucketName string
+	}
+	type test struct {
+		name    string
+		c       Backend
+		args    args
+		wantErr bool
+	}
+	var tests []test
+	tests = append(tests, test{
+		name: "put bucket ",
+		c: &BackendMock{
+			PutBucketFunc: func(bucket string) error {
+				return s3err.GetAPIError(0)
+			},
+		},
+		args: args{
+			ctx:        context.Background(),
+			bucketName: "b1",
+		},
+		wantErr: false,
+	}, test{
+		name: "put bucket error",
+		c: &BackendMock{
+			PutBucketFunc: func(bucket string) error {
+				return s3err.GetAPIError(s3err.ErrNotImplemented)
+			},
+		},
+		args: args{
+			ctx:        context.Background(),
+			bucketName: "b2",
+		},
+		wantErr: true,
+	})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.c.PutBucket(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
+				t.Errorf("Backend.PutBucket() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+func TestBackend_DeleteBucket(t *testing.T) {
+	type args struct {
+		ctx        context.Context
+		bucketName string
+	}
+	type test struct {
+		name    string
+		c       Backend
+		args    args
+		wantErr bool
+	}
+	var tests []test
+	tests = append(tests, test{
+		name: "Delete Bucket Error",
+		c: &BackendMock{
+			DeleteBucketFunc: func(bucket string) error {
+				return s3err.GetAPIError(s3err.ErrNotImplemented)
+			},
+		},
+		args: args{
+			ctx:        context.Background(),
+			bucketName: "b1",
+		},
+		wantErr: true,
+	})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.c.DeleteBucket(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
+				t.Errorf("Backend.DeleteBucket() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

backend/common.go

@@ -0,0 +1,85 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"errors"
+	"io/fs"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+var (
+	// RFC3339TimeFormat RFC3339 time format
+	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
+)
+
+func IsValidBucketName(name string) bool { return true }
+
+type ByBucketName []types.Bucket
+
+func (d ByBucketName) Len() int           { return len(d) }
+func (d ByBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d ByBucketName) Less(i, j int) bool { return *d[i].Name < *d[j].Name }
+
+type ByObjectName []types.Object
+
+func (d ByObjectName) Len() int           { return len(d) }
+func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
+
+func GetStringPtr(s string) *string {
+	return &s
+}
+
+func GetTimePtr(t time.Time) *time.Time {
+	return &t
+}
+
+func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
+	if acceptRange == "" {
+		return 0, file.Size(), nil
+	}
+
+	rangeKv := strings.Split(acceptRange, "=")
+
+	if len(rangeKv) < 2 {
+		return 0, 0, errors.New("invalid range parameter")
+	}
+
+	bRange := strings.Split(rangeKv[1], "-")
+	if len(bRange) < 2 {
+		return 0, 0, errors.New("invalid range parameter")
+	}
+
+	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
+	if err != nil {
+		return 0, 0, errors.New("invalid range parameter")
+	}
+
+	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	if err != nil {
+		return 0, 0, errors.New("invalid range parameter")
+	}
+
+	if endOffset < startOffset {
+		return 0, 0, errors.New("invalid range parameter")
+	}
+
+	return int64(startOffset), int64(endOffset - startOffset + 1), nil
+}

backend/posix/posix_darwin.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/posix/posix_linux.go

@@ -0,0 +1,163 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+const procfddir = "/proc/self/fd"
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
+	// This can help reduce contention within the namespace (parent directories),
+	// etc. And will auto cleanup the inode on close if we never link this
+	// file descriptor into the namespace.
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	if err != nil {
+		// O_TMPFILE not supported, try fallback
+		err := os.MkdirAll(dir, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("make temp dir: %w", err)
+		}
+		f, err := os.CreateTemp(dir,
+			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+		if err != nil {
+			return nil, err
+		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		// falloc is best effort, its fine if this fails
+		if size > 0 {
+			tmp.falloc()
+		}
+		return tmp, nil
+	}
+
+	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
+	// later to link file into namespace
+	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
+
+	tmp := &tmpfile{f: f, isOTmp: true, size: size}
+	// falloc is best effort, its fine if this fails
+	if size > 0 {
+		tmp.falloc()
+	}
+	return tmp, nil
+}
+
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
+func (tmp *tmpfile) link() error {
+	// We use Linkat/Rename as the atomic operation for object puts. The
+	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
+	// with any other simultaneous uploads. The final operation is to move the
+	// temp file into place for the object. This ensures the object semantics
+	// of last upload completed wins and is not some combination of writes
+	// from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
+	}
+
+	procdir, err := os.Open(procfddir)
+	if err != nil {
+		return fmt.Errorf("open proc dir: %w", err)
+	}
+	defer procdir.Close()
+
+	dir, err := os.Open(filepath.Dir(objPath))
+	if err != nil {
+		return fmt.Errorf("open parent dir: %w", err)
+	}
+	defer dir.Close()
+
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if err != nil {
+		return fmt.Errorf("link tmpfile: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/posix/posix_windows.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length")
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/scoutfs/scoutfs.go

@@ -0,0 +1,26 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package scoutfs
+
+import (
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+type ScoutFS struct {
+	*posix.Posix
+}
+
+var _ backend.Backend = ScoutFS{}

backend/walk.go

@@ -0,0 +1,242 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+type WalkResults struct {
+	CommonPrefixes []types.CommonPrefix
+	Objects        []types.Object
+	Truncated      bool
+	NextMarker     string
+}
+
+type DirObjCheck func(path string) (bool, error)
+type GetETag func(path string) (string, error)
+
+// Walk walks the supplied fs.FS and returns results compatible with list
+// objects responses
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk DirObjCheck, getetag GetETag, skipdirs []string) (WalkResults, error) {
+	cpmap := make(map[string]struct{})
+	var objects []types.Object
+
+	var pastMarker bool
+	if marker == "" {
+		pastMarker = true
+	}
+
+	var pastMax bool
+	var newMarker string
+	var truncated bool
+
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if pastMax {
+			newMarker = path
+			truncated = true
+			return fs.SkipAll
+		}
+
+		if d.IsDir() {
+			// Ignore the root directory
+			if path == "." {
+				return nil
+			}
+
+			if contains(d.Name(), skipdirs) {
+				return fs.SkipDir
+			}
+
+			// If prefix is defined and the directory does not match prefix,
+			// do not descend into the directory because nothing will
+			// match this prefix. Make sure to append the / at the end of
+			// directories since this is implied as a directory path name.
+			// If path is a prefix of prefix, then path could still be
+			// building to match. So only skip if path isnt a prefix of prefix
+			// and prefix isnt a prefix of path.
+			if prefix != "" &&
+				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
+				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
+				return fs.SkipDir
+			}
+
+			// TODO: can we do better here rather than a second readdir
+			// per directory?
+			ents, err := fs.ReadDir(fileSystem, path)
+			if err != nil {
+				return fmt.Errorf("readdir %q: %w", path, err)
+			}
+			if len(ents) == 0 {
+				dirobj, err := dirchk(path)
+				if err != nil {
+					return fmt.Errorf("directory object check %q: %w", path, err)
+				}
+				if dirobj {
+					fi, err := d.Info()
+					if err != nil {
+						return fmt.Errorf("dir info %q: %w", path, err)
+					}
+					etag, err := getetag(path)
+					if err != nil {
+						return fmt.Errorf("get etag %q: %w", path, err)
+					}
+					path := path + "/"
+					objects = append(objects, types.Object{
+						ETag:         &etag,
+						Key:          &path,
+						LastModified: GetTimePtr(fi.ModTime()),
+					})
+				}
+			}
+
+			return nil
+		}
+
+		if !pastMarker {
+			if path != marker {
+				return nil
+			}
+			pastMarker = true
+		}
+
+		// If object doesnt have prefix, dont include in results.
+		if prefix != "" && !strings.HasPrefix(path, prefix) {
+			return nil
+		}
+
+		if delimiter == "" {
+			// If no delimeter specified, then all files with matching
+			// prefix are included in results
+			fi, err := d.Info()
+			if err != nil {
+				return fmt.Errorf("get info for %v: %w", path, err)
+			}
+			etag, err := getetag(path)
+			if err != nil {
+				return fmt.Errorf("get etag %q: %w", path, err)
+			}
+
+			objects = append(objects, types.Object{
+				ETag:         &etag,
+				Key:          &path,
+				LastModified: GetTimePtr(fi.ModTime()),
+				Size:         fi.Size(),
+			})
+
+			if max > 0 && (len(objects)+len(cpmap)) == max {
+				pastMax = true
+			}
+
+			return nil
+		}
+
+		// Since delimiter is specified, we only want results that
+		// do not contain the delimiter beyond the prefix.  If the
+		// delimiter exists past the prefix, then the substring
+		// between the prefix and delimiter is part of common prefixes.
+		//
+		// For example:
+		// prefix = A/
+		// delimeter = /
+		// and objects:
+		// A/file
+		// A/B/file
+		// B/C
+		// would return:
+		// objects: A/file
+		// common prefix: A/B/
+		//
+		// Note: No obects are included past the common prefix since
+		// these are all rolled up into the common prefix.
+		// Note: The delimeter can be anything, so we have to operate on
+		// the full path without any assumptions on posix directory heirarchy
+		// here.  Usually the delimeter will be "/", but thats not required.
+		suffix := strings.TrimPrefix(path, prefix)
+		before, _, found := strings.Cut(suffix, delimiter)
+		if !found {
+			fi, err := d.Info()
+			if err != nil {
+				return fmt.Errorf("get info for %v: %w", path, err)
+			}
+			etag, err := getetag(path)
+			if err != nil {
+				return fmt.Errorf("get etag %q: %w", path, err)
+			}
+			objects = append(objects, types.Object{
+				ETag:         &etag,
+				Key:          &path,
+				LastModified: GetTimePtr(fi.ModTime()),
+				Size:         fi.Size(),
+			})
+			if (len(objects) + len(cpmap)) == max {
+				pastMax = true
+			}
+			return nil
+		}
+
+		// Common prefixes are a set, so should not have duplicates.
+		// These are abstractly a "directory", so need to include the
+		// delimeter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
+		if (len(objects) + len(cpmap)) == max {
+			pastMax = true
+		}
+
+		return nil
+	})
+	if err != nil {
+		return WalkResults{}, err
+	}
+
+	var commonPrefixStrings []string
+	for k := range cpmap {
+		commonPrefixStrings = append(commonPrefixStrings, k)
+	}
+	sort.Strings(commonPrefixStrings)
+	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
+	for _, cp := range commonPrefixStrings {
+		pfx := cp
+		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
+			Prefix: &pfx,
+		})
+	}
+
+	return WalkResults{
+		CommonPrefixes: commonPrefixes,
+		Objects:        objects,
+		Truncated:      truncated,
+		NextMarker:     newMarker,
+	}, nil
+}
+
+func contains(a string, strs []string) bool {
+	for _, s := range strs {
+		if s == a {
+			return true
+		}
+	}
+	return false
+}

backend/walk_test.go

@@ -0,0 +1,167 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend_test
+
+import (
+	"io/fs"
+	"testing"
+	"testing/fstest"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
+)
+
+type walkTest struct {
+	fsys     fs.FS
+	expected backend.WalkResults
+	dc       backend.DirObjCheck
+}
+
+func gettag(string) (string, error) { return "myetag", nil }
+
+func TestWalk(t *testing.T) {
+	tests := []walkTest{
+		{
+			// test case from
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
+			fsys: fstest.MapFS{
+				"sample.jpg":                       {},
+				"photos/2006/January/sample.jpg":   {},
+				"photos/2006/February/sample2.jpg": {},
+				"photos/2006/February/sample3.jpg": {},
+				"photos/2006/February/sample4.jpg": {},
+			},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("photos/"),
+				}},
+				Objects: []types.Object{{
+					Key: backend.GetStringPtr("sample.jpg"),
+				}},
+			},
+			dc: func(string) (bool, error) { return false, nil },
+		},
+		{
+			// test case single dir/single file
+			fsys: fstest.MapFS{
+				"test/file": {},
+			},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("test/"),
+				}},
+				Objects: []types.Object{},
+			},
+			dc: func(string) (bool, error) { return true, nil },
+		},
+	}
+
+	for _, tt := range tests {
+		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.dc, gettag, []string{})
+		if err != nil {
+			t.Fatalf("walk: %v", err)
+		}
+
+		compareResults(res, tt.expected, t)
+	}
+}
+
+func compareResults(got, wanted backend.WalkResults, t *testing.T) {
+	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
+		t.Errorf("unexpected common prefix, got %v wanted %v",
+			printCommonPrefixes(got.CommonPrefixes),
+			printCommonPrefixes(wanted.CommonPrefixes))
+	}
+
+	if !compareObjects(got.Objects, wanted.Objects) {
+		t.Errorf("unexpected object, got %v wanted %v",
+			printObjects(got.Objects),
+			printObjects(wanted.Objects))
+	}
+}
+
+func compareCommonPrefix(a, b []types.CommonPrefix) bool {
+	if len(a) == 0 && len(b) == 0 {
+		return true
+	}
+	if len(a) != len(b) {
+		return false
+	}
+
+	for _, cp := range a {
+		if containsCommonPrefix(cp, b) {
+			return true
+		}
+	}
+	return false
+}
+
+func containsCommonPrefix(c types.CommonPrefix, list []types.CommonPrefix) bool {
+	for _, cp := range list {
+		if *c.Prefix == *cp.Prefix {
+			return true
+		}
+	}
+	return false
+}
+
+func printCommonPrefixes(list []types.CommonPrefix) string {
+	res := "["
+	for _, cp := range list {
+		if res == "[" {
+			res = res + *cp.Prefix
+		} else {
+			res = res + ", " + *cp.Prefix
+		}
+	}
+	return res + "]"
+}
+
+func compareObjects(a, b []types.Object) bool {
+	if len(a) == 0 && len(b) == 0 {
+		return true
+	}
+	if len(a) != len(b) {
+		return false
+	}
+
+	for _, cp := range a {
+		if containsObject(cp, b) {
+			return true
+		}
+	}
+	return false
+}
+
+func containsObject(c types.Object, list []types.Object) bool {
+	for _, cp := range list {
+		if *c.Key == *cp.Key {
+			return true
+		}
+	}
+	return false
+}
+
+func printObjects(list []types.Object) string {
+	res := "["
+	for _, cp := range list {
+		if res == "[" {
+			res = res + *cp.Key
+		} else {
+			res = res + ", " + *cp.Key
+		}
+	}
+	return res + "]"
+}

cmd/versitygw/admin.go

@@ -0,0 +1,145 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	adminAccess string
+	adminSecret string
+	adminRegion string
+)
+
+func adminCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "admin",
+		Usage: "admin CLI tool",
+		Description: `admin CLI tool for interacting with admin api.
+		Here is the available api list:
+		create-user
+		`,
+		Subcommands: []*cli.Command{
+			{
+				Name:   "create-user",
+				Usage:  "Create a new user",
+				Action: createUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "access value for the new user",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:     "secret",
+						Usage:    "secret value for the new user",
+						Required: true,
+						Aliases:  []string{"s"},
+					},
+					&cli.StringFlag{
+						Name:     "role",
+						Usage:    "role for the new user",
+						Required: true,
+						Aliases:  []string{"r"},
+					},
+					&cli.StringFlag{
+						Name:    "region",
+						Usage:   "s3 region string for the user",
+						Value:   "us-east-1",
+						Aliases: []string{"rg"},
+					},
+				},
+			},
+		},
+		Flags: []cli.Flag{
+			// TODO: create a configuration file for this
+			&cli.StringFlag{
+				Name:        "adminAccess",
+				Usage:       "admin access account",
+				EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
+				Aliases:     []string{"aa"},
+				Destination: &adminAccess,
+			},
+			&cli.StringFlag{
+				Name:        "adminSecret",
+				Usage:       "admin secret access key",
+				EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
+				Aliases:     []string{"as"},
+				Destination: &adminSecret,
+			},
+			&cli.StringFlag{
+				Name:        "adminRegion",
+				Usage:       "s3 region string",
+				Value:       "us-east-1",
+				Destination: &adminRegion,
+				Aliases:     []string{"ar"},
+			},
+		},
+	}
+}
+
+func createUser(ctx *cli.Context) error {
+	access, secret, role, region := ctx.String("access"), ctx.String("secret"), ctx.String("role"), ctx.String("region")
+	if access == "" || secret == "" || region == "" {
+		return fmt.Errorf("invalid input parameters for the new user")
+	}
+	if role != "admin" && role != "user" {
+		return fmt.Errorf("invalid input parameter for role")
+	}
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:7070/create-user?access=%v&secret=%v&role=%v&region=%v", access, secret, role, region), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%s", body)
+
+	return nil
+}

cmd/versitygw/main.go

@@ -0,0 +1,173 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/auth"
+	"github.com/versity/versitygw/s3api"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+var (
+	port              string
+	rootUserAccess    string
+	rootUserSecret    string
+	region            string
+	certFile, keyFile string
+	debug             bool
+)
+
+var (
+	// Version is the latest tag (set within Makefile)
+	Version = "git"
+	// Build is the commit hash (set within Makefile)
+	Build = "norev"
+	// BuildTime is the date/time of build (set within Makefile)
+	BuildTime = "none"
+)
+
+func main() {
+	app := initApp()
+
+	app.Commands = []*cli.Command{
+		posixCommand(),
+		adminCommand(),
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func initApp() *cli.App {
+	return &cli.App{
+		Name:  "versitygw",
+		Usage: "Start S3 gateway service with specified backend storage.",
+		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.`,
+		Action: func(ctx *cli.Context) error {
+			return ctx.App.Command("help").Run(ctx)
+		},
+		Flags: initFlags(),
+	}
+}
+
+func initFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "version",
+			Usage:   "list versitygw version",
+			Aliases: []string{"v"},
+			Action: func(*cli.Context, bool) error {
+				fmt.Println("Version  :", Version)
+				fmt.Println("Build    :", Build)
+				fmt.Println("BuildTime:", BuildTime)
+				os.Exit(0)
+				return nil
+			},
+		},
+		&cli.StringFlag{
+			Name:        "port",
+			Usage:       "gateway listen address <ip>:<port> or :<port>",
+			Value:       ":7070",
+			Destination: &port,
+			Aliases:     []string{"p"},
+		},
+		&cli.StringFlag{
+			Name:        "access",
+			Usage:       "root user access key",
+			EnvVars:     []string{"ROOT_ACCESS_KEY_ID", "ROOT_ACCESS_KEY"},
+			Aliases:     []string{"a"},
+			Destination: &rootUserAccess,
+		},
+		&cli.StringFlag{
+			Name:        "secret",
+			Usage:       "root user secret access key",
+			EnvVars:     []string{"ROOT_SECRET_ACCESS_KEY", "ROOT_SECRET_KEY"},
+			Aliases:     []string{"s"},
+			Destination: &rootUserSecret,
+		},
+		&cli.StringFlag{
+			Name:        "region",
+			Usage:       "s3 region string",
+			Value:       "us-east-1",
+			Destination: &region,
+			Aliases:     []string{"r"},
+		},
+		&cli.StringFlag{
+			Name:        "cert",
+			Usage:       "TLS cert file",
+			Destination: &certFile,
+		},
+		&cli.StringFlag{
+			Name:        "key",
+			Usage:       "TLS key file",
+			Destination: &keyFile,
+		},
+		&cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "enable debug output",
+			Destination: &debug,
+		},
+	}
+}
+
+func runGateway(be backend.Backend) error {
+	app := fiber.New(fiber.Config{
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+		BodyLimit:    5 * 1024 * 1024 * 1024,
+	})
+
+	var opts []s3api.Option
+
+	if certFile != "" || keyFile != "" {
+		if certFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if keyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		opts = append(opts, s3api.WithTLS(cert))
+	}
+
+	if debug {
+		opts = append(opts, s3api.WithDebug())
+	}
+
+	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
+		Access: rootUserAccess,
+		Secret: rootUserSecret,
+		Region: region,
+	}, port, auth.New(), opts...)
+	if err != nil {
+		return fmt.Errorf("init gateway: %v", err)
+	}
+
+	return srv.Serve()
+}

cmd/versitygw/posix.go

@@ -0,0 +1,53 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/posix"
+)
+
+func posixCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "posix",
+		Usage: "posix filesystem storage backend",
+		Description: `Any posix filesystem that supports extended attributes. The top level
+directory for the gateway must be provided. All sub directories of the
+top level directory are treated as buckets, and all files/directories
+below the "bucket directory" are treated as the objects. The object
+name is split on "/" separator to translate to posix storage.
+For example:
+top level: /mnt/fs/gwroot
+bucket: mybucket
+object: a/b/c/myobject
+will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
+		Action: runPosix,
+	}
+}
+
+func runPosix(ctx *cli.Context) error {
+	if ctx.NArg() == 0 {
+		return fmt.Errorf("no directory provided for operation")
+	}
+
+	be, err := posix.New(ctx.Args().Get(0))
+	if err != nil {
+		return fmt.Errorf("init posix: %v", err)
+	}
+
+	return runGateway(be)
+}

go.mod

@@ -0,0 +1,41 @@
+module github.com/versity/versitygw
+
+go 1.20
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.18.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1
+	github.com/aws/smithy-go v1.13.5
+	github.com/gofiber/fiber/v2 v2.46.0
+	github.com/google/uuid v1.3.0
+	github.com/pkg/xattr v0.4.9
+	github.com/urfave/cli/v2 v2.25.4
+	github.com/valyala/fasthttp v1.47.0
+	golang.org/x/sys v0.8.0
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
+	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+)

go.sum

@@ -0,0 +1,123 @@
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
+github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 h1:AzwRi5OKKwo4QNqPf7TjeO+tK8AyOK3GVSwmRPo7/Cs=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25/go.mod h1:SUbB4wcbSEyCvqBxv/O/IBf93RbEze7U7OnoTlpPB+g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 h1:vGWm5vTpMr39tEZfQeDiDAMgk+5qsnvRny3FjLpnH5w=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28/go.mod h1:spfrICMD6wCAhjhzHuy6DOZZ+LAIY10UxhUmLzpJTTs=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 h1:NbWkRxEEIRSCqxhsHQuMiTH7yo+JZW1gp8v3elSVMTQ=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1o4ZDAM9rQp5NsY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8=
+github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
+github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns=
+github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
+github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
+github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
+github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
+github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
+github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
+github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
+github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
+github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
+github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
+github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
+github.com/urfave/cli/v2 v2.25.4 h1:HyYwPrTO3im9rYhUff/ZNs78eolxt0nJ4LN+9yJKSH4=
+github.com/urfave/cli/v2 v2.25.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
+github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

s3api/controllers/admin.go

@@ -0,0 +1,49 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"fmt"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/backend/auth"
+)
+
+type AdminController struct {
+	IAMService auth.IAMService
+}
+
+func NewAdminController() AdminController {
+	return AdminController{IAMService: auth.New()}
+}
+
+func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
+	access, secret, role, region := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role"), ctx.Query("region")
+	requesterRole := ctx.Locals("role")
+
+	if requesterRole != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+
+	user := auth.Account{Secret: secret, Role: role, Region: region}
+
+	err := c.IAMService.CreateAccount(access, &user)
+	if err != nil {
+		return fmt.Errorf("failed to create a user: %w", err)
+	}
+
+	ctx.SendString("The user has been created successfully")
+	return nil
+}

s3api/controllers/base.go

@@ -0,0 +1,465 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+)
+
+type S3ApiController struct {
+	be backend.Backend
+}
+
+func New(be backend.Backend) S3ApiController {
+	return S3ApiController{be: be}
+}
+
+func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
+	res, err := c.be.ListBuckets()
+	return SendXMLResponse(ctx, res, err)
+}
+
+func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
+	bucket := ctx.Params("bucket")
+	key := ctx.Params("key")
+	keyEnd := ctx.Params("*1")
+	uploadId := ctx.Query("uploadId")
+	maxParts := ctx.QueryInt("max-parts", 0)
+	partNumberMarker := ctx.QueryInt("part-number-marker", 0)
+	acceptRange := ctx.Get("Range")
+	if keyEnd != "" {
+		key = strings.Join([]string{key, keyEnd}, "/")
+	}
+
+	if uploadId != "" {
+		if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts))
+		}
+		if partNumberMarker < 0 || (partNumberMarker == 0 && ctx.Query("part-number-marker") != "") {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker))
+		}
+		res, err := c.be.ListObjectParts(bucket, key, uploadId, partNumberMarker, maxParts)
+		return SendXMLResponse(ctx, res, err)
+	}
+
+	if ctx.Request().URI().QueryArgs().Has("acl") {
+		res, err := c.be.GetObjectAcl(bucket, key)
+		return SendXMLResponse(ctx, res, err)
+	}
+
+	if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
+		res, err := c.be.GetObjectAttributes(bucket, key, strings.Split(attrs, ","))
+		return SendXMLResponse(ctx, res, err)
+	}
+
+	res, err := c.be.GetObject(bucket, key, acceptRange, ctx.Response().BodyWriter())
+	if err != nil {
+		return SendResponse(ctx, err)
+	}
+	if res == nil {
+		return SendResponse(ctx, fmt.Errorf("get object nil response"))
+	}
+
+	utils.SetMetaHeaders(ctx, res.Metadata)
+	var lastmod string
+	if res.LastModified != nil {
+		lastmod = res.LastModified.Format(timefmt)
+	}
+	utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+		{
+			Key:   "Content-Length",
+			Value: fmt.Sprint(res.ContentLength),
+		},
+		{
+			Key:   "Content-Type",
+			Value: getstring(res.ContentType),
+		},
+		{
+			Key:   "Content-Encoding",
+			Value: getstring(res.ContentEncoding),
+		},
+		{
+			Key:   "ETag",
+			Value: getstring(res.ETag),
+		},
+		{
+			Key:   "Last-Modified",
+			Value: lastmod,
+		},
+	})
+	return ctx.SendStatus(http.StatusOK)
+}
+
+func getstring(s *string) string {
+	if s == nil {
+		return ""
+	}
+	return *s
+}
+
+func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
+	bucket := ctx.Params("bucket")
+	prefix := ctx.Query("prefix")
+	marker := ctx.Query("continuation-token")
+	delimiter := ctx.Query("delimiter")
+	maxkeys := ctx.QueryInt("max-keys")
+
+	if ctx.Request().URI().QueryArgs().Has("acl") {
+		res, err := c.be.GetBucketAcl(ctx.Params("bucket"))
+		return SendXMLResponse(ctx, res, err)
+	}
+
+	if ctx.Request().URI().QueryArgs().Has("uploads") {
+		res, err := c.be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
+		return SendXMLResponse(ctx, res, err)
+	}
+
+	if ctx.QueryInt("list-type") == 2 {
+		res, err := c.be.ListObjectsV2(bucket, prefix, marker, delimiter, maxkeys)
+		return SendXMLResponse(ctx, res, err)
+	}
+
+	res, err := c.be.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
+	return SendXMLResponse(ctx, res, err)
+}
+
+func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
+	bucket, acl, grantFullControl, grantRead, grantReadACP, granWrite, grantWriteACP :=
+		ctx.Params("bucket"),
+		ctx.Get("X-Amz-Acl"),
+		ctx.Get("X-Amz-Grant-Full-Control"),
+		ctx.Get("X-Amz-Grant-Read"),
+		ctx.Get("X-Amz-Grant-Read-Acp"),
+		ctx.Get("X-Amz-Grant-Write"),
+		ctx.Get("X-Amz-Grant-Write-Acp")
+
+	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
+
+	if grants != "" || acl != "" {
+		if grants != "" && acl != "" {
+			return errors.New("wrong api call")
+		}
+		err := c.be.PutBucketAcl(&s3.PutBucketAclInput{
+			Bucket:           &bucket,
+			ACL:              types.BucketCannedACL(acl),
+			GrantFullControl: &grantFullControl,
+			GrantRead:        &grantRead,
+			GrantReadACP:     &grantReadACP,
+			GrantWrite:       &granWrite,
+			GrantWriteACP:    &grantWriteACP,
+		})
+
+		return SendResponse(ctx, err)
+	}
+
+	err := c.be.PutBucket(bucket)
+	return SendResponse(ctx, err)
+}
+
+func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
+	bucket := ctx.Params("bucket")
+	keyStart := ctx.Params("key")
+	keyEnd := ctx.Params("*1")
+	uploadId := ctx.Query("uploadId")
+	partNumberStr := ctx.Query("partNumber")
+
+	// Copy source headers
+	copySource := ctx.Get("X-Amz-Copy-Source")
+	copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match")
+	copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match")
+	copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since")
+	copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since")
+
+	// Permission headers
+	acl := ctx.Get("X-Amz-Acl")
+	grantFullControl := ctx.Get("X-Amz-Grant-Full-Control")
+	grantRead := ctx.Get("X-Amz-Grant-Read")
+	grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp")
+	granWrite := ctx.Get("X-Amz-Grant-Write")
+	grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp")
+
+	// Other headers
+	contentLengthStr := ctx.Get("Content-Length")
+
+	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
+
+	if keyEnd != "" {
+		keyStart = strings.Join([]string{keyStart, keyEnd}, "/")
+	}
+	path := ctx.Path()
+	if path[len(path)-1:] == "/" && keyStart[len(keyStart)-1:] != "/" {
+		keyStart = keyStart + "/"
+	}
+
+	var contentLength int64
+	if contentLengthStr != "" {
+		var err error
+		contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+		if err != nil {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
+		}
+	}
+
+	if uploadId != "" && partNumberStr != "" {
+		partNumber := ctx.QueryInt("partNumber", -1)
+		if partNumber < 1 {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart))
+		}
+
+		body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
+		etag, err := c.be.PutObjectPart(bucket, keyStart, uploadId,
+			partNumber, contentLength, body)
+		ctx.Response().Header.Set("Etag", etag)
+		return SendResponse(ctx, err)
+	}
+
+	if grants != "" || acl != "" {
+		if grants != "" && acl != "" {
+			return errors.New("wrong api call")
+		}
+
+		err := c.be.PutObjectAcl(&s3.PutObjectAclInput{
+			Bucket:           &bucket,
+			Key:              &keyStart,
+			ACL:              types.ObjectCannedACL(acl),
+			GrantFullControl: &grantFullControl,
+			GrantRead:        &grantRead,
+			GrantReadACP:     &grantReadACP,
+			GrantWrite:       &granWrite,
+			GrantWriteACP:    &grantWriteACP,
+		})
+		return SendResponse(ctx, err)
+	}
+
+	if copySource != "" {
+		_, _, _, _ = copySrcIfMatch, copySrcIfNoneMatch,
+			copySrcModifSince, copySrcUnmodifSince
+		copySourceSplit := strings.Split(copySource, "/")
+		srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:]
+
+		res, err := c.be.CopyObject(srcBucket, strings.Join(srcObject, "/"), bucket, keyStart)
+		return SendXMLResponse(ctx, res, err)
+	}
+
+	metadata := utils.GetUserMetaData(&ctx.Request().Header)
+
+	etag, err := c.be.PutObject(&s3.PutObjectInput{
+		Bucket:        &bucket,
+		Key:           &keyStart,
+		ContentLength: contentLength,
+		Metadata:      metadata,
+		Body:          bytes.NewReader(ctx.Request().Body()),
+	})
+	ctx.Response().Header.Set("ETag", etag)
+	return SendResponse(ctx, err)
+}
+
+func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
+	err := c.be.DeleteBucket(ctx.Params("bucket"))
+	return SendResponse(ctx, err)
+}
+
+func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
+	var dObj types.Delete
+	if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
+		return errors.New("wrong api call")
+	}
+
+	err := c.be.DeleteObjects(ctx.Params("bucket"), &s3.DeleteObjectsInput{Delete: &dObj})
+	return SendResponse(ctx, err)
+}
+
+func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
+	bucket := ctx.Params("bucket")
+	key := ctx.Params("key")
+	keyEnd := ctx.Params("*1")
+	uploadId := ctx.Query("uploadId")
+
+	if keyEnd != "" {
+		key = strings.Join([]string{key, keyEnd}, "/")
+	}
+
+	if uploadId != "" {
+		expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer")
+
+		err := c.be.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
+			UploadId:            &uploadId,
+			Bucket:              &bucket,
+			Key:                 &key,
+			ExpectedBucketOwner: &expectedBucketOwner,
+			RequestPayer:        types.RequestPayer(requestPayer),
+		})
+		return SendResponse(ctx, err)
+	}
+
+	err := c.be.DeleteObject(bucket, key)
+	return SendResponse(ctx, err)
+}
+
+func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
+	_, err := c.be.HeadBucket(ctx.Params("bucket"))
+	// TODO: set bucket response headers
+	return SendResponse(ctx, err)
+}
+
+const (
+	timefmt = "Mon, 02 Jan 2006 15:04:05 GMT"
+)
+
+func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
+	bucket := ctx.Params("bucket")
+	key := ctx.Params("key")
+	keyEnd := ctx.Params("*1")
+	if keyEnd != "" {
+		key = strings.Join([]string{key, keyEnd}, "/")
+	}
+
+	res, err := c.be.HeadObject(bucket, key)
+	if err != nil {
+		return SendResponse(ctx, err)
+	}
+	if res == nil {
+		return SendResponse(ctx, fmt.Errorf("head object nil response"))
+	}
+
+	utils.SetMetaHeaders(ctx, res.Metadata)
+	var lastmod string
+	if res.LastModified != nil {
+		lastmod = res.LastModified.Format(timefmt)
+	}
+	utils.SetResponseHeaders(ctx, []utils.CustomHeader{
+		{
+			Key:   "Content-Length",
+			Value: fmt.Sprint(res.ContentLength),
+		},
+		{
+			Key:   "Content-Type",
+			Value: getstring(res.ContentType),
+		},
+		{
+			Key:   "Content-Encoding",
+			Value: getstring(res.ContentEncoding),
+		},
+		{
+			Key:   "ETag",
+			Value: getstring(res.ETag),
+		},
+		{
+			Key:   "Last-Modified",
+			Value: lastmod,
+		},
+	})
+
+	return SendResponse(ctx, nil)
+}
+
+func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
+	bucket := ctx.Params("bucket")
+	key := ctx.Params("key")
+	keyEnd := ctx.Params("*1")
+	uploadId := ctx.Query("uploadId")
+
+	if keyEnd != "" {
+		key = strings.Join([]string{key, keyEnd}, "/")
+	}
+
+	var restoreRequest s3.RestoreObjectInput
+	if ctx.Request().URI().QueryArgs().Has("restore") {
+		xmlErr := xml.Unmarshal(ctx.Body(), &restoreRequest)
+		if xmlErr != nil {
+			return errors.New("wrong api call")
+		}
+		err := c.be.RestoreObject(bucket, key, &restoreRequest)
+		return SendResponse(ctx, err)
+	}
+
+	if uploadId != "" {
+		data := struct {
+			Parts []types.Part `xml:"Part"`
+		}{}
+
+		if err := xml.Unmarshal(ctx.Body(), &data); err != nil {
+			return errors.New("wrong api call")
+		}
+
+		res, err := c.be.CompleteMultipartUpload(bucket, key, uploadId, data.Parts)
+		return SendXMLResponse(ctx, res, err)
+	}
+	res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
+	return SendXMLResponse(ctx, res, err)
+}
+
+func SendResponse(ctx *fiber.Ctx, err error) error {
+	if err != nil {
+		serr, ok := err.(s3err.APIError)
+		if ok {
+			ctx.Status(serr.HTTPStatusCode)
+			return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", ""))
+		}
+		return ctx.Send(s3err.GetAPIErrorResponse(
+			s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
+	}
+
+	// https://github.com/gofiber/fiber/issues/2080
+	// ctx.SendStatus() sets incorrect content length on HEAD request
+	ctx.Status(http.StatusOK)
+	return nil
+}
+
+func SendXMLResponse(ctx *fiber.Ctx, resp any, err error) error {
+	if err != nil {
+		serr, ok := err.(s3err.APIError)
+		if ok {
+			ctx.Status(serr.HTTPStatusCode)
+			return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", ""))
+		}
+
+		fmt.Fprintf(os.Stderr, "Internal Error, req:\n%v\nerr:\n%v\n",
+			ctx.Request(), err)
+
+		return ctx.Send(s3err.GetAPIErrorResponse(
+			s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
+	}
+
+	var b []byte
+
+	if resp != nil {
+		if b, err = xml.Marshal(resp); err != nil {
+			return err
+		}
+
+		if len(b) > 0 {
+			ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
+		}
+	}
+
+	return ctx.Send(b)
+}

s3api/middlewares/authentication.go

@@ -0,0 +1,160 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go/logging"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/backend/auth"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	iso8601Format = "20060102T150405Z"
+)
+
+type RootUserConfig struct {
+	Access string
+	Secret string
+	Region string
+}
+
+func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, debug bool) fiber.Handler {
+	acct := accounts{root: root, iam: iam}
+
+	return func(ctx *fiber.Ctx) error {
+		authorization := ctx.Get("Authorization")
+		if authorization == "" {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty))
+		}
+
+		// Check the signature version
+		authParts := strings.Split(authorization, " ")
+		if len(authParts) < 4 {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
+		}
+		if authParts[0] != "AWS4-HMAC-SHA256" {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported))
+		}
+
+		credKv := strings.Split(authParts[1], "=")
+		if len(credKv) != 2 {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
+		}
+		creds := strings.Split(credKv[1], "/")
+		if len(creds) < 4 {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
+		}
+
+		signHdrKv := strings.Split(authParts[2], "=")
+		if len(signHdrKv) != 2 {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed))
+		}
+		signedHdrs := strings.Split(signHdrKv[1], ";")
+
+		account := acct.getAccount(creds[0])
+		if account == nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID))
+		}
+
+		// Check X-Amz-Date header
+		date := ctx.Get("X-Amz-Date")
+		if date == "" {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader))
+		}
+
+		// Parse the date and check the date validity
+		tdate, err := time.Parse(iso8601Format, date)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate))
+		}
+
+		// Calculate the hash of the request payload
+		hashedPayload := sha256.Sum256(ctx.Body())
+		hexPayload := hex.EncodeToString(hashedPayload[:])
+
+		hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
+
+		// Compare the calculated hash with the hash provided
+		if hashPayloadHeader != hexPayload {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch))
+		}
+
+		// Create a new http request instance from fasthttp request
+		req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
+		}
+
+		signer := v4.NewSigner()
+
+		signErr := signer.SignHTTP(req.Context(), aws.Credentials{
+			AccessKeyID:     creds[0],
+			SecretAccessKey: account.Secret,
+		}, req, hexPayload, creds[3], account.Region, tdate, func(options *v4.SignerOptions) {
+			if debug {
+				options.LogSigning = true
+				options.Logger = logging.NewStandardLogger(os.Stderr)
+			}
+		})
+		if signErr != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError))
+		}
+
+		parts := strings.Split(req.Header.Get("Authorization"), " ")
+		if len(parts) < 4 {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields))
+		}
+		calculatedSign := strings.Split(parts[3], "=")[1]
+		expectedSign := strings.Split(authParts[3], "=")[1]
+
+		if expectedSign != calculatedSign {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch))
+		}
+
+		ctx.Locals("role", account.Role)
+
+		return ctx.Next()
+	}
+}
+
+type accounts struct {
+	root RootUserConfig
+	iam  auth.IAMService
+}
+
+func (a accounts) getAccount(access string) *auth.Account {
+	var account *auth.Account
+	if access == a.root.Access {
+		account = &auth.Account{
+			Secret: a.root.Secret,
+			Role:   "admin",
+			Region: a.root.Region,
+		}
+	} else {
+		account = a.iam.GetUserAccount(access)
+	}
+	return account
+}

s3api/middlewares/md5.go

@@ -0,0 +1,43 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+)
+
+func VerifyMD5Body() fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		incomingSum := ctx.Get("Content-Md5")
+		if incomingSum == "" {
+			return ctx.Next()
+		}
+
+		sum := md5.Sum(ctx.Body())
+		calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
+
+		if incomingSum != calculatedSum {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest))
+		}
+
+		return ctx.Next()
+
+	}
+}

s3api/router.go

@@ -0,0 +1,70 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/auth"
+	"github.com/versity/versitygw/s3api/controllers"
+)
+
+type S3ApiRouter struct{}
+
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
+	s3ApiController := controllers.New(be)
+	adminController := controllers.AdminController{IAMService: iam}
+
+	// TODO: think of better routing system
+	app.Post("/create-user", adminController.CreateUser)
+	// ListBuckets action
+	app.Get("/", s3ApiController.ListBuckets)
+
+	// PutBucket action
+	// PutBucketAcl action
+	app.Put("/:bucket", s3ApiController.PutBucketActions)
+
+	// DeleteBucket action
+	app.Delete("/:bucket", s3ApiController.DeleteBucket)
+
+	// HeadBucket
+	app.Head("/:bucket", s3ApiController.HeadBucket)
+	// GetBucketAcl action
+	// ListMultipartUploads action
+	// ListObjects action
+	// ListObjectsV2 action
+	app.Get("/:bucket", s3ApiController.ListActions)
+
+	// HeadObject action
+	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
+	// GetObjectAcl action
+	// GetObject action
+	// ListObjectParts action
+	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
+	// DeleteObject action
+	// AbortMultipartUpload action
+	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
+	// DeleteObjects action
+	app.Post("/:bucket", s3ApiController.DeleteObjects)
+	// CompleteMultipartUpload action
+	// CreateMultipartUpload
+	// RestoreObject action
+	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
+	// CopyObject action
+	// PutObject action
+	// UploadPart action
+	// UploadPartCopy action
+	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
+}

s3api/router_test.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/auth"
+)
+
+func TestS3ApiRouter_Init(t *testing.T) {
+	type args struct {
+		app *fiber.App
+		be  backend.Backend
+		iam auth.IAMService
+	}
+	tests := []struct {
+		name string
+		sa   *S3ApiRouter
+		args args
+	}{
+		{
+			name: "Initialize S3 api router",
+			sa:   &S3ApiRouter{},
+			args: args{
+				app: fiber.New(),
+				be:  backend.BackendUnsupported{},
+				iam: auth.IAMServiceUnsupported{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam)
+		})
+	}
+}

s3api/server.go

@@ -0,0 +1,73 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"crypto/tls"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/auth"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+type S3ApiServer struct {
+	app     *fiber.App
+	backend backend.Backend
+	router  *S3ApiRouter
+	port    string
+	cert    *tls.Certificate
+	debug   bool
+}
+
+func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) {
+	server := &S3ApiServer{
+		app:     app,
+		backend: be,
+		router:  new(S3ApiRouter),
+		port:    port,
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
+
+	app.Use(middlewares.VerifyV4Signature(root, iam, server.debug))
+	app.Use(logger.New())
+	app.Use(middlewares.VerifyMD5Body())
+	server.router.Init(app, be, iam)
+	return server, nil
+}
+
+// Option sets various options for New()
+type Option func(*S3ApiServer)
+
+// WithTLS sets TLS Credentials
+func WithTLS(cert tls.Certificate) Option {
+	return func(s *S3ApiServer) { s.cert = &cert }
+}
+
+// WithDebug sets debug output
+func WithDebug() Option {
+	return func(s *S3ApiServer) { s.debug = true }
+}
+
+func (sa *S3ApiServer) Serve() (err error) {
+	if sa.cert != nil {
+		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
+	}
+	return sa.app.Listen(sa.port)
+}

s3api/server_test.go

@@ -0,0 +1,102 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/auth"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+func TestNew(t *testing.T) {
+	type args struct {
+		app  *fiber.App
+		be   backend.Backend
+		port string
+		root middlewares.RootUserConfig
+	}
+
+	app := fiber.New()
+	be := backend.BackendUnsupported{}
+	router := S3ApiRouter{}
+	port := ":7070"
+
+	tests := []struct {
+		name            string
+		args            args
+		wantS3ApiServer *S3ApiServer
+		wantErr         bool
+	}{
+		{
+			name: "Create S3 api server",
+			args: args{
+				app:  app,
+				be:   be,
+				port: port,
+				root: middlewares.RootUserConfig{},
+			},
+			wantS3ApiServer: &S3ApiServer{
+				app:     app,
+				port:    port,
+				router:  &router,
+				backend: be,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
+				tt.args.port, auth.IAMServiceUnsupported{})
+			if (err != nil) != tt.wantErr {
+				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotS3ApiServer, tt.wantS3ApiServer) {
+				t.Errorf("New() = %v, want %v", gotS3ApiServer, tt.wantS3ApiServer)
+			}
+		})
+	}
+}
+
+func TestS3ApiServer_Serve(t *testing.T) {
+	tests := []struct {
+		name    string
+		sa      *S3ApiServer
+		wantErr bool
+	}{
+		{
+			name:    "Return error when serving S3 api server with invalid address",
+			wantErr: true,
+			sa: &S3ApiServer{
+				app:     fiber.New(),
+				backend: backend.BackendUnsupported{},
+				port:    "Wrong address",
+				router:  &S3ApiRouter{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.sa.Serve(); (err != nil) != tt.wantErr {
+				t.Errorf("S3ApiServer.Serve() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

s3api/utils/utils.go

@@ -0,0 +1,93 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
+)
+
+func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
+	metadata = make(map[string]string)
+	headers.VisitAll(func(key, value []byte) {
+		if strings.HasPrefix(string(key), "X-Amz-Meta-") {
+			trimmedKey := strings.TrimPrefix(string(key), "X-Amz-Meta-")
+			headerValue := string(value)
+			metadata[trimmedKey] = headerValue
+		}
+	})
+
+	return
+}
+
+func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
+	req := ctx.Request()
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
+	if err != nil {
+		return nil, errors.New("error in creating an http request")
+	}
+
+	// Set the request headers
+	req.Header.VisitAll(func(key, value []byte) {
+		keyStr := string(key)
+		if includeHeader(keyStr, signedHdrs) {
+			httpReq.Header.Add(keyStr, string(value))
+		}
+	})
+
+	// Check if Content-Length in signed headers
+	// If content length is non 0, then the header will be included
+	if !includeHeader("Content-Length", signedHdrs) {
+		httpReq.ContentLength = 0
+	}
+
+	// Set the Host header
+	httpReq.Host = string(req.Header.Host())
+
+	return httpReq, nil
+}
+
+func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
+	for key, val := range meta {
+		ctx.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
+	}
+}
+
+type CustomHeader struct {
+	Key   string
+	Value string
+}
+
+func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
+	for _, header := range headers {
+		ctx.Set(header.Key, header.Value)
+	}
+}
+
+func includeHeader(hdr string, signedHdrs []string) bool {
+	for _, shdr := range signedHdrs {
+		if strings.EqualFold(hdr, shdr) {
+			return true
+		}
+	}
+	return false
+}

s3api/utils/utils_test.go

@@ -0,0 +1,119 @@
+package utils
+
+import (
+	"bytes"
+	"net/http"
+	"reflect"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
+)
+
+func TestCreateHttpRequestFromCtx(t *testing.T) {
+	type args struct {
+		ctx *fiber.Ctx
+	}
+
+	app := fiber.New()
+
+	// Expected output, Case 1
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
+	req := ctx.Request()
+	request, _ := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
+
+	// Case 2
+	ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
+	req2 := ctx2.Request()
+	req2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
+
+	request2, _ := http.NewRequest(string(req2.Header.Method()), req2.URI().String(), bytes.NewReader(req2.Body()))
+	request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
+
+	tests := []struct {
+		name    string
+		args    args
+		want    *http.Request
+		wantErr bool
+	}{
+		{
+			name: "Success-response",
+			args: args{
+				ctx: ctx,
+			},
+			want:    request,
+			wantErr: false,
+		},
+		{
+			name: "Success-response-With-Headers",
+			args: args{
+				ctx: ctx2,
+			},
+			want:    request2,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := CreateHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"})
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if !reflect.DeepEqual(got.Header, tt.want.Header) {
+				t.Errorf("CreateHttpRequestFromCtx() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetUserMetaData(t *testing.T) {
+	type args struct {
+		headers *fasthttp.RequestHeader
+	}
+
+	app := fiber.New()
+
+	// Case 1
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
+	req := ctx.Request()
+
+	// Case 2
+	ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
+	req2 := ctx2.Request()
+
+	req2.Header.Add("X-Amz-Meta-Name", "Nick")
+	req2.Header.Add("X-Amz-Meta-Age", "27")
+
+	tests := []struct {
+		name         string
+		args         args
+		wantMetadata map[string]string
+	}{
+		{
+			name: "Success-empty-response",
+			args: args{
+				headers: &req.Header,
+			},
+			wantMetadata: map[string]string{},
+		},
+		{
+			name: "Success-non-empty-response",
+			args: args{
+				headers: &req2.Header,
+			},
+			wantMetadata: map[string]string{
+				"Age":  "27",
+				"Name": "Nick",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotMetadata := GetUserMetaData(tt.args.headers); !reflect.DeepEqual(gotMetadata, tt.wantMetadata) {
+				t.Errorf("GetUserMetaData() = %v, want %v", gotMetadata, tt.wantMetadata)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -0,0 +1,410 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3err
+
+import (
+	"bytes"
+	"encoding/xml"
+	"net/http"
+)
+
+// APIError structure
+type APIError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// APIErrorResponse - error response format
+type APIErrorResponse struct {
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string
+	Message    string
+	Key        string `xml:"Key,omitempty" json:"Key,omitempty"`
+	BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
+	Resource   string
+	Region     string `xml:"Region,omitempty" json:"Region,omitempty"`
+	RequestID  string `xml:"RequestId" json:"RequestId"`
+	HostID     string `xml:"HostId" json:"HostId"`
+}
+
+func (A APIError) Error() string {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	_ = e.Encode(A)
+	return bytesBuffer.String()
+}
+
+// ErrorCode type of error status.
+type ErrorCode int
+
+// Error codes, see full list at http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+const (
+	ErrNone ErrorCode = iota
+	ErrAccessDenied
+	ErrMethodNotAllowed
+	ErrBucketNotEmpty
+	ErrBucketAlreadyExists
+	ErrBucketAlreadyOwnedByYou
+	ErrNoSuchBucket
+	ErrNoSuchKey
+	ErrNoSuchUpload
+	ErrInvalidBucketName
+	ErrInvalidDigest
+	ErrInvalidMaxKeys
+	ErrInvalidMaxUploads
+	ErrInvalidMaxParts
+	ErrInvalidPartNumberMarker
+	ErrInvalidPart
+	ErrInternalError
+	ErrInvalidCopyDest
+	ErrInvalidCopySource
+	ErrInvalidTag
+	ErrAuthHeaderEmpty
+	ErrSignatureVersionNotSupported
+	ErrMalformedPOSTRequest
+	ErrPOSTFileRequired
+	ErrPostPolicyConditionInvalidFormat
+	ErrEntityTooSmall
+	ErrEntityTooLarge
+	ErrMissingFields
+	ErrMissingCredTag
+	ErrCredMalformed
+	ErrMalformedXML
+	ErrMalformedDate
+	ErrMalformedPresignedDate
+	ErrMalformedCredentialDate
+	ErrMissingSignHeadersTag
+	ErrMissingSignTag
+	ErrUnsignedHeaders
+	ErrInvalidQueryParams
+	ErrInvalidQuerySignatureAlgo
+	ErrExpiredPresignRequest
+	ErrMalformedExpires
+	ErrNegativeExpires
+	ErrMaximumExpires
+	ErrSignatureDoesNotMatch
+	ErrContentSHA256Mismatch
+	ErrInvalidAccessKeyID
+	ErrRequestNotReadyYet
+	ErrMissingDateHeader
+	ErrInvalidRequest
+	ErrAuthNotSetup
+	ErrNotImplemented
+	ErrPreconditionFailed
+
+	ErrExistingObjectIsDirectory
+	ErrObjectParentIsFile
+)
+
+var errorCodeResponse = map[ErrorCode]APIError{
+	ErrAccessDenied: {
+		Code:           "AccessDenied",
+		Description:    "Access Denied.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMethodNotAllowed: {
+		Code:           "MethodNotAllowed",
+		Description:    "The specified method is not allowed against this resource.",
+		HTTPStatusCode: http.StatusMethodNotAllowed,
+	},
+	ErrBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrBucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrBucketAlreadyOwnedByYou: {
+		Code:           "BucketAlreadyOwnedByYou",
+		Description:    "Your previous request to create the named bucket succeeded and you already own it.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrInvalidBucketName: {
+		Code:           "InvalidBucketName",
+		Description:    "The specified bucket is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidDigest: {
+		Code:           "InvalidDigest",
+		Description:    "The Content-Md5 you specified is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxUploads: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxKeys: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxParts: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartNumberMarker: {
+		Code:           "InvalidArgument",
+		Description:    "Argument partNumberMarker must be an integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNoSuchBucket: {
+		Code:           "NoSuchBucket",
+		Description:    "The specified bucket does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchKey: {
+		Code:           "NoSuchKey",
+		Description:    "The specified key does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchUpload: {
+		Code:           "NoSuchUpload",
+		Description:    "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrInternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+
+	ErrInvalidPart: {
+		Code:           "InvalidPart",
+		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	ErrInvalidCopyDest: {
+		Code:           "InvalidRequest",
+		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCopySource: {
+		Code:           "InvalidArgument",
+		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidTag: {
+		Code:           "InvalidArgument",
+		Description:    "The Tag value you have provided is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedXML: {
+		Code:           "MalformedXML",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthHeaderEmpty: {
+		Code:           "InvalidArgument",
+		Description:    "Authorization header is invalid -- one and only one ' ' (space) required.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSignatureVersionNotSupported: {
+		Code:           "InvalidRequest",
+		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPOSTRequest: {
+		Code:           "MalformedPOSTRequest",
+		Description:    "The body of your POST request is not well-formed multipart/form-data.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPOSTFileRequired: {
+		Code:           "InvalidArgument",
+		Description:    "POST requires exactly one file upload per request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPostPolicyConditionInvalidFormat: {
+		Code:           "PostPolicyInvalidKeyName",
+		Description:    "Invalid according to Policy: Policy Condition failed",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrEntityTooSmall: {
+		Code:           "EntityTooSmall",
+		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrEntityTooLarge: {
+		Code:           "EntityTooLarge",
+		Description:    "Your proposed upload exceeds the maximum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingFields: {
+		Code:           "MissingFields",
+		Description:    "Missing fields in request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingCredTag: {
+		Code:           "InvalidRequest",
+		Description:    "Missing Credential field for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrCredMalformed: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedDate: {
+		Code:           "MalformedDate",
+		Description:    "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPresignedDate: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignHeadersTag: {
+		Code:           "InvalidArgument",
+		Description:    "Signature header missing SignedHeaders field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignTag: {
+		Code:           "AccessDenied",
+		Description:    "Signature header missing Signature field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	ErrUnsignedHeaders: {
+		Code:           "AccessDenied",
+		Description:    "There were headers present in the request which were not signed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQueryParams: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQuerySignatureAlgo: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExpiredPresignRequest: {
+		Code:           "AccessDenied",
+		Description:    "Request has expired",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMalformedExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires should be a number",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNegativeExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be non-negative",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMaximumExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	ErrInvalidAccessKeyID: {
+		Code:           "InvalidAccessKeyId",
+		Description:    "The access key ID you provided does not exist in our records.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+
+	ErrRequestNotReadyYet: {
+		Code:           "AccessDenied",
+		Description:    "Request is not valid yet",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+
+	ErrSignatureDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+
+	ErrContentSHA256Mismatch: {
+		Code:           "XAmzContentSHA256Mismatch",
+		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingDateHeader: {
+		Code:           "AccessDenied",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequest: {
+		Code:           "InvalidRequest",
+		Description:    "Invalid Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthNotSetup: {
+		Code:           "InvalidRequest",
+		Description:    "Signed request requires setting up SeaweedFS S3 authentication",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNotImplemented: {
+		Code:           "NotImplemented",
+		Description:    "A header you provided implies functionality that is not implemented",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrPreconditionFailed: {
+		Code:           "PreconditionFailed",
+		Description:    "At least one of the pre-conditions you specified did not hold",
+		HTTPStatusCode: http.StatusPreconditionFailed,
+	},
+	ErrExistingObjectIsDirectory: {
+		Code:           "ExistingObjectIsDirectory",
+		Description:    "Existing Object is a directory.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrObjectParentIsFile: {
+		Code:           "ObjectParentIsFile",
+		Description:    "Object parent already exists as a file.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+}
+
+// GetAPIError provides API Error for input API error code.
+func GetAPIError(code ErrorCode) APIError {
+	return errorCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func GetAPIErrorResponse(err APIError, resource, requestID, hostID string) []byte {
+	return encodeResponse(APIErrorResponse{
+		Code:       err.Code,
+		Message:    err.Description,
+		BucketName: "",
+		Key:        "",
+		Resource:   resource,
+		Region:     "",
+		RequestID:  requestID,
+		HostID:     hostID,
+	})
+}
+
+// Encodes the response headers into XML format.
+func encodeResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}

s3response/s3response.go

@@ -0,0 +1,96 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3response
+
+import (
+	"encoding/xml"
+)
+
+// Part describes part metadata.
+type Part struct {
+	PartNumber   int
+	LastModified string
+	ETag         string
+	Size         int64
+}
+
+// ListPartsResponse - s3 api list parts response.
+type ListPartsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+
+	Initiator Initiator
+	Owner     Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
+}
+
+// ListMultipartUploadsResponse - s3 api list multipart uploads response.
+type ListMultipartUploadsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
+
+	Bucket             string
+	KeyMarker          string
+	UploadIDMarker     string `xml:"UploadIdMarker"`
+	NextKeyMarker      string
+	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
+	Delimiter          string
+	Prefix             string
+	EncodingType       string `xml:"EncodingType,omitempty"`
+	MaxUploads         int
+	IsTruncated        bool
+
+	// List of pending uploads.
+	Uploads []Upload `xml:"Upload"`
+
+	// Delimed common prefixes.
+	CommonPrefixes []CommonPrefix
+}
+
+// Upload desribes in progress multipart upload
+type Upload struct {
+	Key          string
+	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
+	StorageClass string
+	Initiated    string
+}
+
+// CommonPrefix ListObjectsResponse common prefixes (directory abstraction)
+type CommonPrefix struct {
+	Prefix string
+}
+
+// Initiator same fields as Owner
+type Initiator Owner
+
+// Owner bucket ownership
+type Owner struct {
+	ID          string
+	DisplayName string
+}

versitygw.spec.in

@@ -0,0 +1,31 @@
+%global debug_package %{nil}
+%define pkg_version @@VERSION@@
+
+Name:		versitygw
+Version:	%{pkg_version}
+Release:	1%{?dist}
+Summary:	Versity S3 Gateway
+
+License:	Apache-2.0
+URL:		https://github.com/versity/versitygw
+Source0:	%{name}-%{version}.tar.gz
+%description
+The S3 gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.
+
+BuildRequires:	golang >= 1.20
+
+%prep
+%setup
+
+%build
+make
+
+%install
+mkdir -p %{buildroot}%{_bindir}
+install -m 0755 %{name} %{buildroot}%{_bindir}/
+
+%post
+
+%files
+%{_bindir}/%{name}