feat: Added s3 proxy comparison to upload, download and throughput benchmark tests

Throughput performance test

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      dev-dependencies:
+        patterns:
+          - "*"

.github/workflows/functional.yml

@@ -7,15 +7,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-
     - name: Get Dependencies
       run: |
         go get -v -t -d ./...

.github/workflows/go.yml

@@ -7,15 +7,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-
     - name: Verify all files pass gofmt formatting
       run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
 

.github/workflows/goreleaser.yml

@@ -0,0 +1,31 @@
+name: goreleaser
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+  # packages: write
+  # issues: write
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/static.yml

@@ -1,23 +1,22 @@
 name: staticcheck
 on: pull_request
 jobs: 
+
   build: 
     name: Check
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: "1.20"
-        id: go
-      - 
-        name: "Set up repo"
-        uses: actions/checkout@v1
-        with: 
-          fetch-depth: 1
-      - 
-        name: "staticcheck"
-        uses: dominikh/staticcheck-action@v1.3.0
-        with: 
-          install-go: false
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+ 
+    - name: "staticcheck"
+      uses: dominikh/staticcheck-action@v1.3.0
+      with: 
+        install-go: false

.gitignore

@@ -34,3 +34,5 @@ VERSION
 *.tar.gz
 **/rand.data
 /profile.txt
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,51 @@
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - goos:
+      - linux
+      - darwin
+      # windows is untested, we can start doing windows releases
+      # if someone is interested in taking on testing
+      # - windows
+    main: ./cmd/versitygw
+    binary: ./cmd/versitygw
+    id: versitygw
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge '
+
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

README.md

@@ -1,4 +1,4 @@
-# The Versity Gateway:<br/>A High-Performance S3 to Storage System Translation Service
+# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
 
 <picture>
   <source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
@@ -8,10 +8,14 @@
 
  [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-**Current status:** Alpha, in development not yet suited for production use
+**Current status:** Beta: Most clients functional, work in progress for more test coverage. Issue reports welcome.
 
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
+* Share filesystem directory via S3 protocol
+* Simple to deploy S3 server with a single command
+* Protocol compatibility allows common access to files via posix or S3 
+
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.

auth/acl.go

@@ -17,7 +17,6 @@ package auth
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -105,7 +104,6 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 
 			if *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
-				fmt.Println(fullControlList)
 				for _, str := range fullControlList {
 					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
 				}
@@ -148,7 +146,7 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		}
 
 		// Check if the specified accounts exist
-		accList, err := checkIfAccountsExist(accs, iam)
+		accList, err := CheckIfAccountsExist(accs, iam)
 		if err != nil {
 			return nil, err
 		}
@@ -168,17 +166,21 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 	return result, nil
 }
 
-func checkIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
+func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
 	result := []string{}
 
 	for _, acc := range accs {
 		_, err := iam.GetUserAccount(acc)
-		if err != nil && err != ErrNoSuchUser {
+		if err != nil {
+			if err == ErrNoSuchUser {
+				result = append(result, acc)
+				continue
+			}
+			if err == ErrNotSupported {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			}
 			return nil, fmt.Errorf("check user account: %w", err)
 		}
-		if err == ErrNoSuchUser {
-			result = append(result, acc)
-		}
 	}
 	return result, nil
 }
@@ -198,7 +200,7 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRoot bool) error {
+func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
@@ -237,29 +239,14 @@ func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRo
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
-func IsAdmin(access string, isRoot bool) error {
-	var data IAMConfig
-
+func IsAdmin(acct Account, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
 
-	file, err := os.ReadFile("users.json")
-	if err != nil {
-		return fmt.Errorf("unable to read config file: %w", err)
-	}
-
-	if err := json.Unmarshal(file, &data); err != nil {
-		return err
-	}
-
-	acc, ok := data.AccessAccounts[access]
-	if !ok {
-		return fmt.Errorf("user does not exist")
-	}
-
-	if acc.Role == "admin" {
+	if acct.Role == "admin" {
 		return nil
 	}
-	return fmt.Errorf("only admin users have access to this resource")
+
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }

auth/iam.go

@@ -16,21 +16,72 @@ package auth
 
 import (
 	"errors"
+	"time"
 )
 
 // Account is a gateway IAM account
 type Account struct {
-	Secret string `json:"secret"`
-	Role   string `json:"role"`
+	Access    string `json:"access"`
+	Secret    string `json:"secret"`
+	Role      string `json:"role"`
+	UserID    int    `json:"userID"`
+	GroupID   int    `json:"groupID"`
+	ProjectID int    `json:"projectID"`
 }
 
 // IAMService is the interface for all IAM service implementations
 //
 //go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
 type IAMService interface {
-	CreateAccount(access string, account Account) error
+	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
 	DeleteUserAccount(access string) error
+	ListUserAccounts() ([]Account, error)
+	Shutdown() error
 }
 
 var ErrNoSuchUser = errors.New("user not found")
+
+type Opts struct {
+	Dir            string
+	LDAPServerURL  string
+	LDAPBindDN     string
+	LDAPPassword   string
+	LDAPQueryBase  string
+	LDAPObjClasses string
+	LDAPAccessAtr  string
+	LDAPSecretAtr  string
+	LDAPRoleAtr    string
+	CacheDisable   bool
+	CacheTTL       int
+	CachePrune     int
+}
+
+func New(o *Opts) (IAMService, error) {
+	var svc IAMService
+	var err error
+
+	switch {
+	case o.Dir != "":
+		svc, err = NewInternal(o.Dir)
+	case o.LDAPServerURL != "":
+		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
+			o.LDAPObjClasses)
+	default:
+		// if no iam options selected, default to the single user mode
+		return IAMServiceSingle{}, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if o.CacheDisable {
+		return svc, nil
+	}
+
+	return NewCache(svc,
+		time.Duration(o.CacheTTL)*time.Second,
+		time.Duration(o.CachePrune)*time.Second), nil
+}

auth/iam_cache.go

@@ -0,0 +1,179 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+)
+
+// IAMCache is an in memory cache of the IAM accounts
+// with expiration. This helps to alleviate the load on
+// the real IAM service if the gateway is handling
+// many requests. This forwards account updates to the
+// underlying service, and returns cached results while
+// the in memory account is not expired.
+type IAMCache struct {
+	service  IAMService
+	iamcache *icache
+	cancel   context.CancelFunc
+}
+
+var _ IAMService = &IAMCache{}
+
+type item struct {
+	value Account
+	exp   time.Time
+}
+
+type icache struct {
+	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
+}
+
+func (i *icache) set(k string, v Account) {
+	cpy := v
+	i.Lock()
+	i.items[k] = item{
+		exp:   time.Now().Add(i.expire),
+		value: cpy,
+	}
+	i.Unlock()
+}
+
+func (i *icache) get(k string) (Account, bool) {
+	i.RLock()
+	v, ok := i.items[k]
+	i.RUnlock()
+	if !ok || !v.exp.After(time.Now()) {
+		return Account{}, false
+	}
+	return v.value, true
+}
+
+func (i *icache) Delete(k string) {
+	i.Lock()
+	delete(i.items, k)
+	i.Unlock()
+}
+
+func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+
+		now := time.Now()
+
+		i.Lock()
+		// prune expired entries
+		for k, v := range i.items {
+			if now.After(v.exp) {
+				delete(i.items, k)
+			}
+		}
+		i.Unlock()
+
+		// sleep for the clean interval or context cancelation,
+		// whichever comes first
+		select {
+		case <-ctx.Done():
+		case <-time.After(interval):
+		}
+	}
+}
+
+// NewCache initializes an IAM cache for the provided service. The expireTime
+// is the duration a cache entry can be valid, and the cleanupInterval is
+// how often to scan cache and cleanup expired entries.
+func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
+	i := &IAMCache{
+		service: service,
+		iamcache: &icache{
+			items:  make(map[string]item),
+			expire: expireTime,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go i.iamcache.gcCache(ctx, cleanupInterval)
+	i.cancel = cancel
+
+	return i
+}
+
+// CreateAccount send create to IAM service and creates an account cache entry
+func (c *IAMCache) CreateAccount(account Account) error {
+	err := c.service.CreateAccount(account)
+	if err != nil {
+		return err
+	}
+
+	// we need a copy of account to be able to store beyond the
+	// lifetime of the request, otherwise Fiber will reuse and corrupt
+	// these entries
+	acct := Account{
+		Access: strings.Clone(account.Access),
+		Secret: strings.Clone(account.Secret),
+		Role:   strings.Clone(account.Role),
+	}
+
+	c.iamcache.set(acct.Access, acct)
+	return nil
+}
+
+// GetUserAccount retrieves the cache account if it is in the cache and not
+// expired. Otherwise retrieves from underlying IAM service and caches
+// result for the expire duration.
+func (c *IAMCache) GetUserAccount(access string) (Account, error) {
+	acct, found := c.iamcache.get(access)
+	if found {
+		return acct, nil
+	}
+
+	a, err := c.service.GetUserAccount(access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	c.iamcache.set(access, a)
+	return a, nil
+}
+
+// DeleteUserAccount deletes account from IAM service and cache
+func (c *IAMCache) DeleteUserAccount(access string) error {
+	err := c.service.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.Delete(access)
+	return nil
+}
+
+// ListUserAccounts is a passthrough to the underlying service and
+// does not make use of the cache
+func (c *IAMCache) ListUserAccounts() ([]Account, error) {
+	return c.service.ListUserAccounts()
+}
+
+// Shutdown graceful termination of service
+func (c *IAMCache) Shutdown() error {
+	c.cancel()
+	return nil
+}

auth/iam_internal.go

@@ -16,47 +16,44 @@ package auth
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
-	"hash/crc32"
-	"sync"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+const (
+	iamFile       = "users.json"
+	iamBackupFile = "users.json.backup"
 )
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	storer Storer
-
-	mu     sync.RWMutex
-	accts  IAMConfig
-	serial uint32
+	dir string
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
 type UpdateAcctFunc func([]byte) ([]byte, error)
 
-// Storer is the interface to manage the peristent IAM data for the internal
-// IAM service
-type Storer interface {
-	InitIAM() error
-	GetIAM() ([]byte, error)
-	StoreIAM(UpdateAcctFunc) error
-}
-
-// IAMConfig stores all internal IAM accounts
-type IAMConfig struct {
+// iAMConfig stores all internal IAM accounts
+type iAMConfig struct {
 	AccessAccounts map[string]Account `json:"accessAccounts"`
 }
 
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(s Storer) (*IAMServiceInternal, error) {
+func NewInternal(dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		storer: s,
+		dir: dir,
 	}
 
-	err := i.updateCache()
+	err := i.initIAM()
 	if err != nil {
-		return nil, fmt.Errorf("refresh iam cache: %w", err)
+		return nil, fmt.Errorf("init iam: %w", err)
 	}
 
 	return i, nil
@@ -64,32 +61,23 @@ func NewInternal(s Storer) (*IAMServiceInternal, error) {
 
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
-func (s *IAMServiceInternal) CreateAccount(access string, account Account) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
-		var conf IAMConfig
-
-		if len(data) > 0 {
-			if err := json.Unmarshal(data, &conf); err != nil {
-				return nil, fmt.Errorf("failed to parse iam: %w", err)
-			}
-		} else {
-			conf = IAMConfig{AccessAccounts: map[string]Account{}}
+func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
 		}
 
-		_, ok := conf.AccessAccounts[access]
+		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
 			return nil, fmt.Errorf("account already exists")
 		}
-		conf.AccessAccounts[access] = account
+		conf.AccessAccounts[account.Access] = account
 
 		b, err := json.Marshal(conf)
 		if err != nil {
 			return nil, fmt.Errorf("failed to serialize iam: %w", err)
 		}
-		s.accts = conf
 
 		return b, nil
 	})
@@ -98,25 +86,12 @@ func (s *IAMServiceInternal) CreateAccount(access string, account Account) error
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	data, err := s.storer.GetIAM()
+	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
 	}
 
-	serial := crc32.ChecksumIEEE(data)
-	if serial != s.serial {
-		s.mu.RUnlock()
-		err := s.updateCache()
-		s.mu.RLock()
-		if err != nil {
-			return Account{}, fmt.Errorf("refresh iam cache: %w", err)
-		}
-	}
-
-	acct, ok := s.accts.AccessAccounts[access]
+	acct, ok := conf.AccessAccounts[access]
 	if !ok {
 		return Account{}, ErrNoSuchUser
 	}
@@ -124,47 +99,13 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
-// updateCache must be called with no locks held
-func (s *IAMServiceInternal) updateCache() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	data, err := s.storer.GetIAM()
-	if err != nil {
-		return fmt.Errorf("get iam data: %w", err)
-	}
-
-	serial := crc32.ChecksumIEEE(data)
-
-	if len(data) > 0 {
-		if err := json.Unmarshal(data, &s.accts); err != nil {
-			return fmt.Errorf("failed to parse the config file: %w", err)
-		}
-	} else {
-		s.accts.AccessAccounts = make(map[string]Account)
-	}
-
-	s.serial = serial
-
-	return nil
-}
-
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
-		if len(data) == 0 {
-			// empty config, do nothing
-			return data, nil
-		}
-
-		var conf IAMConfig
-
-		if err := json.Unmarshal(data, &conf); err != nil {
-			return nil, fmt.Errorf("failed to parse iam: %w", err)
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
 		}
 
 		delete(conf.AccessAccounts, access)
@@ -174,8 +115,221 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 			return nil, fmt.Errorf("failed to serialize iam: %w", err)
 		}
 
-		s.accts = conf
-
 		return b, nil
 	})
 }
+
+// ListUserAccounts lists all the user accounts stored.
+func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getIAM()
+	if err != nil {
+		return []Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// Shutdown graceful termination of service
+func (s *IAMServiceInternal) Shutdown() error {
+	return nil
+}
+
+const (
+	iamMode = 0600
+)
+
+func (s *IAMServiceInternal) initIAM() error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	_, err := os.ReadFile(fname)
+	if errors.Is(err, fs.ErrNotExist) {
+		b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
+		if err != nil {
+			return fmt.Errorf("marshal default iam: %w", err)
+		}
+		err = os.WriteFile(fname, b, iamMode)
+		if err != nil {
+			return fmt.Errorf("write default iam: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
+	b, err := s.readIAMData()
+	if err != nil {
+		return iAMConfig{}, err
+	}
+
+	return parseIAM(b)
+}
+
+func parseIAM(b []byte) (iAMConfig, error) {
+	var conf iAMConfig
+	if err := json.Unmarshal(b, &conf); err != nil {
+		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
+	}
+
+	return conf, nil
+}
+
+const (
+	backoff  = 100 * time.Millisecond
+	maxretry = 300
+)
+
+func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
+	// We are going to be racing with other running gateways without any
+	// coordination. So we might find the file does not exist at times.
+	// For this case we need to retry for a while assuming the other gateway
+	// will eventually write the file. If it doesn't after the max retries,
+	// then we will return the error.
+
+	retries := 0
+
+	for {
+		b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+			return nil, fmt.Errorf("read iam file: %w", err)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		return b, nil
+	}
+}
+
+func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
+	// We are going to be racing with other running gateways without any
+	// coordination. So the strategy here is to read the current file data.
+	// If the file doesn't exist, then we assume someone else is currently
+	// updating the file. So we just need to keep retrying. We also need
+	// to make sure the data is consistent within a single update. So racing
+	// writes to a file would possibly leave this in some invalid state.
+	// We can get atomic updates with rename. If we read the data, update
+	// the data, write to a temp file, then rename the tempfile back to the
+	// data file. This should always result in a complete data image.
+
+	// There is at least one unsolved failure mode here.
+	// If a gateway removes the data file and then crashes, all other
+	// gateways will retry forever thinking that the original will eventually
+	// write the file.
+
+	retries := 0
+	fname := filepath.Join(s.dir, iamFile)
+
+	for {
+		b, err := os.ReadFile(fname)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+
+			// we have been unsuccessful trying to read the iam file
+			// so this must be the case where something happened and
+			// the file did not get updated successfully, and probably
+			// isn't going to be. The recovery procedure would be to
+			// copy the backup file into place of the original.
+			return fmt.Errorf("no iam file, needs backup recovery")
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("read iam file: %w", err)
+		}
+
+		// reset retries on successful read
+		retries = 0
+
+		err = os.Remove(iamFile)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			time.Sleep(backoff)
+			continue
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("remove old iam file: %w", err)
+		}
+
+		// save copy of data
+		datacopy := make([]byte, len(b))
+		copy(datacopy, b)
+
+		// make a backup copy in case we crash before update
+		// this is after remove, so there is a small window something
+		// can go wrong, but the remove should barrier other gateways
+		// from trying to write backup at the same time. Only one
+		// gateway will successfully remove the file.
+		os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
+
+		b, err = update(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return fmt.Errorf("update iam data: %w", err)
+		}
+
+		err = s.writeTempFile(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return err
+		}
+
+		break
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) writeTempFile(b []byte) error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	f, err := os.CreateTemp(s.dir, iamFile)
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	_, err = f.Write(b)
+	if err != nil {
+		return fmt.Errorf("write temp file: %w", err)
+	}
+
+	err = os.Rename(f.Name(), fname)
+	if err != nil {
+		return fmt.Errorf("rename temp file: %w", err)
+	}
+
+	return nil
+}

auth/iam_ldap.go

@@ -0,0 +1,133 @@
+package auth
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-ldap/ldap/v3"
+)
+
+type LdapIAMService struct {
+	conn       *ldap.Conn
+	queryBase  string
+	objClasses []string
+	accessAtr  string
+	secretAtr  string
+	roleAtr    string
+}
+
+var _ IAMService = &LdapIAMService{}
+
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+		return nil, fmt.Errorf("required parameters list not fully provided")
+	}
+	conn, err := ldap.Dial("tcp", url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
+	}
+
+	err = conn.Bind(bindDN, pass)
+	if err != nil {
+		return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
+	}
+	return &LdapIAMService{
+		conn:       conn,
+		queryBase:  queryBase,
+		objClasses: strings.Split(objClasses, ","),
+		accessAtr:  accAtr,
+		secretAtr:  secAtr,
+		roleAtr:    roleAtr,
+	}, nil
+}
+
+func (ld *LdapIAMService) CreateAccount(account Account) error {
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry.Attribute("objectClass", ld.objClasses)
+	userEntry.Attribute(ld.accessAtr, []string{account.Access})
+	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
+	userEntry.Attribute(ld.roleAtr, []string{account.Role})
+
+	err := ld.conn.Add(userEntry)
+	if err != nil {
+		return fmt.Errorf("error adding an entry: %w", err)
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	result, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return Account{}, err
+	}
+
+	entry := result.Entries[0]
+	return Account{
+		Access: entry.GetAttributeValue(ld.accessAtr),
+		Secret: entry.GetAttributeValue(ld.secretAtr),
+		Role:   entry.GetAttributeValue(ld.roleAtr),
+	}, nil
+}
+
+func (ld *LdapIAMService) DeleteUserAccount(access string) error {
+	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+
+	err := ld.conn.Del(delReq)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
+	searchFilter := ""
+	for _, el := range ld.objClasses {
+		searchFilter += fmt.Sprintf("(objectClass=%v)", el)
+	}
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(&%v)", searchFilter),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	resp, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	result := []Account{}
+	for _, el := range resp.Entries {
+		result = append(result, Account{
+			Access: el.GetAttributeValue(ld.accessAtr),
+			Secret: el.GetAttributeValue(ld.secretAtr),
+			Role:   el.GetAttributeValue(ld.roleAtr),
+		})
+	}
+
+	return result, nil
+}
+
+// Shutdown graceful termination of service
+func (ld *LdapIAMService) Shutdown() error {
+	return ld.conn.Close()
+}

auth/iam_single.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+)
+
+// IAMServiceSingle manages the single tenant (root-only) IAM service
+type IAMServiceSingle struct{}
+
+var _ IAMService = &IAMServiceSingle{}
+
+var ErrNotSupported = errors.New("method is not supported")
+
+// CreateAccount not valid in single tenant mode
+func (IAMServiceSingle) CreateAccount(account Account) error {
+	return ErrNotSupported
+}
+
+// GetUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	return Account{}, ErrNotSupported
+}
+
+// DeleteUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) DeleteUserAccount(access string) error {
+	return ErrNotSupported
+}
+
+// ListUserAccounts no accounts in single tenant mode
+func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
+	return []Account{}, nil
+}
+
+// Shutdown graceful termination of service
+func (IAMServiceSingle) Shutdown() error {
+	return nil
+}

backend/backend.go

@@ -15,6 +15,7 @@
 package backend
 
 import (
+	"context"
 	"fmt"
 	"io"
 
@@ -28,37 +29,48 @@ type Backend interface {
 	fmt.Stringer
 	Shutdown()
 
-	ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error)
-	HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
-	GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error)
-	CreateBucket(*s3.CreateBucketInput) error
-	PutBucketAcl(bucket string, data []byte) error
-	DeleteBucket(*s3.DeleteBucketInput) error
+	// bucket operations
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
+	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
+	CreateBucket(context.Context, *s3.CreateBucketInput) error
+	PutBucketAcl(_ context.Context, bucket string, data []byte) error
+	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
 
-	CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
-	CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
-	AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
-	ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
-	ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error)
-	UploadPart(*s3.UploadPartInput) (etag string, err error)
-	UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
+	// multipart operations
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
+	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
+	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
+	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
+	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
 
-	PutObject(*s3.PutObjectInput) (string, error)
-	HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
-	GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
-	CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
-	ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
-	ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
-	DeleteObject(*s3.DeleteObjectInput) error
-	DeleteObjects(*s3.DeleteObjectsInput) error
-	PutObjectAcl(*s3.PutObjectAclInput) error
-	RestoreObject(*s3.RestoreObjectInput) error
+	// standard object operations
+	PutObject(context.Context, *s3.PutObjectInput) (string, error)
+	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
+	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
 
-	GetTags(bucket, object string) (map[string]string, error)
-	SetTags(bucket, object string, tags map[string]string) error
-	RemoveTags(bucket, object string) error
+	// special case object operations
+	RestoreObject(context.Context, *s3.RestoreObjectInput) error
+	SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
+
+	// object tags operations
+	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
+	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
+	DeleteObjectTagging(_ context.Context, bucket, object string) error
+
+	// non AWS actions
+	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
 type BackendUnsupported struct{}
@@ -72,90 +84,101 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets(string, bool) (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error {
+func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
+func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) RestoreObject(*s3.RestoreObjectInput) error {
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
 	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) CreateBucket(*s3.CreateBucketInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteBucket(*s3.DeleteBucketInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
 
-func (BackendUnsupported) CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) AbortMultipartUpload(*s3.AbortMultipartUploadInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
-	return s3response.ListMultipartUploadsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error) {
-	return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) UploadPart(*s3.UploadPartInput) (etag string, err error) {
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
 	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-
-func (BackendUnsupported) PutObject(*s3.PutObjectInput) (string, error) {
-	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObject(*s3.DeleteObjectInput) error {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObjects(*s3.DeleteObjectsInput) error {
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
 
-func (BackendUnsupported) GetTags(bucket, object string) (map[string]string, error) {
+func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
+	return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) SetTags(bucket, object string, tags map[string]string) error {
+func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) RemoveTags(bucket, object string) error {
+func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
+	return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}

backend/common.go

@@ -56,14 +56,14 @@ func GetTimePtr(t time.Time) *time.Time {
 }
 
 var (
-	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRequest)
+	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
 )
 
 // ParseRange parses input range header and returns startoffset, length, and
 // error. If no endoffset specified, then length is set to -1.
-func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
+func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, file.Size(), nil
+		return 0, fi.Size(), nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")

backend/s3proxy/client.go

@@ -0,0 +1,86 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+	"github.com/versity/versitygw/auth"
+)
+
+func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		return nil, fmt.Errorf("invalid account in context")
+	}
+
+	cfg, err := s.getConfig(ctx, acct.Access, acct.Secret)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg), nil
+}
+
+func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.disableChecksum {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(ctx, opts...)
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *S3be) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}

backend/s3proxy/s3.go

@@ -0,0 +1,500 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3response"
+)
+
+type S3be struct {
+	backend.BackendUnsupported
+
+	endpoint        string
+	awsRegion       string
+	disableChecksum bool
+	sslSkipVerify   bool
+	debug           bool
+}
+
+func New(endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
+	return &S3be{
+		endpoint:        endpoint,
+		awsRegion:       region,
+		disableChecksum: disableChecksum,
+		sslSkipVerify:   sslSkipVerify,
+		debug:           debug,
+	}
+}
+
+func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, err
+	}
+
+	output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, err
+	}
+
+	var buckets []s3response.ListAllMyBucketsEntry
+	for _, b := range output.Buckets {
+		buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+			Name:         *b.Name,
+			CreationDate: *b.CreationDate,
+		})
+	}
+
+	return s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: buckets,
+		},
+	}, nil
+}
+
+func (s *S3be) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.HeadBucket(ctx, input)
+}
+
+func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.CreateBucket(ctx, input)
+	return err
+}
+
+func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteBucket(ctx, input)
+	return err
+}
+
+func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.CreateMultipartUpload(ctx, input)
+}
+
+func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.CompleteMultipartUpload(ctx, input)
+}
+
+func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.AbortMultipartUpload(ctx, input)
+	return err
+}
+
+const (
+	iso8601Format = "20060102T150405Z"
+)
+
+func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, err
+	}
+
+	output, err := client.ListMultipartUploads(ctx, input)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, err
+	}
+
+	var uploads []s3response.Upload
+	for _, u := range output.Uploads {
+		uploads = append(uploads, s3response.Upload{
+			Key:      *u.Key,
+			UploadID: *u.UploadId,
+			Initiator: s3response.Initiator{
+				ID:          *u.Initiator.ID,
+				DisplayName: *u.Initiator.DisplayName,
+			},
+			Owner: s3response.Owner{
+				ID:          *u.Owner.ID,
+				DisplayName: *u.Owner.DisplayName,
+			},
+			StorageClass: string(u.StorageClass),
+			Initiated:    u.Initiated.Format(iso8601Format),
+		})
+	}
+
+	var cps []s3response.CommonPrefix
+	for _, c := range output.CommonPrefixes {
+		cps = append(cps, s3response.CommonPrefix{
+			Prefix: *c.Prefix,
+		})
+	}
+
+	return s3response.ListMultipartUploadsResult{
+		Bucket:             *output.Bucket,
+		KeyMarker:          *output.KeyMarker,
+		UploadIDMarker:     *output.UploadIdMarker,
+		NextKeyMarker:      *output.NextKeyMarker,
+		NextUploadIDMarker: *output.NextUploadIdMarker,
+		Delimiter:          *output.Delimiter,
+		Prefix:             *output.Prefix,
+		EncodingType:       string(output.EncodingType),
+		MaxUploads:         int(output.MaxUploads),
+		IsTruncated:        output.IsTruncated,
+		Uploads:            uploads,
+		CommonPrefixes:     cps,
+	}, nil
+}
+
+func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListPartsResult{}, err
+	}
+
+	output, err := client.ListParts(ctx, input)
+	if err != nil {
+		return s3response.ListPartsResult{}, err
+	}
+
+	var parts []s3response.Part
+	for _, p := range output.Parts {
+		parts = append(parts, s3response.Part{
+			PartNumber:   int(p.PartNumber),
+			LastModified: p.LastModified.Format(iso8601Format),
+			ETag:         *p.ETag,
+			Size:         p.Size,
+		})
+	}
+	pnm, err := strconv.Atoi(*output.PartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse part number marker: %w", err)
+	}
+
+	npmn, err := strconv.Atoi(*output.NextPartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse next part number marker: %w", err)
+	}
+
+	return s3response.ListPartsResult{
+		Bucket:   *output.Bucket,
+		Key:      *output.Key,
+		UploadID: *output.UploadId,
+		Initiator: s3response.Initiator{
+			ID:          *output.Initiator.ID,
+			DisplayName: *output.Initiator.DisplayName,
+		},
+		Owner: s3response.Owner{
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
+		},
+		StorageClass:         string(output.StorageClass),
+		PartNumberMarker:     pnm,
+		NextPartNumberMarker: npmn,
+		MaxParts:             int(output.MaxParts),
+		IsTruncated:          output.IsTruncated,
+		Parts:                parts,
+	}, nil
+}
+
+func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	output, err := client.UploadPart(ctx, input)
+	if err != nil {
+		return "", err
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.CopyObjectResult{}, err
+	}
+
+	output, err := client.UploadPartCopy(ctx, input)
+	if err != nil {
+		return s3response.CopyObjectResult{}, err
+	}
+
+	return s3response.CopyObjectResult{
+		LastModified: *output.CopyPartResult.LastModified,
+		ETag:         *output.CopyPartResult.ETag,
+	}, nil
+}
+
+func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	output, err := client.PutObject(ctx, input)
+	if err != nil {
+		return "", err
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3be) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.HeadObject(ctx, input)
+}
+
+func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetObject(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+	defer output.Body.Close()
+
+	_, err = io.Copy(w, output.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}
+
+func (s *S3be) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.GetObjectAttributes(ctx, input)
+}
+
+func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.CopyObject(ctx, input)
+}
+
+func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.ListObjects(ctx, input)
+}
+
+func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.ListObjectsV2(ctx, input)
+}
+
+func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteObject(ctx, input)
+	return err
+}
+
+func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, err
+	}
+
+	output, err := client.DeleteObjects(ctx, input)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, err
+	}
+
+	return s3response.DeleteObjectsResult{
+		Deleted: output.Deleted,
+		Error:   output.Errors,
+	}, nil
+}
+
+func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetBucketAcl(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	var acl auth.ACL
+
+	acl.Owner = *output.Owner.ID
+	for _, el := range output.Grants {
+		acl.Grantees = append(acl.Grantees, auth.Grantee{
+			Permission: el.Permission,
+			Access:     *el.Grantee.ID,
+		})
+	}
+
+	return json.Marshal(acl)
+}
+
+func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	acl, err := auth.ParseACL(data)
+	if err != nil {
+		return err
+	}
+
+	input := &s3.PutBucketAclInput{
+		Bucket: &bucket,
+		ACL:    acl.ACL,
+		AccessControlPolicy: &types.AccessControlPolicy{
+			Owner: &types.Owner{
+				ID: &acl.Owner,
+			},
+		},
+	}
+
+	for _, el := range acl.Grantees {
+		input.AccessControlPolicy.Grants = append(input.AccessControlPolicy.Grants, types.Grant{
+			Permission: el.Permission,
+			Grantee: &types.Grantee{
+				ID:   &el.Access,
+				Type: types.TypeCanonicalUser,
+			},
+		})
+	}
+
+	_, err = client.PutBucketAcl(ctx, input)
+	return err
+}
+
+func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	tagging := &types.Tagging{
+		TagSet: []types.Tag{},
+	}
+	for key, val := range tags {
+		tagging.TagSet = append(tagging.TagSet, types.Tag{
+			Key:   &key,
+			Value: &val,
+		})
+	}
+
+	_, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
+		Bucket:  &bucket,
+		Key:     &object,
+		Tagging: tagging,
+	})
+	return err
+}
+
+func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	tags := make(map[string]string)
+	for _, el := range output.TagSet {
+		tags[*el.Key] = *el.Value
+	}
+
+	return tags, nil
+}
+
+func (s *S3be) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	return err
+}

backend/scoutfs/scoutfs.go

@@ -15,6 +15,7 @@
 package scoutfs
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/json"
 	"errors"
@@ -29,7 +30,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
-	"github.com/versity/scoutfs-go"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
@@ -114,7 +114,7 @@ func (*ScoutFS) String() string {
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
-func (s *ScoutFS) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
@@ -165,7 +165,9 @@ func (s *ScoutFS) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput
 		if err != nil {
 			etag = ""
 		}
-		parts[i].ETag = &etag
+		if etag != *parts[i].ETag {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
 	}
 
 	// use totalsize=0 because we wont be writing to the file, only moving
@@ -185,7 +187,7 @@ func (s *ScoutFS) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = scoutfs.MoveData(pf, f.f)
+		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
 			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
@@ -352,7 +354,7 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
 	return nil
 }
 
-func (s *ScoutFS) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 
@@ -389,7 +391,7 @@ func (s *ScoutFS) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, e
 
 		// Check if there are any offline exents associated with this file.
 		// If so, we will set storage class to glacier.
-		st, err := scoutfs.StatMore(objPath)
+		st, err := statMore(objPath)
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		}
@@ -425,7 +427,7 @@ func (s *ScoutFS) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, e
 	}, nil
 }
 
-func (s *ScoutFS) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	acceptRange := *input.Range
@@ -463,7 +465,7 @@ func (s *ScoutFS) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.Get
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
-		st, err := scoutfs.StatMore(objPath)
+		st, err := statMore(objPath)
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		}
@@ -539,7 +541,7 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
 	return tags, nil
 }
 
-func (s *ScoutFS) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 	bucket := *input.Bucket
 	prefix := *input.Prefix
 	marker := *input.Marker
@@ -574,7 +576,7 @@ func (s *ScoutFS) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput
 	}, nil
 }
 
-func (s *ScoutFS) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
 	bucket := *input.Bucket
 	prefix := *input.Prefix
 	marker := *input.ContinuationToken
@@ -663,7 +665,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if s.glaciermode {
 			// Check if there are any offline exents associated with this file.
 			// If so, we will return the InvalidObjectState error.
-			st, err := scoutfs.StatMore(objPath)
+			st, err := statMore(objPath)
 			if errors.Is(err, fs.ErrNotExist) {
 				return types.Object{}, backend.ErrSkipObj
 			}
@@ -687,7 +689,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 
 // RestoreObject will set stage request on file if offline and do nothing if
 // file is online
-func (s *ScoutFS) RestoreObject(input *s3.RestoreObjectInput) error {
+func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
 	bucket := *input.Bucket
 	object := *input.Key
 

backend/scoutfs/scoutfs_compat.go

@@ -12,6 +12,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build linux && amd64
+
 package scoutfs
 
 import (
@@ -26,6 +28,7 @@ import (
 
 	"golang.org/x/sys/unix"
 
+	"github.com/versity/scoutfs-go"
 	"github.com/versity/versitygw/backend/posix"
 )
 
@@ -182,3 +185,25 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
+
+func moveData(from *os.File, to *os.File) error {
+	return scoutfs.MoveData(from, to)
+}
+
+func statMore(path string) (stat, error) {
+	st, err := scoutfs.StatMore(path)
+	if err != nil {
+		return stat{}, err
+	}
+	var s stat
+
+	s.Meta_seq = st.Meta_seq
+	s.Data_seq = st.Data_seq
+	s.Data_version = st.Data_version
+	s.Online_blocks = st.Online_blocks
+	s.Offline_blocks = st.Offline_blocks
+	s.Crtime_sec = st.Crtime_sec
+	s.Crtime_nsec = st.Crtime_nsec
+
+	return s, nil
+}

backend/scoutfs/scoutfs_incompat.go

@@ -12,6 +12,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build !(linux && amd64)
+
 package scoutfs
 
 import (
@@ -46,3 +48,11 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 
 func (tmp *tmpfile) cleanup() {
 }
+
+func moveData(from *os.File, to *os.File) error {
+	return errNotSupported
+}
+
+func statMore(path string) (stat, error) {
+	return stat{}, errNotSupported
+}

backend/scoutfs/stat.go

@@ -14,35 +14,12 @@
 
 package scoutfs
 
-import (
-	"errors"
-	"fmt"
-	"os"
-)
-
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
-	return nil, fmt.Errorf("scoutfs only available on linux")
-}
-
-type tmpfile struct {
-	f *os.File
-}
-
-var (
-	errNotSupported = errors.New("not supported")
-)
-
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
-	return nil, errNotSupported
-}
-
-func (tmp *tmpfile) link() error {
-	return errNotSupported
-}
-
-func (tmp *tmpfile) Write(b []byte) (int, error) {
-	return 0, errNotSupported
-}
-
-func (tmp *tmpfile) cleanup() {
+type stat struct {
+	Meta_seq       uint64
+	Data_seq       uint64
+	Data_version   uint64
+	Online_blocks  uint64
+	Offline_blocks uint64
+	Crtime_sec     uint64
+	Crtime_nsec    uint32
 }

backend/walk.go

@@ -47,7 +47,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		pastMarker = true
 	}
 
-	var pastMax bool
+	pastMax := max == 0
 	var newMarker string
 	var truncated bool
 
@@ -55,30 +55,30 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		if err != nil {
 			return err
 		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
 
 		if pastMax {
-			newMarker = path
-			truncated = true
+			if len(objects) != 0 {
+				newMarker = *objects[len(objects)-1].Key
+				truncated = true
+			}
 			return fs.SkipAll
 		}
 
 		if d.IsDir() {
-			// Ignore the root directory
-			if path == "." {
-				return nil
-			}
-
-			if contains(d.Name(), skipdirs) {
-				return fs.SkipDir
-			}
-
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
 			// match this prefix. Make sure to append the / at the end of
 			// directories since this is implied as a directory path name.
 			// If path is a prefix of prefix, then path could still be
-			// building to match. So only skip if path isnt a prefix of prefix
-			// and prefix isnt a prefix of path.
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
 			if prefix != "" &&
 				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
 				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
@@ -106,10 +106,13 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		}
 
 		if !pastMarker {
-			if path != marker {
+			if path == marker {
+				pastMarker = true
+				return nil
+			}
+			if path < marker {
 				return nil
 			}
-			pastMarker = true
 		}
 
 		// If object doesn't have prefix, don't include in results.

cmd/versitygw/admin.go

@@ -15,31 +15,35 @@
 package main
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"text/tabwriter"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
 )
 
 var (
-	adminAccess string
-	adminSecret string
+	adminAccess   string
+	adminSecret   string
+	adminEndpoint string
 )
 
 func adminCommand() *cli.Command {
 	return &cli.Command{
-		Name:  "admin",
-		Usage: "admin CLI tool",
-		Description: `admin CLI tool for interacting with admin api.
-		Here is the available api list:
-		create-user
-		`,
+		Name:        "admin",
+		Usage:       "admin CLI tool",
+		Description: `Admin CLI tool for interacting with admin APIs.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:   "create-user",
@@ -48,13 +52,13 @@ func adminCommand() *cli.Command {
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:     "access",
-						Usage:    "access value for the new user",
+						Usage:    "access key id for the new user",
 						Required: true,
 						Aliases:  []string{"a"},
 					},
 					&cli.StringFlag{
 						Name:     "secret",
-						Usage:    "secret value for the new user",
+						Usage:    "secret access key for the new user",
 						Required: true,
 						Aliases:  []string{"s"},
 					},
@@ -64,6 +68,21 @@ func adminCommand() *cli.Command {
 						Required: true,
 						Aliases:  []string{"r"},
 					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+					&cli.IntFlag{
+						Name:    "project-id",
+						Usage:   "projectID for the new user",
+						Aliases: []string{"pi"},
+					},
 				},
 			},
 			{
@@ -73,20 +92,50 @@ func adminCommand() *cli.Command {
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:     "access",
-						Usage:    "access value for the user to be deleted",
+						Usage:    "access key id of the user to be deleted",
 						Required: true,
 						Aliases:  []string{"a"},
 					},
 				},
 			},
+			{
+				Name:   "list-users",
+				Usage:  "List all the gateway users",
+				Action: listUsers,
+			},
+			{
+				Name:  "change-bucket-owner",
+				Usage: "Changes the bucket owner",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "bucket",
+						Usage:    "the bucket name to change the owner",
+						Required: true,
+						Aliases:  []string{"b"},
+					},
+					&cli.StringFlag{
+						Name:     "owner",
+						Usage:    "the user access key id, who should be the bucket owner",
+						Required: true,
+						Aliases:  []string{"o"},
+					},
+				},
+				Action: changeBucketOwner,
+			},
+			{
+				Name:   "list-buckets",
+				Usage:  "Lists all the gateway buckets and owners.",
+				Action: listBuckets,
+			},
 		},
 		Flags: []cli.Flag{
 			// TODO: create a configuration file for this
 			&cli.StringFlag{
 				Name:        "access",
-				Usage:       "admin access account",
+				Usage:       "admin access key id",
 				EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
 				Aliases:     []string{"a"},
+				Required:    true,
 				Destination: &adminAccess,
 			},
 			&cli.StringFlag{
@@ -94,14 +143,24 @@ func adminCommand() *cli.Command {
 				Usage:       "admin secret access key",
 				EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
 				Aliases:     []string{"s"},
+				Required:    true,
 				Destination: &adminSecret,
 			},
+			&cli.StringFlag{
+				Name:        "endpoint-url",
+				Usage:       "admin apis endpoint url",
+				EnvVars:     []string{"ADMIN_ENDPOINT_URL"},
+				Aliases:     []string{"er"},
+				Required:    true,
+				Destination: &adminEndpoint,
+			},
 		},
 	}
 }
 
 func createUser(ctx *cli.Context) error {
 	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
+	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
 	if access == "" || secret == "" {
 		return fmt.Errorf("invalid input parameters for the new user")
 	}
@@ -109,14 +168,28 @@ func createUser(ctx *cli.Context) error {
 		return fmt.Errorf("invalid input parameter for role")
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:7070/create-user?access=%v&secret=%v&role=%v", access, secret, role), nil)
+	acc := auth.Account{
+		Access:    access,
+		Secret:    secret,
+		Role:      role,
+		UserID:    userID,
+		GroupID:   groupID,
+		ProjectID: projectID,
+	}
+
+	accJson, err := json.Marshal(acc)
+	if err != nil {
+		return fmt.Errorf("failed to parse user data: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256([]byte{})
+	hashedPayload := sha256.Sum256(accJson)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
@@ -137,6 +210,7 @@ func createUser(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	fmt.Printf("%s\n", body)
 
@@ -149,7 +223,7 @@ func deleteUser(ctx *cli.Context) error {
 		return fmt.Errorf("invalid input parameter for the new user")
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:7070/delete-user?access=%v", access), nil)
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
@@ -177,8 +251,166 @@ func deleteUser(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	fmt.Printf("%s\n", body)
 
 	return nil
 }
+
+func listUsers(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
+		return err
+	}
+	fmt.Println(accs)
+
+	printAcctTable(accs)
+
+	return nil
+}
+
+const (
+	// account table formatting
+	minwidth int  = 2   // minimal cell width including any padding
+	tabwidth int  = 0   // width of tab characters (equivalent number of spaces)
+	padding  int  = 2   // padding added to a cell before computing its width
+	padchar  byte = ' ' // ASCII char used for padding
+	flags    uint = 0   // formatting control flags
+)
+
+func printAcctTable(accs []auth.Account) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
+	for _, acc := range accs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func changeBucketOwner(ctx *cli.Context) error {
+	bucket, owner := ctx.String("bucket"), ctx.String("owner")
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Println(string(body))
+
+	return nil
+}
+
+func printBuckets(buckets []s3response.Bucket) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Bucket\tOwner")
+	fmt.Fprintln(w, "-------\t----")
+	for _, acc := range buckets {
+		fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func listBuckets(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf(string(body))
+	}
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return err
+	}
+
+	printBuckets(buckets)
+
+	return nil
+}

cmd/versitygw/main.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strconv"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
@@ -32,16 +33,24 @@ import (
 )
 
 var (
-	port                           string
-	rootUserAccess                 string
-	rootUserSecret                 string
-	region                         string
-	certFile, keyFile              string
-	kafkaURL, kafkaTopic, kafkaKey string
-	natsURL, natsTopic             string
-	logWebhookURL                  string
-	accessLog                      bool
-	debug                          bool
+	port, admPort                          string
+	rootUserAccess                         string
+	rootUserSecret                         string
+	region                                 string
+	admCertFile, admKeyFile                string
+	certFile, keyFile                      string
+	kafkaURL, kafkaTopic, kafkaKey         string
+	natsURL, natsTopic                     string
+	logWebhookURL                          string
+	accessLog                              string
+	debug                                  bool
+	iamDir                                 string
+	ldapURL, ldapBindDN, ldapPassword      string
+	ldapQueryBase, ldapObjClasses          string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	iamCacheDisable                        bool
+	iamCacheTTL                            int
+	iamCachePrune                          int
 )
 
 var (
@@ -61,6 +70,7 @@ func main() {
 	app.Commands = []*cli.Command{
 		posixCommand(),
 		scoutfsCommand(),
+		s3Command(),
 		adminCommand(),
 		testCommand(),
 	}
@@ -142,19 +152,37 @@ func initFlags() []cli.Flag {
 			Usage:       "TLS key file",
 			Destination: &keyFile,
 		},
+		&cli.StringFlag{
+			Name:        "admin-port",
+			Usage:       "gateway admin server listen address <ip>:<port> or :<port>",
+			Destination: &admPort,
+			Aliases:     []string{"ap"},
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert",
+			Usage:       "TLS cert file for admin server",
+			Destination: &admCertFile,
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert-key",
+			Usage:       "TLS key file for admin server",
+			Destination: &admKeyFile,
+		},
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
 			Destination: &debug,
 		},
-		&cli.BoolFlag{
+		&cli.StringFlag{
 			Name:        "access-log",
-			Usage:       "enable server access logging in the root directory",
+			Usage:       "enable server access logging to specified file",
+			EnvVars:     []string{"LOGFILE"},
 			Destination: &accessLog,
 		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
+			EnvVars:     []string{"WEBHOOK"},
 			Destination: &logWebhookURL,
 		},
 		&cli.StringFlag{
@@ -187,14 +215,83 @@ func initFlags() []cli.Flag {
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
+		&cli.StringFlag{
+			Name:        "iam-dir",
+			Usage:       "if defined, run internal iam service within this directory",
+			Destination: &iamDir,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-url",
+			Usage:       "ldap server url to store iam data",
+			Destination: &ldapURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-dn",
+			Usage:       "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
+			Destination: &ldapBindDN,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-pass",
+			Usage:       "ldap server user password",
+			Destination: &ldapPassword,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-query-base",
+			Usage:       "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
+			Destination: &ldapQueryBase,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-object-classes",
+			Usage:       "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
+			Destination: &ldapObjClasses,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-access-atr",
+			Usage:       "ldap server user access key id attribute name",
+			Destination: &ldapAccessAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-secret-atr",
+			Usage:       "ldap server user secret access key attribute name",
+			Destination: &ldapSecAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-role-atr",
+			Usage:       "ldap server user role attribute name",
+			Destination: &ldapRoleAtr,
+		},
+		&cli.BoolFlag{
+			Name:        "iam-cache-disable",
+			Usage:       "disable local iam cache",
+			Destination: &iamCacheDisable,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-ttl",
+			Usage:       "local iam cache entry ttl (seconds)",
+			Value:       120,
+			Destination: &iamCacheTTL,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-prune",
+			Usage:       "local iam cache cleanup interval (seconds)",
+			Value:       3600,
+			Destination: &iamCachePrune,
+		},
 	}
 }
 
-func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
+func runGateway(ctx *cli.Context, be backend.Backend) error {
+	// int32 max for 32 bit arch
+	blimit := int64(2*1024*1024*1024 - 1)
+	if strconv.IntSize > 32 {
+		// 5GB max for 64 bit arch
+		blimit = int64(5 * 1024 * 1024 * 1024)
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:      "versitygw",
 		ServerHeader: "VERSITYGW",
-		BodyLimit:    5 * 1024 * 1024 * 1024,
+		BodyLimit:    int(blimit),
 	})
 
 	var opts []s3api.Option
@@ -213,23 +310,55 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
 		}
 		opts = append(opts, s3api.WithTLS(cert))
 	}
-
 	if debug {
 		opts = append(opts, s3api.WithDebug())
 	}
-
-	err := s.InitIAM()
-	if err != nil {
-		return fmt.Errorf("init iam: %w", err)
+	if admPort == "" {
+		opts = append(opts, s3api.WithAdminServer())
 	}
 
-	iam, err := auth.NewInternal(s)
+	admApp := fiber.New(fiber.Config{
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+	})
+
+	var admOpts []s3api.AdminOpt
+
+	if admCertFile != "" || admKeyFile != "" {
+		if admCertFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if admKeyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert))
+	}
+
+	iam, err := auth.New(&auth.Opts{
+		Dir:            iamDir,
+		LDAPServerURL:  ldapURL,
+		LDAPBindDN:     ldapBindDN,
+		LDAPPassword:   ldapPassword,
+		LDAPQueryBase:  ldapQueryBase,
+		LDAPObjClasses: ldapObjClasses,
+		LDAPAccessAtr:  ldapAccessAtr,
+		LDAPSecretAtr:  ldapSecAtr,
+		LDAPRoleAtr:    ldapRoleAtr,
+		CacheDisable:   iamCacheDisable,
+		CacheTTL:       iamCacheTTL,
+		CachePrune:     iamCachePrune,
+	})
 	if err != nil {
-		return fmt.Errorf("setup internal iam service: %w", err)
+		return fmt.Errorf("setup iam: %w", err)
 	}
 
 	logger, err := s3log.InitLogger(&s3log.LogConfig{
-		IsFile:     accessLog,
+		LogFile:    accessLog,
 		WebhookURL: logWebhookURL,
 	})
 	if err != nil {
@@ -255,15 +384,48 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	c := make(chan error, 1)
-	go func() { c <- srv.Serve() }()
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
 
-	select {
-	case <-ctx.Done():
-		be.Shutdown()
-		return ctx.Err()
-	case err := <-c:
-		be.Shutdown()
-		return err
+	c := make(chan error, 2)
+	go func() { c <- srv.Serve() }()
+	if admPort != "" {
+		go func() { c <- admSrv.Serve() }()
 	}
+
+	// for/select blocks until shutdown
+Loop:
+	for {
+		select {
+		case <-ctx.Done():
+			err = ctx.Err()
+			break Loop
+		case err = <-c:
+			break Loop
+		case <-sigHup:
+			if logger != nil {
+				err = logger.HangUp()
+				if err != nil {
+					err = fmt.Errorf("HUP logger: %w", err)
+					break Loop
+				}
+			}
+		}
+	}
+	saveErr := err
+
+	be.Shutdown()
+
+	err = iam.Shutdown()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
+	}
+
+	if logger != nil {
+		err := logger.Shutdown()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
+		}
+	}
+
+	return saveErr
 }

cmd/versitygw/posix.go

@@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("init posix: %v", err)
 	}
 
-	return runGateway(ctx, be, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/s3.go

@@ -0,0 +1,76 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/s3proxy"
+)
+
+var (
+	s3proxyEndpoint        string
+	s3proxyRegion          string
+	s3proxyDisableChecksum bool
+	s3proxySslSkipVerify   bool
+	s3proxyDebug           bool
+)
+
+func s3Command() *cli.Command {
+	return &cli.Command{
+		Name:  "s3",
+		Usage: "s3 storage backend",
+		Description: `This runs the gateway like an s3 proxy redirecting requests
+to an s3 storage backend service.`,
+		Action: runS3,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "endpoint",
+				Usage:       "s3 service endpoint, default AWS if not specified",
+				Value:       "",
+				Destination: &s3proxyEndpoint,
+			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "s3 service region, default 'us-east-1' if not specified",
+				Value:       "us-east-1",
+				Destination: &s3proxyRegion,
+			},
+			&cli.BoolFlag{
+				Name:        "disable-checksum",
+				Usage:       "disable gateway to server object checksums",
+				Value:       false,
+				Destination: &s3proxyDisableChecksum,
+			},
+			&cli.BoolFlag{
+				Name:        "ssl-skip-verify",
+				Usage:       "skip ssl cert verification for s3 service",
+				Value:       false,
+				Destination: &s3proxySslSkipVerify,
+			},
+			&cli.BoolFlag{
+				Name:        "debug",
+				Usage:       "output extra debug tracing",
+				Value:       false,
+				Destination: &s3proxyDebug,
+			},
+		},
+	}
+}
+
+func runS3(ctx *cli.Context) error {
+	be := s3proxy.New(s3proxyEndpoint, s3proxyRegion,
+		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	return runGateway(ctx, be)
+}

cmd/versitygw/scoutfs.go

@@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}
 
-	return runGateway(ctx, be, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/signal.go

@@ -23,6 +23,7 @@ import (
 
 var (
 	sigDone = make(chan bool, 1)
+	sigHup  = make(chan bool, 1)
 )
 
 func setupSignalHandler() {
@@ -36,6 +37,7 @@ func setupSignalHandler() {
 			case syscall.SIGINT, syscall.SIGTERM:
 				sigDone <- true
 			case syscall.SIGHUP:
+				sigHup <- true
 			}
 		}
 	}()

cmd/versitygw/test.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"fmt"
+	"math"
+	"os"
+	"text/tabwriter"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/integration"
@@ -13,10 +16,12 @@ var (
 	endpoint        string
 	prefix          string
 	dstBucket       string
+	proxyURL        string
 	partSize        int64
 	objSize         int64
 	concurrency     int
 	files           int
+	totalReqs       int
 	upload          bool
 	download        bool
 	pathStyle       bool
@@ -67,120 +72,17 @@ func initTestFlags() []cli.Flag {
 
 func initTestCommands() []*cli.Command {
 	return []*cli.Command{
-		{
-			Name:  "bucket-actions",
-			Usage: "Test bucket creation, checking the existence, deletes it.",
-			Description: `Calls s3 gateway create-bucket action to create a new bucket,
-		calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`,
-			Action: getAction(integration.TestMakeBucket),
-		},
-		{
-			Name:  "object-actions",
-			Usage: "Test put/get/delete/copy objects.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying,
-			copies the object, get's the object to check the length and content, 
-			get's the copied object to check the length and content, deletes all the objects inside the source bucket, 
-			deletes both the objects and buckets.`,
-			Action: getAction(integration.TestPutGetObject),
-		},
-		{
-			Name:  "put-get-mp-object",
-			Usage: "Test put & get multipart object.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload,
-			gets the object from the bucket, deletes both the object and bucket.`,
-			Action: getAction(integration.TestPutGetMPObject),
-		},
-		{
-			Name:  "put-dir-object",
-			Usage: "Test put directory object.",
-			Description: `Creates a bucket with s3 gateway action, puts a directory object in it,
-			lists the bucket's objects, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestPutDirObject),
-		},
-		{
-			Name:  "list-objects",
-			Usage: "Test list-objects action.",
-			Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it,
-			lists the bucket's objects, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListObject),
-		},
-		{
-			Name:  "abort-mp",
-			Usage: "Tests abort-multipart-upload action.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			lists the multipart upload, aborts the multipart upload, lists the multipart upload again,
-			deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListAbortMultiPartObject),
-		},
-		{
-			Name:  "list-parts",
-			Usage: "Tests list-parts action.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			lists the upload parts, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListMultiParts),
-		},
-		{
-			Name:  "incorrect-mp",
-			Usage: "Tests incorrect multipart case.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			uploads different parts, completes the multipart upload with incorrect part numbers,
-			calls the head-object action, compares the content length, removes both the object and bucket`,
-			Action: getAction(integration.TestIncorrectMultiParts),
-		},
-		{
-			Name:  "incomplete-mp",
-			Usage: "Tests incomplete multi parts.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			upload a part, lists the parts, checks if the uploaded part is in the list, 
-			removes both the object and the bucket`,
-			Action: getAction(integration.TestIncompleteMultiParts),
-		},
-		{
-			Name:  "incomplete-put-object",
-			Usage: "Tests incomplete put objects case.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object with head-object action, expects the object to be got, 
-			removes both the object and bucket`,
-			Action: getAction(integration.TestIncompletePutObject),
-		},
-		{
-			Name:  "get-range",
-			Usage: "Tests get object by range.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object by specifying the object range, compares the range with the original one,
-			removes both the object and the bucket`,
-			Action: getAction(integration.TestRangeGet),
-		},
-		{
-			Name:  "invalid-mp",
-			Usage: "Tests invalid multi part case.",
-			Description: `Creates a bucket with s3 gateway action, creates a multi part upload,
-			uploads an invalid part, gets the object with head-object action, expects to get error,
-			removes both the object and bucket`,
-			Action: getAction(integration.TestInvalidMultiParts),
-		},
-		{
-			Name:  "object-tag-actions",
-			Usage: "Tests get/put/delete object tag actions.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			puts some tags for the object, gets the tags, compares the results, removes the tags,
-			gets the tags again, checks it to be empty, then removes both the object and bucket`,
-			Action: getAction(integration.TestPutGetRemoveTags),
-		},
-		{
-			Name:  "bucket-acl-actions",
-			Usage: "Tests put/get bucket actions.",
-			Description: `Creates a bucket with s3 gateway action, puts some bucket acls
-			gets the acl, verifies it, then removes the bucket`,
-			Action: getAction(integration.TestAclActions),
-		},
 		{
 			Name:        "full-flow",
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
 		},
+		{
+			Name:   "posix",
+			Usage:  "Tests posix specific features",
+			Action: getAction(integration.TestPosix),
+		},
 		{
 			Name:  "bench",
 			Usage: "Runs download/upload performance test on the gateway",
@@ -220,6 +122,7 @@ func initTestCommands() []*cli.Command {
 					Name:        "bucket",
 					Usage:       "Destination bucket name to read/write data",
 					Destination: &dstBucket,
+					Required:    true,
 				},
 				&cli.Int64Flag{
 					Name:        "partSize",
@@ -245,6 +148,11 @@ func initTestCommands() []*cli.Command {
 					Value:       false,
 					Destination: &checksumDisable,
 				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
 			},
 			Action: func(ctx *cli.Context) error {
 				if upload && download {
@@ -254,10 +162,6 @@ func initTestCommands() []*cli.Command {
 					return fmt.Errorf("must specify one of upload or download")
 				}
 
-				if dstBucket == "" {
-					return fmt.Errorf("must specify bucket")
-				}
-
 				opts := []integration.Option{
 					integration.WithAccess(awsID),
 					integration.WithSecret(awsSecret),
@@ -278,7 +182,123 @@ func initTestCommands() []*cli.Command {
 
 				s3conf := integration.NewS3Conf(opts...)
 
-				return integration.TestPerformance(s3conf, upload, download, files, objSize, dstBucket, prefix)
+				if upload {
+					if proxyURL == "" {
+						integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
+				} else {
+					if proxyURL == "" {
+						integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
+				}
+			},
+		},
+		{
+			Name:        "throughput",
+			Usage:       "Runs throughput performance test on the gateway",
+			Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "reqs",
+					Usage:       "Total number of requests to send.",
+					Value:       1000,
+					Destination: &totalReqs,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to make the requests",
+					Destination: &dstBucket,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "threads per request",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				if proxyURL == "" {
+					_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					return err
+				} else {
+					elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					opts = append(opts, integration.WithEndpoint(proxyURL))
+					s3proxy := integration.NewS3Conf(opts...)
+					proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
+					if err != nil || proxyErr != nil {
+						return nil
+					}
+
+					printProxyResultsTable([][4]string{
+						{"    #    ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
+						{"---------", "--------------", "----------", "----------------------------"},
+						{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
+						{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
+					})
+
+					return nil
+				}
 			},
 		},
 	}
@@ -309,3 +329,13 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		return nil
 	}
 }
+
+func printProxyResultsTable(stats [][4]string) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	for _, elem := range stats {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}

go.mod

@@ -3,61 +3,57 @@ module github.com/versity/versitygw
 go 1.20
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.18.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0
-	github.com/aws/smithy-go v1.13.5
-	github.com/gofiber/fiber/v2 v2.47.0
-	github.com/google/uuid v1.3.0
-	github.com/nats-io/nats.go v1.28.0
+	github.com/aws/aws-sdk-go-v2 v1.22.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
+	github.com/aws/smithy-go v1.16.0
+	github.com/go-ldap/ldap/v3 v3.4.6
+	github.com/gofiber/fiber/v2 v2.50.0
+	github.com/google/uuid v1.4.0
+	github.com/nats-io/nats.go v1.31.0
 	github.com/pkg/xattr v0.4.9
-	github.com/segmentio/kafka-go v0.4.42
+	github.com/segmentio/kafka-go v0.4.44
 	github.com/urfave/cli/v2 v2.25.7
-	github.com/valyala/fasthttp v1.48.0
+	github.com/valyala/fasthttp v1.50.0
 	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.10.0
+	golang.org/x/sys v0.14.0
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/nats-io/nats-server/v2 v2.9.20 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.27
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.26
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.24.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.15.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
-	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
-	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

go.sum

@@ -1,88 +1,85 @@
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
-github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
-github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA=
-github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71/go.mod h1:ZNo5H4PR3/fwsXYqb+Ld5YAfvHcYCbltaTTtSay4l2o=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w=
-github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE=
-github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg=
-github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
-github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
+github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
+github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
+github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
+github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
+github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gofiber/fiber/v2 v2.47.0 h1:EN5lHVCc+Pyqh5OEsk8fzRiifgwpbrP0rulQ4iNf3fs=
-github.com/gofiber/fiber/v2 v2.47.0/go.mod h1:mbFMVN1lQuzziTkkakgtKKdjfsXSw9BKR5lmcNksUoU=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
+github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
+github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
+github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
+github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
-github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
-github.com/nats-io/nats-server/v2 v2.9.20 h1:bt1dW6xsL1hWWwv7Hovm+EJt5L6iplyqlgEFkoEUk0k=
-github.com/nats-io/nats-server/v2 v2.9.20/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw=
-github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
-github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
-github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
-github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
+github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
+github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
+github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
-github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
-github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
 github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -92,13 +89,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
-github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
-github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
-github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
-github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
-github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU=
-github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg=
+github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4=
+github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -106,15 +98,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
-github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
-github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
 github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
 github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc=
-github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
+github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
@@ -127,68 +116,57 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

integration/bench.go

@@ -0,0 +1,142 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type prefResult struct {
+	elapsed time.Duration
+	size    int64
+	err     error
+}
+
+func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+	if objSize == 0 {
+		return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
+	}
+
+	if objSize > (int64(10000) * s.PartSize) {
+		return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
+	}
+
+	runF("performance test: upload objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			var r io.Reader = NewDataReader(int(objSize), int(s.PartSize))
+
+			start := time.Now()
+			err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = objSize
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed = time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			return 0, time.Since(start), res.err
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run upload: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return tot, time.Since(start), nil
+}
+
+func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+
+	runF("performance test: download objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			nw := NewNullWriter()
+			start := time.Now()
+			n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = n
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed = time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			return 0, elapsed, err
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run download: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return tot, elapsed, nil
+}
+
+func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
+	client := s3.NewFromConfig(s.Config())
+	var wg sync.WaitGroup
+	var resErr error
+
+	// Record the start time
+	startTime := time.Now()
+	runF("performance test: measuring request per second")
+
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < totalReqs/s.Concurrency; i++ {
+				_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket})
+				if err != nil && resErr != nil {
+					resErr = err
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	if resErr != nil {
+		failF("performance test failed with error: %w", resErr)
+		return time.Since(startTime), 0, resErr
+	}
+	elapsedTime := time.Since(startTime)
+	rps := int(float64(totalReqs) / elapsedTime.Seconds())
+
+	passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
+	return elapsedTime, rps, nil
+}

integration/group-tests.go

@@ -0,0 +1,224 @@
+package integration
+
+func TestAuthentication(s *S3Conf) {
+	Authentication_empty_auth_header(s)
+	Authentication_invalid_auth_header(s)
+	Authentication_unsupported_signature_version(s)
+	Authentication_malformed_credentials(s)
+	Authentication_malformed_credentials_invalid_parts(s)
+	Authentication_credentials_terminated_string(s)
+	Authentication_credentials_incorrect_service(s)
+	Authentication_credentials_incorrect_region(s)
+	Authentication_credentials_invalid_date(s)
+	Authentication_credentials_future_date(s)
+	Authentication_credentials_past_date(s)
+	Authentication_credentials_non_existing_access_key(s)
+	Authentication_invalid_signed_headers(s)
+	Authentication_missing_date_header(s)
+	Authentication_invalid_date_header(s)
+	Authentication_date_mismatch(s)
+	Authentication_incorrect_payload_hash(s)
+	Authentication_incorrect_md5(s)
+	Authentication_signature_error_incorrect_secret_key(s)
+}
+
+func TestCreateBucket(s *S3Conf) {
+	CreateBucket_invalid_bucket_name(s)
+	CreateBucket_existing_bucket(s)
+	CreateBucket_as_user(s)
+	CreateDeleteBucket_success(s)
+}
+
+func TestHeadBucket(s *S3Conf) {
+	HeadBucket_non_existing_bucket(s)
+	HeadBucket_success(s)
+}
+
+func TestListBuckets(s *S3Conf) {
+	ListBuckets_as_user(s)
+	ListBuckets_as_admin(s)
+	ListBuckets_success(s)
+}
+
+func TestDeleteBucket(s *S3Conf) {
+	DeleteBucket_non_existing_bucket(s)
+	DeleteBucket_non_empty_bucket(s)
+	DeleteBucket_success_status_code(s)
+}
+
+func TestPutObject(s *S3Conf) {
+	PutObject_non_existing_bucket(s)
+	PutObject_special_chars(s)
+	PutObject_invalid_long_tags(s)
+	PutObject_success(s)
+}
+
+func TestHeadObject(s *S3Conf) {
+	HeadObject_non_existing_object(s)
+	HeadObject_success(s)
+}
+
+func TestGetObject(s *S3Conf) {
+	GetObject_non_existing_key(s)
+	GetObject_invalid_ranges(s)
+	GetObject_with_meta(s)
+	GetObject_success(s)
+	GetObject_by_range_success(s)
+}
+
+func TestListObjects(s *S3Conf) {
+	ListObjects_non_existing_bucket(s)
+	ListObjects_with_prefix(s)
+	ListObject_truncated(s)
+	ListObjects_invalid_max_keys(s)
+	ListObjects_max_keys_0(s)
+	ListObjects_delimiter(s)
+	ListObjects_max_keys_none(s)
+	ListObjects_marker_not_from_obj_list(s)
+}
+
+func TestDeleteObject(s *S3Conf) {
+	DeleteObject_non_existing_object(s)
+	DeleteObject_success(s)
+	DeleteObject_success_status_code(s)
+}
+
+func TestDeleteObjects(s *S3Conf) {
+	DeleteObjects_empty_input(s)
+	DeleteObjects_non_existing_objects(s)
+	DeleteObjects_success(s)
+}
+
+func TestCopyObject(s *S3Conf) {
+	CopyObject_non_existing_dst_bucket(s)
+	CopyObject_not_owned_source_bucket(s)
+	CopyObject_copy_to_itself(s)
+	CopyObject_to_itself_with_new_metadata(s)
+	CopyObject_success(s)
+}
+
+func TestPutObjectTagging(s *S3Conf) {
+	PutObjectTagging_non_existing_object(s)
+	PutObjectTagging_long_tags(s)
+	PutObjectTagging_success(s)
+}
+
+func TestGetObjectTagging(s *S3Conf) {
+	GetObjectTagging_non_existing_object(s)
+	GetObjectTagging_success(s)
+}
+
+func TestDeleteObjectTagging(s *S3Conf) {
+	DeleteObjectTagging_non_existing_object(s)
+	DeleteObjectTagging_success_status(s)
+	DeleteObjectTagging_success(s)
+}
+
+func TestCreateMultipartUpload(s *S3Conf) {
+	CreateMultipartUpload_non_existing_bucket(s)
+	CreateMultipartUpload_success(s)
+}
+
+func TestUploadPart(s *S3Conf) {
+	UploadPart_non_existing_bucket(s)
+	UploadPart_invalid_part_number(s)
+	UploadPart_non_existing_key(s)
+	UploadPart_non_existing_mp_upload(s)
+	UploadPart_success(s)
+}
+
+func TestUploadPartCopy(s *S3Conf) {
+	UploadPartCopy_non_existing_bucket(s)
+	UploadPartCopy_incorrect_uploadId(s)
+	UploadPartCopy_incorrect_object_key(s)
+	UploadPartCopy_invalid_part_number(s)
+	UploadPartCopy_invalid_copy_source(s)
+	UploadPartCopy_non_existing_source_bucket(s)
+	UploadPartCopy_non_existing_source_object_key(s)
+	UploadPartCopy_success(s)
+	UploadPartCopy_by_range_invalid_range(s)
+	UploadPartCopy_greater_range_than_obj_size(s)
+	UploadPartCopy_by_range_success(s)
+}
+
+func TestListParts(s *S3Conf) {
+	ListParts_incorrect_uploadId(s)
+	ListParts_incorrect_object_key(s)
+	ListParts_success(s)
+}
+
+func TestListMultipartUploads(s *S3Conf) {
+	ListMultipartUploads_non_existing_bucket(s)
+	ListMultipartUploads_empty_result(s)
+	ListMultipartUploads_invalid_max_uploads(s)
+	ListMultipartUploads_max_uploads(s)
+	ListMultipartUploads_incorrect_next_key_marker(s)
+	ListMultipartUploads_ignore_upload_id_marker(s)
+	ListMultipartUploads_success(s)
+}
+
+func TestAbortMultipartUpload(s *S3Conf) {
+	AbortMultipartUpload_non_existing_bucket(s)
+	AbortMultipartUpload_incorrect_uploadId(s)
+	AbortMultipartUpload_incorrect_object_key(s)
+	AbortMultipartUpload_success(s)
+	AbortMultipartUpload_success_status_code(s)
+}
+
+func TestCompleteMultipartUpload(s *S3Conf) {
+	CompletedMultipartUpload_non_existing_bucket(s)
+	CompleteMultipartUpload_invalid_part_number(s)
+	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_success(s)
+}
+
+func TestPutBucketAcl(s *S3Conf) {
+	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_invalid_acl_canned_and_acp(s)
+	PutBucketAcl_invalid_acl_canned_and_grants(s)
+	PutBucketAcl_invalid_acl_acp_and_grants(s)
+	PutBucketAcl_invalid_owner(s)
+	PutBucketAcl_success_access_denied(s)
+	PutBucketAcl_success_grants(s)
+	PutBucketAcl_success_canned_acl(s)
+	PutBucketAcl_success_acp(s)
+}
+
+func TestGetBucketAcl(s *S3Conf) {
+	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_access_denied(s)
+	GetBucketAcl_success(s)
+}
+
+func TestFullFlow(s *S3Conf) {
+	TestAuthentication(s)
+	TestCreateBucket(s)
+	TestHeadBucket(s)
+	TestListBuckets(s)
+	TestDeleteBucket(s)
+	TestPutObject(s)
+	TestHeadObject(s)
+	TestGetObject(s)
+	TestListObjects(s)
+	TestDeleteObject(s)
+	TestDeleteObjects(s)
+	TestCopyObject(s)
+	TestPutObjectTagging(s)
+	TestDeleteObjectTagging(s)
+	TestCreateMultipartUpload(s)
+	TestUploadPart(s)
+	TestUploadPartCopy(s)
+	TestListParts(s)
+	TestListMultipartUploads(s)
+	TestAbortMultipartUpload(s)
+	TestCompleteMultipartUpload(s)
+	TestPutBucketAcl(s)
+	TestGetBucketAcl(s)
+}
+
+func TestPosix(s *S3Conf) {
+	PutObject_overwrite_dir_obj(s)
+	PutObject_overwrite_file_obj(s)
+	PutObject_dir_obj_with_data(s)
+	CreateMultipartUpload_dir_obj(s)
+}

integration/utils.go

@@ -1,15 +1,42 @@
 package integration
 
 import (
+	"bytes"
 	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
 	"fmt"
+	"io"
+	rnd "math/rand"
+	"net/http"
+	"os"
 	"os/exec"
 	"strings"
+	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
+var (
+	bcktCount  = 0
+	succUsrCrt = "The user has been created successfully"
+	failUsrCrt = "failed to create a user: update iam data: account already exists"
+)
+
+func getBucketName() string {
+	bcktCount++
+	return fmt.Sprintf("test-bucket-%v", bcktCount)
+}
+
 func setup(s *S3Conf, bucket string) error {
 	s3client := s3.NewFromConfig(s.Config())
 
@@ -69,6 +96,170 @@ func teardown(s *S3Conf, bucket string) error {
 	return err
 }
 
+func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
+	runF(testName)
+	bucketName := getBucketName()
+	err := setup(s, bucketName)
+	if err != nil {
+		failF("%v: failed to create a bucket: %v", testName, err.Error())
+		return
+	}
+	client := s3.NewFromConfig(s.Config())
+	handlerErr := handler(client, bucketName)
+	if handlerErr != nil {
+		failF("%v: %v", testName, handlerErr.Error())
+	}
+
+	err = teardown(s, bucketName)
+	if err != nil {
+		if handlerErr == nil {
+			failF("%v: failed to delete the bucket: %v", testName, err.Error())
+		} else {
+			fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
+		}
+	}
+	if handlerErr == nil {
+		passF(testName)
+	}
+}
+
+type authConfig struct {
+	testName string
+	path     string
+	method   string
+	body     []byte
+	service  string
+	date     time.Time
+}
+
+func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
+	runF(cfg.testName)
+	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err.Error())
+		return
+	}
+
+	err = handler(req)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err.Error())
+		return
+	}
+	passF(cfg.testName)
+}
+
+func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
+	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(body)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date)
+	if signErr != nil {
+		return nil, fmt.Errorf("failed to sign the request: %w", signErr)
+	}
+
+	return req, nil
+}
+
+func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var errResp s3err.APIErrorResponse
+	err = xml.Unmarshal(body, &errResp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != apiErr.HTTPStatusCode {
+		return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
+	}
+	if errResp.Code != apiErr.Code {
+		return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
+	}
+	if errResp.Message != apiErr.Description {
+		return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
+	}
+
+	return nil
+}
+
+func checkApiErr(err error, apiErr s3err.APIError) error {
+	if err == nil {
+		return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
+			return nil
+		}
+
+		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+	} else {
+		return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
+	}
+}
+
+func checkSdkApiErr(err error, code string) error {
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() != code {
+			return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
+		}
+		return nil
+	}
+	return err
+}
+
+func putObjects(client *s3.Client, objs []string, bucket string) error {
+	for _, key := range objs {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Key:    &key,
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
+	data = make([]byte, lgth)
+	rand.Read(data)
+	csum = sha256.Sum256(data)
+	r := bytes.NewReader(data)
+	input.Body = r
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err = client.PutObject(ctx, input)
+	cancel()
+
+	return
+}
+
+func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: &bucket,
+		Key:    &key,
+	})
+	cancel()
+	return out, err
+}
+
 func isEqual(a, b []byte) bool {
 	if len(a) != len(b) {
 		return false
@@ -83,32 +274,33 @@ func isEqual(a, b []byte) bool {
 	return true
 }
 
-func contains(name string, list []types.Object) bool {
-	for _, item := range list {
-		fmt.Println(*item.Key)
-		if strings.EqualFold(name, *item.Key) {
-			return true
+func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+	for i, item := range list1 {
+		if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
+			return false
 		}
 	}
-	return false
+
+	return true
 }
 
-func containsUID(name, id string, list []types.MultipartUpload) bool {
-	for _, item := range list {
-		if strings.EqualFold(name, *item.Key) && strings.EqualFold(id, *item.UploadId) {
-			return true
-		}
+func compareParts(parts1, parts2 []types.Part) bool {
+	if len(parts1) != len(parts2) {
+		return false
 	}
-	return false
-}
 
-func containsPart(part int32, list []types.Part) bool {
-	for _, item := range list {
-		if item.PartNumber == part {
-			return true
+	for i, prt := range parts1 {
+		if prt.PartNumber != parts2[i].PartNumber {
+			return false
+		}
+		if *prt.ETag != *parts2[i].ETag {
+			return false
 		}
 	}
-	return false
+	return true
 }
 
 func areTagsSame(tags1, tags2 []types.Tag) bool {
@@ -133,7 +325,7 @@ func containsTag(tag types.Tag, list []types.Tag) bool {
 	return false
 }
 
-func checkGrants(grts1, grts2 []types.Grant) bool {
+func compareGrants(grts1, grts2 []types.Grant) bool {
 	if len(grts1) != len(grts2) {
 		return false
 	}
@@ -154,3 +346,204 @@ func execCommand(args ...string) ([]byte, error) {
 
 	return cmd.CombinedOutput()
 }
+
+func getString(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
+}
+
+func getPtr(str string) *string {
+	return &str
+}
+
+func areMapsSame(mp1, mp2 map[string]string) bool {
+	if len(mp1) != len(mp2) {
+		return false
+	}
+	for key, val := range mp1 {
+		if mp2[key] != val {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[*elem.Name] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[elem.Name]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareObjects(list1 []string, list2 []types.Object) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Prefix]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
+	dr := NewDataReader(size, size)
+	datafile := "rand.data"
+	w, err := os.Create(datafile)
+	if err != nil {
+		return parts, err
+	}
+	defer w.Close()
+
+	_, err = io.Copy(w, dr)
+	if err != nil {
+		return parts, err
+	}
+
+	fileInfo, err := w.Stat()
+	if err != nil {
+		return parts, err
+	}
+
+	partSize := fileInfo.Size() / int64(partCount)
+	var offset int64
+
+	for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
+		partStart := (partNumber - 1) * partSize
+		partEnd := partStart + partSize - 1
+		if partEnd > fileInfo.Size()-1 {
+			partEnd = fileInfo.Size() - 1
+		}
+		partBuffer := make([]byte, partEnd-partStart+1)
+		_, err := w.ReadAt(partBuffer, partStart)
+		if err != nil {
+			return parts, err
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     &bucket,
+			Key:        &key,
+			UploadId:   &uploadId,
+			Body:       bytes.NewReader(partBuffer),
+			PartNumber: int32(partNumber),
+		})
+		cancel()
+		if err != nil {
+			return parts, err
+		} else {
+			parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
+			offset += partSize
+		}
+	}
+
+	return parts, err
+}
+
+type user struct {
+	access string
+	secret string
+	role   string
+}
+
+func createUsers(s *S3Conf, users []user) error {
+	for _, usr := range users {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
+			return fmt.Errorf("failed to create a user account")
+		}
+	}
+	return nil
+}
+
+func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
+	for _, bucket := range buckets {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
+			return fmt.Errorf(string(out))
+		}
+	}
+
+	return nil
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func genRandString(length int) string {
+	source := rnd.NewSource(time.Now().UnixNano())
+	random := rnd.New(source)
+	result := make([]byte, length)
+	for i := range result {
+		result[i] = charset[random.Intn(len(charset))]
+	}
+	return string(result)
+}

runtests.sh

@@ -6,7 +6,7 @@ rm -rf /tmp/covdata
 mkdir /tmp/covdata
 
 # run server in background
-GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass posix /tmp/gw &
+GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
 GW_PID=$!
 
 # wait a second for server to start up

s3api/admin-router.go

@@ -0,0 +1,43 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+)
+
+type S3AdminRouter struct{}
+
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
+	controller := controllers.NewAdminController(iam, be)
+
+	// CreateUser admin api
+	app.Patch("/create-user", controller.CreateUser)
+
+	// DeleteUsers admin api
+	app.Patch("/delete-user", controller.DeleteUser)
+
+	// ListUsers admin api
+	app.Patch("/list-users", controller.ListUsers)
+
+	// ChangeBucketOwner admin api
+	app.Patch("/change-bucket-owner", controller.ChangeBucketOwner)
+
+	// ListBucketsAndOwners admin api
+	app.Patch("/list-buckets", controller.ListBuckets)
+}

s3api/admin-server.go

@@ -0,0 +1,72 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"crypto/tls"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+type S3AdminServer struct {
+	app     *fiber.App
+	backend backend.Backend
+	router  *S3AdminRouter
+	port    string
+	cert    *tls.Certificate
+}
+
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
+	server := &S3AdminServer{
+		app:     app,
+		backend: be,
+		router:  new(S3AdminRouter),
+		port:    port,
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
+
+	// Logging middlewares
+	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(nil))
+
+	// Authentication middlewares
+	app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(nil))
+	app.Use(middlewares.AclParser(be, nil))
+
+	server.router.Init(app, be, iam)
+
+	return server
+}
+
+type AdminOpt func(s *S3AdminServer)
+
+func WithAdminSrvTLS(cert tls.Certificate) AdminOpt {
+	return func(s *S3AdminServer) { s.cert = &cert }
+}
+
+func (sa *S3AdminServer) Serve() (err error) {
+	if sa.cert != nil {
+		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
+	}
+	return sa.app.Listen(sa.port)
+}

s3api/controllers/admin.go

@@ -15,30 +15,39 @@
 package controllers
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 )
 
 type AdminController struct {
-	IAMService auth.IAMService
+	iam auth.IAMService
+	be  backend.Backend
+}
+
+func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
+	return AdminController{iam: iam, be: be}
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
-	access, secret, role := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role")
-	requesterRole := ctx.Locals("role").(string)
-
-	if requesterRole != "admin" {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
 		return fmt.Errorf("access denied: only admin users have access to this resource")
 	}
-	if role != "user" && role != "admin" {
+	var usr auth.Account
+	err := json.Unmarshal(ctx.Body(), &usr)
+	if err != nil {
+		return fmt.Errorf("failed to parse request body: %w", err)
+	}
+
+	if usr.Role != "user" && usr.Role != "admin" {
 		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
 	}
 
-	user := auth.Account{Secret: secret, Role: role}
-
-	err := c.IAMService.CreateAccount(access, user)
+	err = c.iam.CreateAccount(usr)
 	if err != nil {
 		return fmt.Errorf("failed to create a user: %w", err)
 	}
@@ -48,15 +57,66 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
-	requesterRole := ctx.Locals("role").(string)
-	if requesterRole != "admin" {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
 		return fmt.Errorf("access denied: only admin users have access to this resource")
 	}
 
-	err := c.IAMService.DeleteUserAccount(access)
+	err := c.iam.DeleteUserAccount(access)
 	if err != nil {
 		return err
 	}
 
 	return ctx.SendString("The user has been deleted successfully")
 }
+
+func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	accs, err := c.iam.ListUserAccounts()
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(accs)
+}
+
+func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	owner := ctx.Query("owner")
+	bucket := ctx.Query("bucket")
+
+	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return fmt.Errorf("user specified as the new bucket owner does not exist")
+	}
+
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
+	if err != nil {
+		return err
+	}
+
+	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+}
+
+func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+
+	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(buckets)
+}

s3api/controllers/admin_test.go

@@ -15,12 +15,17 @@
 package controllers
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
 )
 
 func TestAdminController_CreateUser(t *testing.T) {
@@ -29,8 +34,8 @@ func TestAdminController_CreateUser(t *testing.T) {
 	}
 
 	adminController := AdminController{
-		IAMService: &IAMServiceMock{
-			CreateAccountFunc: func(access string, account auth.Account) error {
+		iam: &IAMServiceMock{
+			CreateAccountFunc: func(account auth.Account) error {
 				return nil
 			},
 		},
@@ -39,7 +44,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 	app := fiber.New()
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "admin")
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
 		return ctx.Next()
 	})
 
@@ -48,10 +53,22 @@ func TestAdminController_CreateUser(t *testing.T) {
 	appErr := fiber.New()
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "user")
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
 		return ctx.Next()
 	})
 
+	usr := auth.Account{
+		Access: "access",
+		Secret: "secret",
+		Role:   "invalid role",
+	}
+
+	user, _ := json.Marshal(&usr)
+
+	usr.Role = "admin"
+
+	succUsr, _ := json.Marshal(&usr)
+
 	appErr.Patch("/create-user", adminController.CreateUser)
 
 	tests := []struct {
@@ -65,7 +82,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 			name: "Admin-create-user-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=user", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
 			},
 			wantErr:    false,
 			statusCode: 200,
@@ -74,7 +91,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 			name: "Admin-create-user-invalid-user-role",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=invalid", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -83,7 +100,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 			name: "Admin-create-user-invalid-requester-role",
 			app:  appErr,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=admin", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -108,7 +125,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	}
 
 	adminController := AdminController{
-		IAMService: &IAMServiceMock{
+		iam: &IAMServiceMock{
 			DeleteUserAccountFunc: func(access string) error {
 				return nil
 			},
@@ -118,7 +135,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	app := fiber.New()
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "admin")
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
 		return ctx.Next()
 	})
 
@@ -127,7 +144,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	appErr := fiber.New()
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "user")
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
 		return ctx.Next()
 	})
 
@@ -171,3 +188,294 @@ func TestAdminController_DeleteUser(t *testing.T) {
 		}
 	}
 }
+
+func TestAdminController_ListUsers(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, fmt.Errorf("server error")
+			},
+		},
+	}
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/list-users", adminControllerErr.ListUsers)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-users", adminController.ListUsers)
+
+	appSucc := fiber.New()
+
+	appSucc.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appSucc.Patch("/list-users", adminController.ListUsers)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-list-users-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-iam-error",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-success",
+			app:  appSucc,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListUsers() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListUsers() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ChangeBucketOwner(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
+				return nil
+			},
+		},
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerIamErr := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, fmt.Errorf("unknown server error")
+			},
+		},
+	}
+
+	adminControllerIamAccDoesNotExist := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appIamErr := fiber.New()
+
+	appIamErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
+
+	appIamNoSuchUser := fiber.New()
+
+	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Change-bucket-owner-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-check-account-server-error",
+			app:  appIamErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-acc-does-not-exist",
+			app:  appIamNoSuchUser,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 201,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ChangeBucketOwner() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ChangeBucketOwner() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ListBuckets(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
+				return []s3response.Bucket{}, nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/list-buckets", adminController.ListBuckets)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "List-buckets-incorrect-role",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "List-buckets-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListBuckets() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}

s3api/controllers/base.go

@@ -25,7 +25,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
@@ -54,11 +53,8 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs
 }
 
 func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
-	access, isRoot := ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
-	if err := auth.IsAdmin(access, isRoot); err != nil {
-		return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"})
-	}
-	res, err := c.be.ListBuckets(access, isRoot)
+	acct := ctx.Locals("account").(auth.Account)
+	res, err := c.be.ListBuckets(ctx.Context(), acct.Access, acct.Role == "admin")
 	return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"})
 }
 
@@ -70,28 +66,19 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	maxParts := ctx.QueryInt("max-parts", 0)
 	partNumberMarker := ctx.Query("part-number-marker")
 	acceptRange := ctx.Get("Range")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
+	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
 
-		tags, err := c.be.GetTags(bucket, key)
+		tags, err := c.be.GetObjectTagging(ctx.Context(), bucket, key)
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
@@ -115,11 +102,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			}
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
 		}
 
-		res, err := c.be.ListParts(&s3.ListPartsInput{
+		res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
 			Bucket:           &bucket,
 			Key:              &key,
 			UploadId:         &uploadId,
@@ -130,10 +117,10 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner})
 		}
-		res, err := c.be.GetObjectAcl(&s3.GetObjectAclInput{
+		res, err := c.be.GetObjectAcl(ctx.Context(), &s3.GetObjectAclInput{
 			Bucket: &bucket,
 			Key:    &key,
 		})
@@ -141,14 +128,14 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner})
 		}
 		var oattrs []types.ObjectAttributes
 		for _, a := range strings.Split(attrs, ",") {
 			oattrs = append(oattrs, types.ObjectAttributes(a))
 		}
-		res, err := c.be.GetObjectAttributes(&s3.GetObjectAttributesInput{
+		res, err := c.be.GetObjectAttributes(ctx.Context(), &s3.GetObjectAttributesInput{
 			Bucket:           &bucket,
 			Key:              &key,
 			ObjectAttributes: oattrs,
@@ -156,12 +143,12 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
 	}
 
 	ctx.Locals("logResBody", false)
-	res, err := c.be.GetObject(&s3.GetObjectInput{
+	res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{
 		Bucket: &bucket,
 		Key:    &key,
 		Range:  &acceptRange,
@@ -178,6 +165,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	if res.LastModified != nil {
 		lastmod = res.LastModified.Format(timefmt)
 	}
+
 	utils.SetResponseHeaders(ctx, []utils.CustomHeader{
 		{
 			Key:   "Content-Length",
@@ -203,6 +191,18 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			Key:   "x-amz-storage-class",
 			Value: string(res.StorageClass),
 		},
+		{
+			Key:   "Content-Range",
+			Value: getstring(res.ContentRange),
+		},
+		{
+			Key:   "accept-ranges",
+			Value: getstring(res.AcceptRanges),
+		},
+		{
+			Key:   "x-amz-tagging-count",
+			Value: fmt.Sprint(res.TagCount),
+		},
 	})
 	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
 }
@@ -217,107 +217,129 @@ func getstring(s *string) string {
 func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	bucket := ctx.Params("bucket")
 	prefix := ctx.Query("prefix")
-	marker := ctx.Query("continuation-token")
+	cToken := ctx.Query("continuation-token")
+	marker := ctx.Query("marker")
 	delimiter := ctx.Query("delimiter")
-	maxkeys := ctx.QueryInt("max-keys")
-	access := ctx.Locals("access").(string)
+	maxkeysStr := ctx.Query("max-keys")
+	keyMarker := ctx.Query("key-marker")
+	maxUploadsStr := ctx.Query("max-uploads")
+	uploadIdMarker := ctx.Query("upload-id-marker")
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
-
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
+	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner})
 		}
 
+		data, err := c.be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
+		if err != nil {
+			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
+		}
+
 		res, err := auth.ParseACLOutput(data)
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner})
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("uploads") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
 		}
-		res, err := c.be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
+		maxUploads, err := utils.ParseUint(maxUploadsStr)
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err, &MetaOpts{
+				Logger:      c.logger,
+				Action:      "ListMultipartUploads",
+				BucketOwner: parsedAcl.Owner,
+			})
+		}
+		res, err := c.be.ListMultipartUploads(ctx.Context(), &s3.ListMultipartUploadsInput{
+			Bucket:         &bucket,
+			Delimiter:      &delimiter,
+			Prefix:         &prefix,
+			UploadIdMarker: &uploadIdMarker,
+			MaxUploads:     maxUploads,
+			KeyMarker:      &keyMarker,
+		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
 	}
 
 	if ctx.QueryInt("list-type") == 2 {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
 		}
-		res, err := c.be.ListObjectsV2(&s3.ListObjectsV2Input{
+		maxkeys, err := utils.ParseUint(maxkeysStr)
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err, &MetaOpts{
+				Logger:      c.logger,
+				Action:      "ListObjectsV2",
+				BucketOwner: parsedAcl.Owner,
+			})
+		}
+		res, err := c.be.ListObjectsV2(ctx.Context(), &s3.ListObjectsV2Input{
 			Bucket:            &bucket,
 			Prefix:            &prefix,
-			ContinuationToken: &marker,
+			ContinuationToken: &cToken,
 			Delimiter:         &delimiter,
-			MaxKeys:           int32(maxkeys),
+			MaxKeys:           maxkeys,
 		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 		return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
 	}
 
-	res, err := c.be.ListObjects(&s3.ListObjectsInput{
+	maxkeys, err := utils.ParseUint(maxkeysStr)
+	if err != nil {
+		return SendXMLResponse(ctx, nil, err, &MetaOpts{
+			Logger:      c.logger,
+			Action:      "ListObjects",
+			BucketOwner: parsedAcl.Owner,
+		})
+	}
+
+	res, err := c.be.ListObjects(ctx.Context(), &s3.ListObjectsInput{
 		Bucket:    &bucket,
 		Prefix:    &prefix,
 		Marker:    &marker,
 		Delimiter: &delimiter,
-		MaxKeys:   int32(maxkeys),
+		MaxKeys:   maxkeys,
 	})
 	return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
 }
 
 func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
-	bucket, bucketOwner, acl, grantFullControl, grantRead, grantReadACP, granWrite, grantWriteACP, access, isRoot :=
+	bucket, acl, grantFullControl, grantRead, grantReadACP, granWrite, grantWriteACP, acct, isRoot :=
 		ctx.Params("bucket"),
-		ctx.Get("X-Amz-Expected-Bucket-Owner"),
 		ctx.Get("X-Amz-Acl"),
 		ctx.Get("X-Amz-Grant-Full-Control"),
 		ctx.Get("X-Amz-Grant-Read"),
 		ctx.Get("X-Amz-Grant-Read-Acp"),
 		ctx.Get("X-Amz-Grant-Write"),
 		ctx.Get("X-Amz-Grant-Write-Acp"),
-		ctx.Locals("access").(string),
+		ctx.Locals("account").(auth.Account),
 		ctx.Locals("isRoot").(bool)
 
 	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
 		var input *s3.PutBucketAclInput
+		var accessControlPolicy auth.AccessControlPolicy
 
-		data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-		if err != nil {
-			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl"})
-		}
-
-		parsedAcl, err := auth.ParseACL(data)
-		if err != nil {
-			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl"})
-		}
-
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil {
+		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE_ACP", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 		}
 
-		if len(ctx.Body()) > 0 {
-			if grants+acl != "" {
-				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
-			}
+		err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
+		if err != nil {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
+		}
 
-			var accessControlPolicy auth.AccessControlPolicy
-			err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
-			if err != nil {
+		if len(accessControlPolicy.AccessControlList.Grants) > 0 {
+			if grants+acl != "" {
 				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 			}
 
@@ -331,14 +353,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			if acl != "private" && acl != "public-read" && acl != "public-read-write" {
 				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 			}
-			if len(ctx.Body()) > 0 || grants != "" {
+			if len(accessControlPolicy.AccessControlList.Grants) > 0 || grants != "" {
 				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 			}
 
 			input = &s3.PutBucketAclInput{
 				Bucket:              &bucket,
 				ACL:                 types.BucketCannedACL(acl),
-				AccessControlPolicy: &types.AccessControlPolicy{Owner: &types.Owner{ID: &bucketOwner}},
+				AccessControlPolicy: &types.AccessControlPolicy{Owner: &accessControlPolicy.Owner},
 			}
 		}
 		if grants != "" {
@@ -349,7 +371,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				GrantReadACP:        &grantReadACP,
 				GrantWrite:          &granWrite,
 				GrantWriteACP:       &grantWriteACP,
-				AccessControlPolicy: &types.AccessControlPolicy{Owner: &types.Owner{ID: &bucketOwner}},
+				AccessControlPolicy: &types.AccessControlPolicy{Owner: &accessControlPolicy.Owner},
 				ACL:                 "",
 			}
 		}
@@ -359,15 +381,19 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 		}
 
-		err = c.be.PutBucketAcl(bucket, updAcl)
+		err = c.be.PutBucketAcl(ctx.Context(), bucket, updAcl)
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 	}
 
-	err := c.be.CreateBucket(&s3.CreateBucketInput{
+	if ok := utils.IsValidBucketName(bucket); !ok {
+		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketName), &MetaOpts{Logger: c.logger, Action: "CreateBucket"})
+	}
+
+	err := c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{
 		Bucket:          &bucket,
-		ObjectOwnership: types.ObjectOwnership(access),
+		ObjectOwnership: types.ObjectOwnership(acct.Access),
 	})
-	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucket", BucketOwner: ctx.Locals("access").(string)})
+	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "CreateBucket", BucketOwner: acct.Access})
 }
 
 func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
@@ -375,8 +401,10 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	keyStart := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
+	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
+	tagging := ctx.Get("x-amz-tagging")
 
 	// Copy source headers
 	copySource := ctx.Get("X-Amz-Copy-Source")
@@ -408,16 +436,6 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		keyStart = keyStart + "/"
 	}
 
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
 		var objTagging s3response.Tagging
 		err := xml.Unmarshal(ctx.Body(), &objTagging)
@@ -428,14 +446,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		tags := make(map[string]string, len(objTagging.TagSet.Tags))
 
 		for _, tag := range objTagging.TagSet.Tags {
+			if len(tag.Key) > 128 || len(tag.Value) > 256 {
+				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidTag), &MetaOpts{Logger: c.logger, Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner})
+			}
 			tags[tag.Key] = tag.Value
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
 
-		err = c.be.SetTags(bucket, keyStart, tags)
+		err = c.be.PutObjectTagging(ctx.Context(), bucket, keyStart, tags)
 		return SendResponse(ctx, err, &MetaOpts{
 			Logger:      c.logger,
 			EvSender:    c.evSender,
@@ -451,7 +472,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner})
 		}
 
-		resp, err := c.be.UploadPartCopy(&s3.UploadPartCopyInput{
+		resp, err := c.be.UploadPartCopy(ctx.Context(), &s3.UploadPartCopyInput{
 			Bucket:              &bucket,
 			Key:                 &keyStart,
 			CopySource:          &copySource,
@@ -469,7 +490,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -480,7 +501,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 
 		body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
 		ctx.Locals("logReqBody", false)
-		etag, err := c.be.UploadPart(&s3.UploadPartInput{
+		etag, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{
 			Bucket:        &bucket,
 			Key:           &keyStart,
 			UploadId:      &uploadId,
@@ -542,7 +563,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			}
 		}
 
-		err = c.be.PutObjectAcl(input)
+		err := c.be.PutObjectAcl(ctx.Context(), input)
 		return SendResponse(ctx, err, &MetaOpts{
 			Logger:      c.logger,
 			EvSender:    c.evSender,
@@ -553,11 +574,12 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if copySource != "" {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
 		}
 
 		var mtime time.Time
+		var err error
 		if copySrcModifSince != "" {
 			mtime, err = time.Parse(iso8601Format, copySrcModifSince)
 			if err != nil {
@@ -571,7 +593,10 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
 			}
 		}
-		res, err := c.be.CopyObject(&s3.CopyObjectInput{
+
+		metadata := utils.GetUserMetaData(&ctx.Request().Header)
+
+		res, err := c.be.CopyObject(ctx.Context(), &s3.CopyObjectInput{
 			Bucket:                      &bucket,
 			Key:                         &keyStart,
 			CopySource:                  &copySource,
@@ -579,6 +604,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			CopySourceIfNoneMatch:       &copySrcIfNoneMatch,
 			CopySourceIfModifiedSince:   &mtime,
 			CopySourceIfUnmodifiedSince: &umtime,
+			ExpectedBucketOwner:         &acct.Access,
+			Metadata:                    metadata,
 		})
 		if err == nil {
 			return SendXMLResponse(ctx, res, err, &MetaOpts{
@@ -601,7 +628,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 
 	metadata := utils.GetUserMetaData(&ctx.Request().Header)
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -611,12 +638,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	ctx.Locals("logReqBody", false)
-	etag, err := c.be.PutObject(&s3.PutObjectInput{
+	etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{
 		Bucket:        &bucket,
 		Key:           &keyStart,
 		ContentLength: contentLength,
 		Metadata:      metadata,
 		Body:          bytes.NewReader(ctx.Request().Body()),
+		Tagging:       &tagging,
 	})
 	ctx.Response().Header.Set("ETag", etag)
 	return SendResponse(ctx, err, &MetaOpts{
@@ -631,55 +659,37 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 }
 
 func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
-	bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
 
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBuckets"})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBuckets"})
-	}
-
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
 	}
 
-	err = c.be.DeleteBucket(&s3.DeleteBucketInput{
+	err := c.be.DeleteBucket(ctx.Context(), &s3.DeleteBucketInput{
 		Bucket: &bucket,
 	})
-	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
+	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner, Status: 204})
 }
 
 func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
-	bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
-	var dObj types.Delete
-
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects"})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects"})
-	}
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
+	var dObj s3response.DeleteObjects
 
 	if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
 		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
 	}
 
-	err = c.be.DeleteObjects(&s3.DeleteObjectsInput{
+	res, err := c.be.DeleteObjects(ctx.Context(), &s3.DeleteObjectsInput{
 		Bucket: &bucket,
-		Delete: &dObj,
+		Delete: &types.Delete{
+			Objects: dObj.Objects,
+		},
 	})
-	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
+	return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
 }
 
 func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
@@ -687,33 +697,25 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
+	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
 
-		err = c.be.RemoveTags(bucket, key)
+		err := c.be.DeleteObjectTagging(ctx.Context(), bucket, key)
 		return SendResponse(ctx, err, &MetaOpts{
+			Status:      http.StatusNoContent,
 			Logger:      c.logger,
 			EvSender:    c.evSender,
-			Action:      "RemoveObjectTagging",
+			Action:      "DeleteObjectTagging",
 			BucketOwner: parsedAcl.Owner,
 			EventName:   s3event.EventObjectTaggingDelete,
 		})
@@ -722,25 +724,25 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	if uploadId != "" {
 		expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer")
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner})
 		}
 
-		err := c.be.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
+		err := c.be.AbortMultipartUpload(ctx.Context(), &s3.AbortMultipartUploadInput{
 			UploadId:            &uploadId,
 			Bucket:              &bucket,
 			Key:                 &key,
 			ExpectedBucketOwner: &expectedBucketOwner,
 			RequestPayer:        types.RequestPayer(requestPayer),
 		})
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner})
+		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner, Status: 204})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObject", BucketOwner: parsedAcl.Owner})
 	}
 
-	err = c.be.DeleteObject(&s3.DeleteObjectInput{
+	err := c.be.DeleteObject(ctx.Context(), &s3.DeleteObjectInput{
 		Bucket: &bucket,
 		Key:    &key,
 	})
@@ -750,27 +752,18 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 		Action:      "DeleteObject",
 		BucketOwner: parsedAcl.Owner,
 		EventName:   s3event.EventObjectDelete,
+		Status:      204,
 	})
 }
 
 func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
-	bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
 
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket"})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket"})
-	}
-
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket", BucketOwner: parsedAcl.Owner})
 	}
 
-	_, err = c.be.HeadBucket(&s3.HeadBucketInput{
+	_, err := c.be.HeadBucket(ctx.Context(), &s3.HeadBucketInput{
 		Bucket: &bucket,
 	})
 	// TODO: set bucket response headers
@@ -782,30 +775,18 @@ const (
 )
 
 func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
-	bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{
-		Bucket: &bucket,
-	})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject"})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject"})
-	}
-
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner})
 	}
 
-	res, err := c.be.HeadObject(&s3.HeadObjectInput{
+	res, err := c.be.HeadObject(ctx.Context(), &s3.HeadObjectInput{
 		Bucket: &bucket,
 		Key:    &key,
 	})
@@ -860,21 +841,17 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
+	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
-	data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket})
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
-	}
-
-	parsedAcl, err := auth.ParseACL(data)
-	if err != nil {
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger})
+	path := ctx.Path()
+	if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" {
+		key = key + "/"
 	}
 
 	var restoreRequest s3.RestoreObjectInput
@@ -884,14 +861,14 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RestoreObject", BucketOwner: parsedAcl.Owner})
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RestoreObject", BucketOwner: parsedAcl.Owner})
 		}
 
 		restoreRequest.Bucket = &bucket
 		restoreRequest.Key = &key
 
-		err = c.be.RestoreObject(&restoreRequest)
+		err = c.be.RestoreObject(ctx.Context(), &restoreRequest)
 		return SendResponse(ctx, err, &MetaOpts{
 			Logger:      c.logger,
 			EvSender:    c.evSender,
@@ -901,20 +878,52 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 		})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("select") && ctx.Query("select-type") == "2" {
+		var payload s3response.SelectObjectContentPayload
+
+		if err := xml.Unmarshal(ctx.Body(), &payload); err != nil {
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrMalformedXML), &MetaOpts{
+				Logger:      c.logger,
+				Action:      "SelectObjectContent",
+				BucketOwner: parsedAcl.Owner,
+			})
+		}
+
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
+			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
+		}
+
+		res, err := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
+			Bucket:              &bucket,
+			Key:                 &key,
+			Expression:          payload.Expression,
+			ExpressionType:      payload.ExpressionType,
+			InputSerialization:  payload.InputSerialization,
+			OutputSerialization: payload.OutputSerialization,
+			RequestProgress:     payload.RequestProgress,
+			ScanRange:           payload.ScanRange,
+		})
+		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
+	}
+
 	if uploadId != "" {
 		data := struct {
 			Parts []types.CompletedPart `xml:"Part"`
 		}{}
 
 		if err := xml.Unmarshal(ctx.Body(), &data); err != nil {
+			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrMalformedXML), &MetaOpts{
+				Logger:      c.logger,
+				Action:      "CompleteMultipartUpload",
+				BucketOwner: parsedAcl.Owner,
+			})
+		}
+
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner})
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
-			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner})
-		}
-
-		res, err := c.be.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
+		res, err := c.be.CompleteMultipartUpload(ctx.Context(), &s3.CompleteMultipartUploadInput{
 			Bucket:   &bucket,
 			Key:      &key,
 			UploadId: &uploadId,
@@ -941,11 +950,12 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 		}
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner})
 	}
 
-	res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
+	res, err := c.be.CreateMultipartUpload(ctx.Context(),
+		&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
 	return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner})
 }
 
@@ -958,6 +968,7 @@ type MetaOpts struct {
 	EventName   s3event.EventType
 	ObjectETag  *string
 	VersionId   *string
+	Status      int
 }
 
 func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
@@ -992,9 +1003,12 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
 
 	utils.LogCtxDetails(ctx, []byte{})
 
+	if l.Status == 0 {
+		l.Status = http.StatusOK
+	}
 	// https://github.com/gofiber/fiber/issues/2080
 	// ctx.SendStatus() sets incorrect content length on HEAD request
-	ctx.Status(http.StatusOK)
+	ctx.Status(l.Status)
 	return nil
 }
 
@@ -1028,6 +1042,7 @@ func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *MetaOpts) error {
 		}
 
 		if len(b) > 0 {
+			ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(b)))
 			ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
 		}
 	}

s3api/controllers/base_test.go

@@ -15,6 +15,7 @@
 package controllers
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -28,6 +29,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
@@ -89,18 +91,14 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
-				return acldata, nil
-			},
-			ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, nil
 			},
 		},
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
-		ctx.Locals("isRoot", true)
+		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
 		ctx.Locals("isDebug", false)
 		return ctx.Next()
 	})
@@ -110,33 +108,19 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr := fiber.New()
 	s3ApiControllerErr := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
-				return acldata, nil
-			},
-			ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) {
+			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
 			},
 		},
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
-		ctx.Locals("isRoot", true)
+		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
 		ctx.Locals("isDebug", false)
 		return ctx.Next()
 	})
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
 
-	//Admin error case
-	admErr := fiber.New()
-	admErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
-		ctx.Locals("isRoot", false)
-		ctx.Locals("isDebug", false)
-		return ctx.Next()
-	})
-	admErr.Get("/", s3ApiController.ListBuckets)
-
 	tests := []struct {
 		name       string
 		args       args
@@ -162,15 +146,6 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
-		{
-			name: "admin-error-case",
-			args: args{
-				req: httptest.NewRequest(http.MethodGet, "/", nil),
-			},
-			app:        admErr,
-			wantErr:    false,
-			statusCode: 500,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -201,19 +176,19 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			ListPartsFunc: func(*s3.ListPartsInput) (s3response.ListPartsResponse, error) {
-				return s3response.ListPartsResponse{}, nil
+			ListPartsFunc: func(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+				return s3response.ListPartsResult{}, nil
 			},
-			GetObjectAclFunc: func(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
+			GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 				return &s3.GetObjectAclOutput{}, nil
 			},
-			GetObjectAttributesFunc: func(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+			GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
 				return &s3.GetObjectAttributesOutput{}, nil
 			},
-			GetObjectFunc: func(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+			GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
 				return &s3.GetObjectOutput{
 					Metadata:        map[string]string{"hello": "world"},
 					ContentType:     getPtr("application/xml"),
@@ -224,15 +199,16 @@ func TestS3ApiController_GetActions(t *testing.T) {
 					StorageClass:    "storage class",
 				}, nil
 			},
-			GetTagsFunc: func(bucket, object string) (map[string]string, error) {
+			GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
 				return map[string]string{"hello": "world"}, nil
 			},
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
@@ -353,25 +329,26 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
-				return s3response.ListMultipartUploadsResponse{}, nil
+			ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+				return s3response.ListMultipartUploadsResult{}, nil
 			},
-			ListObjectsV2Func: func(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+			ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
 				return &s3.ListObjectsV2Output{}, nil
 			},
-			ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 				return &s3.ListObjectsOutput{}, nil
 			},
 		},
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 
@@ -380,19 +357,20 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	//Error case
 	s3ApiControllerError := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 			},
 		},
 	}
 	appError := fiber.New()
 	appError.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
@@ -496,24 +474,41 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</AccessControlPolicy>
 	`
 
+	invOwnerBody := `
+	<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Owner>
+			<ID>hello</ID>
+		</Owner>
+	</AccessControlPolicy>
+	`
+
+	succBody := `
+	<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Owner>
+			<ID>valid access</ID>
+		</Owner>
+	</AccessControlPolicy>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			PutBucketAclFunc: func(string, []byte) error {
+			PutBucketAclFunc: func(context.Context, string, []byte) error {
 				return nil
 			},
-			CreateBucketFunc: func(*s3.CreateBucketInput) error {
+			CreateBucketFunc: func(context.Context, *s3.CreateBucketInput) error {
 				return nil
 			},
 		},
 	}
 	// Mock ctx.Locals
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{Owner: "valid access"})
 		return ctx.Next()
 	})
 	app.Put("/:bucket", s3ApiController.PutBucketActions)
@@ -528,14 +523,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	errAclReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	// PutBucketAcl incorrect bucket owner case
-	incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
+	incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
 	incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
-	incorrectBucketOwner.Header.Set("X-Amz-Expected-Bucket-Owner", "invalid access")
 
 	// PutBucketAcl acl success
-	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
+	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
 	aclSuccReq.Header.Set("X-Amz-Acl", "private")
-	aclSuccReq.Header.Set("X-Amz-Expected-Bucket-Owner", "valid access")
 
 	// Invalid acl body case
 	errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
@@ -593,6 +586,15 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Put-bucket-invalid-bucket-name",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/aa", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
 		{
 			name: "Put-bucket-success",
 			app:  app,
@@ -650,35 +652,36 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			PutObjectAclFunc: func(*s3.PutObjectAclInput) error {
+			PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
 				return nil
 			},
-			CopyObjectFunc: func(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+			CopyObjectFunc: func(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 				return &s3.CopyObjectOutput{
 					CopyObjectResult: &types.CopyObjectResult{},
 				}, nil
 			},
-			PutObjectFunc: func(*s3.PutObjectInput) (string, error) {
+			PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
 				return "ETag", nil
 			},
-			UploadPartFunc: func(*s3.UploadPartInput) (string, error) {
+			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
 				return "hello", nil
 			},
-			SetTagsFunc: func(bucket, object string, tags map[string]string) error {
+			PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
 				return nil
 			},
-			UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
 				return s3response.CopyObjectResult{}, nil
 			},
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
@@ -864,19 +867,20 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteBucketFunc: func(*s3.DeleteBucketInput) error {
+			DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
 				return nil
 			},
 		},
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 
@@ -896,7 +900,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 	}
 	for _, tt := range tests {
@@ -920,19 +924,20 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteObjectsFunc: func(*s3.DeleteObjectsInput) error {
-				return nil
+			DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+				return s3response.DeleteObjectsResult{}, nil
 			},
 		},
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	app.Post("/:bucket", s3ApiController.DeleteObjects)
@@ -990,25 +995,26 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteObjectFunc: func(*s3.DeleteObjectInput) error {
+			DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
 				return nil
 			},
-			AbortMultipartUploadFunc: func(*s3.AbortMultipartUploadInput) error {
+			AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
 				return nil
 			},
-			RemoveTagsFunc: func(bucket, object string) error {
+			DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error {
 				return nil
 			},
 		},
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
@@ -1017,18 +1023,19 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	appErr := fiber.New()
 
 	s3ApiControllerErr := S3ApiController{be: &BackendMock{
-		GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+		GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 			return acldata, nil
 		},
-		DeleteObjectFunc: func(*s3.DeleteObjectInput) error {
+		DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
 			return s3err.GetAPIError(7)
 		},
 	}}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
@@ -1047,7 +1054,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key?uploadId=324234", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 		{
 			name: "Remove-object-tagging-success",
@@ -1056,7 +1063,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key/key2?tagging", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 		{
 			name: "Delete-object-success",
@@ -1065,7 +1072,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 		{
 			name: "Delete-object-error",
@@ -1098,19 +1105,20 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+			HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 				return &s3.HeadBucketOutput{}, nil
 			},
 		},
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 
@@ -1120,19 +1128,20 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	appErr := fiber.New()
 
 	s3ApiControllerErr := S3ApiController{be: &BackendMock{
-		GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+		GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 			return acldata, nil
 		},
-		HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+		HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 			return nil, s3err.GetAPIError(3)
 		},
 	},
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 
@@ -1192,10 +1201,10 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+			HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 				return &s3.HeadObjectOutput{
 					ContentEncoding: &contentEncoding,
 					ContentLength:   64,
@@ -1208,9 +1217,10 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
@@ -1220,19 +1230,20 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 
 	s3ApiControllerErr := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+			HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 				return nil, s3err.GetAPIError(42)
 			},
 		},
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
@@ -1283,25 +1294,36 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) {
+			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			RestoreObjectFunc: func(restoreRequest *s3.RestoreObjectInput) error {
+			RestoreObjectFunc: func(context.Context, *s3.RestoreObjectInput) error {
 				return nil
 			},
-			CompleteMultipartUploadFunc: func(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+			CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 				return &s3.CompleteMultipartUploadOutput{}, nil
 			},
-			CreateMultipartUploadFunc: func(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 				return &s3.CreateMultipartUploadOutput{}, nil
 			},
+			SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
+				return s3response.SelectObjectContentResult{}, nil
+			},
 		},
 	}
 
+	bdy := `
+		<SelectObjectContentRequest>
+			<Expression>string</Expression>
+			<ExpressionType>string</ExpressionType>
+		</SelectObjectContentRequest>
+	`
+
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
+		ctx.Locals("parsedAcl", auth.ACL{})
 		return ctx.Next()
 	})
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
@@ -1331,6 +1353,24 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 500,
 		},
+		{
+			name: "Select-object-content-invalid-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?select&select-type=2", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Select-object-content-invalid-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?select&select-type=2", strings.NewReader(bdy)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Complete-multipart-upload-error",
 			app:  app,
@@ -1338,7 +1378,7 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 400,
 		},
 		{
 			name: "Complete-multipart-upload-success",
@@ -1378,16 +1418,9 @@ func Test_XMLresponse(t *testing.T) {
 		resp any
 		err  error
 	}
+
 	app := fiber.New()
-
-	var ctx fiber.Ctx
-	// Mocking the fiber ctx
-	app.Get("/:bucket/:key", func(c *fiber.Ctx) error {
-		ctx = *c
-		return nil
-	})
-
-	app.Test(httptest.NewRequest(http.MethodGet, "/my-bucket/my-key", nil))
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
 		name       string
@@ -1398,9 +1431,9 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Internal-server-error",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(16),
+				err:  s3err.GetAPIError(s3err.ErrInternalError),
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -1408,9 +1441,9 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Error-not-implemented",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(50),
+				err:  s3err.GetAPIError(s3err.ErrNotImplemented),
 			},
 			wantErr:    false,
 			statusCode: 501,
@@ -1418,7 +1451,7 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Invalid-request-body",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: make(chan int),
 				err:  nil,
 			},
@@ -1428,7 +1461,7 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Successful-response",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: "Valid response",
 				err:  nil,
 			},
@@ -1458,16 +1491,11 @@ func Test_response(t *testing.T) {
 		ctx  *fiber.Ctx
 		resp any
 		err  error
+		opts *MetaOpts
 	}
-	app := fiber.New()
-	var ctx fiber.Ctx
-	// Mocking the fiber ctx
-	app.Get("/:bucket/:key", func(c *fiber.Ctx) error {
-		ctx = *c
-		return nil
-	})
 
-	app.Test(httptest.NewRequest(http.MethodGet, "/my-bucket/my-key", nil))
+	app := fiber.New()
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
 		name       string
@@ -1478,9 +1506,10 @@ func Test_response(t *testing.T) {
 		{
 			name: "Internal-server-error",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(16),
+				err:  s3err.GetAPIError(s3err.ErrInternalError),
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -1488,9 +1517,10 @@ func Test_response(t *testing.T) {
 		{
 			name: "Internal-server-error-not-api",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
 				err:  fmt.Errorf("custom error"),
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -1498,9 +1528,10 @@ func Test_response(t *testing.T) {
 		{
 			name: "Error-not-implemented",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(50),
+				err:  s3err.GetAPIError(s3err.ErrNotImplemented),
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 501,
@@ -1508,17 +1539,31 @@ func Test_response(t *testing.T) {
 		{
 			name: "Successful-response",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: "Valid response",
 				err:  nil,
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Successful-response-status-204",
+			args: args{
+				ctx:  ctx,
+				resp: "Valid response",
+				err:  nil,
+				opts: &MetaOpts{
+					Status: 204,
+				},
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := SendResponse(tt.args.ctx, tt.args.err, &MetaOpts{}); (err != nil) != tt.wantErr {
+			if err := SendResponse(tt.args.ctx, tt.args.err, tt.args.opts); (err != nil) != tt.wantErr {
 				t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr)
 			}
 

s3api/controllers/iam_moq_test.go

@@ -18,7 +18,7 @@ var _ auth.IAMService = &IAMServiceMock{}
 //
 //		// make and configure a mocked auth.IAMService
 //		mockedIAMService := &IAMServiceMock{
-//			CreateAccountFunc: func(access string, account auth.Account) error {
+//			CreateAccountFunc: func(account auth.Account) error {
 //				panic("mock out the CreateAccount method")
 //			},
 //			DeleteUserAccountFunc: func(access string) error {
@@ -27,6 +27,12 @@ var _ auth.IAMService = &IAMServiceMock{}
 //			GetUserAccountFunc: func(access string) (auth.Account, error) {
 //				panic("mock out the GetUserAccount method")
 //			},
+//			ListUserAccountsFunc: func() ([]auth.Account, error) {
+//				panic("mock out the ListUserAccounts method")
+//			},
+//			ShutdownFunc: func() error {
+//				panic("mock out the Shutdown method")
+//			},
 //		}
 //
 //		// use mockedIAMService in code that requires auth.IAMService
@@ -35,7 +41,7 @@ var _ auth.IAMService = &IAMServiceMock{}
 //	}
 type IAMServiceMock struct {
 	// CreateAccountFunc mocks the CreateAccount method.
-	CreateAccountFunc func(access string, account auth.Account) error
+	CreateAccountFunc func(account auth.Account) error
 
 	// DeleteUserAccountFunc mocks the DeleteUserAccount method.
 	DeleteUserAccountFunc func(access string) error
@@ -43,12 +49,16 @@ type IAMServiceMock struct {
 	// GetUserAccountFunc mocks the GetUserAccount method.
 	GetUserAccountFunc func(access string) (auth.Account, error)
 
+	// ListUserAccountsFunc mocks the ListUserAccounts method.
+	ListUserAccountsFunc func() ([]auth.Account, error)
+
+	// ShutdownFunc mocks the Shutdown method.
+	ShutdownFunc func() error
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// CreateAccount holds details about calls to the CreateAccount method.
 		CreateAccount []struct {
-			// Access is the access argument value.
-			Access string
 			// Account is the account argument value.
 			Account auth.Account
 		}
@@ -62,28 +72,34 @@ type IAMServiceMock struct {
 			// Access is the access argument value.
 			Access string
 		}
+		// ListUserAccounts holds details about calls to the ListUserAccounts method.
+		ListUserAccounts []struct {
+		}
+		// Shutdown holds details about calls to the Shutdown method.
+		Shutdown []struct {
+		}
 	}
 	lockCreateAccount     sync.RWMutex
 	lockDeleteUserAccount sync.RWMutex
 	lockGetUserAccount    sync.RWMutex
+	lockListUserAccounts  sync.RWMutex
+	lockShutdown          sync.RWMutex
 }
 
 // CreateAccount calls CreateAccountFunc.
-func (mock *IAMServiceMock) CreateAccount(access string, account auth.Account) error {
+func (mock *IAMServiceMock) CreateAccount(account auth.Account) error {
 	if mock.CreateAccountFunc == nil {
 		panic("IAMServiceMock.CreateAccountFunc: method is nil but IAMService.CreateAccount was just called")
 	}
 	callInfo := struct {
-		Access  string
 		Account auth.Account
 	}{
-		Access:  access,
 		Account: account,
 	}
 	mock.lockCreateAccount.Lock()
 	mock.calls.CreateAccount = append(mock.calls.CreateAccount, callInfo)
 	mock.lockCreateAccount.Unlock()
-	return mock.CreateAccountFunc(access, account)
+	return mock.CreateAccountFunc(account)
 }
 
 // CreateAccountCalls gets all the calls that were made to CreateAccount.
@@ -91,11 +107,9 @@ func (mock *IAMServiceMock) CreateAccount(access string, account auth.Account) e
 //
 //	len(mockedIAMService.CreateAccountCalls())
 func (mock *IAMServiceMock) CreateAccountCalls() []struct {
-	Access  string
 	Account auth.Account
 } {
 	var calls []struct {
-		Access  string
 		Account auth.Account
 	}
 	mock.lockCreateAccount.RLock()
@@ -167,3 +181,57 @@ func (mock *IAMServiceMock) GetUserAccountCalls() []struct {
 	mock.lockGetUserAccount.RUnlock()
 	return calls
 }
+
+// ListUserAccounts calls ListUserAccountsFunc.
+func (mock *IAMServiceMock) ListUserAccounts() ([]auth.Account, error) {
+	if mock.ListUserAccountsFunc == nil {
+		panic("IAMServiceMock.ListUserAccountsFunc: method is nil but IAMService.ListUserAccounts was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockListUserAccounts.Lock()
+	mock.calls.ListUserAccounts = append(mock.calls.ListUserAccounts, callInfo)
+	mock.lockListUserAccounts.Unlock()
+	return mock.ListUserAccountsFunc()
+}
+
+// ListUserAccountsCalls gets all the calls that were made to ListUserAccounts.
+// Check the length with:
+//
+//	len(mockedIAMService.ListUserAccountsCalls())
+func (mock *IAMServiceMock) ListUserAccountsCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockListUserAccounts.RLock()
+	calls = mock.calls.ListUserAccounts
+	mock.lockListUserAccounts.RUnlock()
+	return calls
+}
+
+// Shutdown calls ShutdownFunc.
+func (mock *IAMServiceMock) Shutdown() error {
+	if mock.ShutdownFunc == nil {
+		panic("IAMServiceMock.ShutdownFunc: method is nil but IAMService.Shutdown was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockShutdown.Lock()
+	mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo)
+	mock.lockShutdown.Unlock()
+	return mock.ShutdownFunc()
+}
+
+// ShutdownCalls gets all the calls that were made to Shutdown.
+// Check the length with:
+//
+//	len(mockedIAMService.ShutdownCalls())
+func (mock *IAMServiceMock) ShutdownCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockShutdown.RLock()
+	calls = mock.calls.Shutdown
+	mock.lockShutdown.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -0,0 +1,61 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3log"
+)
+
+func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
+		path := ctx.Path()
+		pathParts := strings.Split(path, "/")
+		bucket := pathParts[1]
+		if path == "/" && ctx.Method() == http.MethodGet {
+			return ctx.Next()
+		}
+		if ctx.Method() == http.MethodPatch {
+			return ctx.Next()
+		}
+		if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
+			if err := auth.IsAdmin(acct, isRoot); err != nil {
+				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
+			}
+			return ctx.Next()
+		}
+		//TODO: provide correct action names for the logger, after implementing DetectAction middleware
+		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
+		parsedAcl, err := auth.ParseACL(data)
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
+		ctx.Locals("parsedAcl", parsedAcl)
+		return ctx.Next()
+	}
+}

s3api/middlewares/authentication.go

@@ -17,6 +17,9 @@ package middlewares
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"fmt"
+	"math"
+	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -34,6 +37,7 @@ import (
 
 const (
 	iso8601Format = "20060102T150405Z"
+	YYYYMMDD      = "20060102"
 )
 
 type RootUserConfig struct {
@@ -72,14 +76,32 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if len(credKv) != 2 {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
 		}
+		// Credential variables validation
 		creds := strings.Split(credKv[1], "/")
-		if len(creds) < 4 {
+		if len(creds) != 5 {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
 		}
+		if creds[4] != "aws4_request" {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureTerminationStr), &controllers.MetaOpts{Logger: logger})
+		}
+		if creds[3] != "s3" {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureIncorrService), &controllers.MetaOpts{Logger: logger})
+		}
+		if creds[2] != region {
+			return controllers.SendResponse(ctx, s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
+				HTTPStatusCode: http.StatusForbidden,
+			}, &controllers.MetaOpts{Logger: logger})
+		}
 
-		ctx.Locals("access", creds[0])
 		ctx.Locals("isRoot", creds[0] == root.Access)
 
+		_, err := time.Parse(YYYYMMDD, creds[1])
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
+		}
+
 		signHdrKv := strings.Split(authParts[1], "=")
 		if len(signHdrKv) != 2 {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
@@ -93,7 +115,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if err != nil {
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 		}
-		ctx.Locals("role", account.Role)
+		ctx.Locals("account", account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
@@ -107,6 +129,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
 		}
 
+		if date[:8] != creds[1] {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
+		}
+
+		// Validate the dates difference
+		err = validateDate(tdate)
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
 		hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
 		ok := isSpecialPayload(hashPayloadHeader)
 
@@ -133,6 +165,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			AccessKeyID:     creds[0],
 			SecretAccessKey: account.Secret,
 		}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
+			options.DisableURIPathEscaping = true
 			if debug {
 				options.LogSigning = true
 				options.Logger = logging.NewStandardLogger(os.Stderr)
@@ -165,6 +198,7 @@ type accounts struct {
 func (a accounts) getAccount(access string) (auth.Account, error) {
 	if access == a.root.Access {
 		return auth.Account{
+			Access: a.root.Access,
 			Secret: a.root.Secret,
 			Role:   "admin",
 		}, nil
@@ -185,3 +219,27 @@ func isSpecialPayload(str string) bool {
 
 	return specialValues[str]
 }
+
+func validateDate(date time.Time) error {
+	now := time.Now().UTC()
+	diff := date.Unix() - now.Unix()
+
+	// Checks the dates difference to be less than a minute
+	if math.Abs(float64(diff)) > 60 {
+		if diff > 0 {
+			return s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+				HTTPStatusCode: http.StatusForbidden,
+			}
+		} else {
+			return s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+				HTTPStatusCode: http.StatusForbidden,
+			}
+		}
+	}
+
+	return nil
+}

s3api/middlewares/url-decoder.go

@@ -0,0 +1,36 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/url"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		reqURL := ctx.Request().URI().String()
+		decoded, err := url.Parse(reqURL)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+		}
+		ctx.Path(decoded.Path)
+		return ctx.Next()
+	}
+}

s3api/router.go

@@ -23,20 +23,36 @@ import (
 	"github.com/versity/versitygw/s3log"
 )
 
-type S3ApiRouter struct{}
+type S3ApiRouter struct {
+	WithAdmSrv bool
+}
 
 func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
 	s3ApiController := controllers.New(be, iam, logger, evs)
-	adminController := controllers.AdminController{IAMService: iam}
 
-	app.Patch("/create-user", adminController.CreateUser)
+	if sa.WithAdmSrv {
+		adminController := controllers.NewAdminController(iam, be)
+
+		// CreateUser admin api
+		app.Patch("/create-user", adminController.CreateUser)
+
+		// DeleteUsers admin api
+		app.Patch("/delete-user", adminController.DeleteUser)
+
+		// ListUsers admin api
+		app.Patch("/list-users", adminController.ListUsers)
+
+		// ChangeBucketOwner admin api
+		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+		// ListBucketsAndOwners admin api
+		app.Patch("/list-buckets", adminController.ListBuckets)
+	}
 
-	// Admin Delete api
-	app.Patch("/delete-user", adminController.DeleteUser)
 	// ListBuckets action
 	app.Get("/", s3ApiController.ListBuckets)
 
-	// PutBucket action
+	// CreateBucket action
 	// PutBucketAcl action
 	app.Put("/:bucket", s3ApiController.PutBucketActions)
 
@@ -45,6 +61,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 
 	// HeadBucket
 	app.Head("/:bucket", s3ApiController.HeadBucket)
+
 	// GetBucketAcl action
 	// ListMultipartUploads action
 	// ListObjects action
@@ -53,22 +70,34 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 
 	// HeadObject action
 	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
+
 	// GetObjectAcl action
 	// GetObject action
 	// ListObjectParts action
+	// GetObjectTagging action
+	// ListParts action
+	// GetObjectAttributes action
 	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
+
 	// DeleteObject action
 	// AbortMultipartUpload action
+	// DeleteObjectTagging action
 	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
+
 	// DeleteObjects action
 	app.Post("/:bucket", s3ApiController.DeleteObjects)
+
 	// CompleteMultipartUpload action
 	// CreateMultipartUpload
 	// RestoreObject action
+	// SelectObjectContent action
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
+
 	// CopyObject action
 	// PutObject action
 	// UploadPart action
 	// UploadPartCopy action
+	// PutObjectTagging action
+	// PutObjectAcl action
 	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
 }

s3api/server.go

@@ -49,11 +49,13 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 
 	// Logging middlewares
 	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(l))
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
 	app.Use(middlewares.VerifyMD5Body(l))
+	app.Use(middlewares.AclParser(be, l))
 
 	server.router.Init(app, be, iam, l, evs)
 
@@ -68,6 +70,11 @@ func WithTLS(cert tls.Certificate) Option {
 	return func(s *S3ApiServer) { s.cert = &cert }
 }
 
+// WithAdminServer runs admin endpoints with the gateway in the same network
+func WithAdminServer() Option {
+	return func(s *S3ApiServer) { s.router.WithAdmSrv = true }
+}
+
 // WithDebug sets debug output
 func WithDebug() Option {
 	return func(s *S3ApiServer) { s.debug = true }

s3api/utils/utils.go

@@ -19,21 +19,32 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	bucketNameRegexp   = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]+[a-z0-9]$`)
+	bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
 )
 
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
-	headers.VisitAll(func(key, value []byte) {
-		if strings.HasPrefix(string(key), "X-Amz-Meta-") {
-			trimmedKey := strings.TrimPrefix(string(key), "X-Amz-Meta-")
+	headers.DisableNormalizing()
+	headers.VisitAllInOrder(func(key, value []byte) {
+		hKey := string(key)
+		if strings.HasPrefix(strings.ToLower(hKey), "x-amz-meta-") {
+			trimmedKey := hKey[11:]
 			headerValue := string(value)
 			metadata[trimmedKey] = headerValue
 		}
 	})
+	headers.EnableNormalizing()
 
 	return
 }
@@ -41,7 +52,7 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
 func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
 	req := ctx.Request()
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
+	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body()))
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -67,9 +78,22 @@ func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Reques
 }
 
 func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
+	ctx.Response().Header.DisableNormalizing()
 	for key, val := range meta {
-		ctx.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
+		ctx.Response().Header.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
 	}
+	ctx.Response().Header.EnableNormalizing()
+}
+
+func ParseUint(str string) (int32, error) {
+	if str == "" {
+		return 1000, nil
+	}
+	num, err := strconv.ParseUint(str, 10, 16)
+	if err != nil {
+		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+	}
+	return int32(num), nil
 }
 
 type CustomHeader struct {
@@ -83,6 +107,22 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
 	}
 }
 
+func IsValidBucketName(bucket string) bool {
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	// Checks to contain only digits, lowercase letters, dot, hyphen.
+	// Checks to start and end with only digits and lowercase letters.
+	if !bucketNameRegexp.MatchString(bucket) {
+		return false
+	}
+	// Checks not to be a valid IP address
+	if bucketNameIpRegexp.MatchString(bucket) {
+		return false
+	}
+	return true
+}
+
 func includeHeader(hdr string, signedHdrs []string) bool {
 	for _, shdr := range signedHdrs {
 		if strings.EqualFold(hdr, shdr) {

s3api/utils/utils_test.go

@@ -79,13 +79,6 @@ func TestGetUserMetaData(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 	req := ctx.Request()
 
-	// Case 2
-	ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
-	req2 := ctx2.Request()
-
-	req2.Header.Add("X-Amz-Meta-Name", "Nick")
-	req2.Header.Add("X-Amz-Meta-Age", "27")
-
 	tests := []struct {
 		name         string
 		args         args
@@ -98,16 +91,6 @@ func TestGetUserMetaData(t *testing.T) {
 			},
 			wantMetadata: map[string]string{},
 		},
-		{
-			name: "Success-non-empty-response",
-			args: args{
-				headers: &req2.Header,
-			},
-			wantMetadata: map[string]string{
-				"Age":  "27",
-				"Name": "Nick",
-			},
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -153,3 +136,128 @@ func Test_includeHeader(t *testing.T) {
 		})
 	}
 }
+
+func TestIsValidBucketName(t *testing.T) {
+	type args struct {
+		bucket string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "IsValidBucketName-short-name",
+			args: args{
+				bucket: "a",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-start-with-hyphen",
+			args: args{
+				bucket: "-bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-start-with-dot",
+			args: args{
+				bucket: ".bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-contain-invalid-character",
+			args: args{
+				bucket: "my@bucket",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-end-with-hyphen",
+			args: args{
+				bucket: "bucket-",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-end-with-dot",
+			args: args{
+				bucket: "bucket.",
+			},
+			want: false,
+		},
+		{
+			name: "IsValidBucketName-valid-bucket-name",
+			args: args{
+				bucket: "my-bucket",
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidBucketName(tt.args.bucket); got != tt.want {
+				t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseUint(t *testing.T) {
+	type args struct {
+		str string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    int32
+		wantErr bool
+	}{
+		{
+			name: "Parse-uint-empty-string",
+			args: args{
+				str: "",
+			},
+			want:    1000,
+			wantErr: false,
+		},
+		{
+			name: "Parse-uint-invalid-number-string",
+			args: args{
+				str: "bla",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-invalid-negative-number",
+			args: args{
+				str: "-5",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-success",
+			args: args{
+				str: "23",
+			},
+			want:    23,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseUint(tt.args.str)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseMaxKeys() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ParseMaxKeys() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -97,6 +97,9 @@ const (
 	ErrNegativeExpires
 	ErrMaximumExpires
 	ErrSignatureDoesNotMatch
+	ErrSignatureDateDoesNotMatch
+	ErrSignatureTerminationStr
+	ErrSignatureIncorrService
 	ErrContentSHA256Mismatch
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
@@ -106,9 +109,13 @@ const (
 	ErrNotImplemented
 	ErrPreconditionFailed
 	ErrInvalidObjectState
+	ErrInvalidRange
+	ErrInvalidURI
 
+	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
+	ErrDirectoryObjectContainsData
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -187,13 +194,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "We encountered an internal error, please try again.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
-
 	ErrInvalidPart: {
 		Code:           "InvalidPart",
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -284,7 +289,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Signature header missing Signature field.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
 		Description:    "There were headers present in the request which were not signed",
@@ -320,25 +324,36 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
 		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
 	ErrRequestNotReadyYet: {
 		Code:           "AccessDenied",
 		Description:    "Request is not valid yet",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
 	ErrSignatureDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
 		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
+	ErrSignatureDateDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureTerminationStr: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureIncorrService: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped to correct service: s3",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrContentSHA256Mismatch: {
 		Code:           "XAmzContentSHA256Mismatch",
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
@@ -374,6 +389,16 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The operation is not valid for the current state of the object",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidRange: {
+		Code:           "InvalidRange",
+		Description:    "The requested range is not valid for the request. Try another range.",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
+	ErrInvalidURI: {
+		Code:           "InvalidURI",
+		Description:    "The specified URI couldn't be parsed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrExistingObjectIsDirectory: {
 		Code:           "ExistingObjectIsDirectory",
 		Description:    "Existing Object is a directory.",
@@ -384,6 +409,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Object parent already exists as a file.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrDirectoryObjectContainsData: {
+		Code:           "DirectoryObjectContainsData",
+		Description:    "Directory object contains data payload.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3log/audit-logger.go

@@ -27,6 +27,8 @@ import (
 
 type AuditLogger interface {
 	Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
+	HangUp() error
+	Shutdown() error
 }
 
 type LogMeta struct {
@@ -36,7 +38,7 @@ type LogMeta struct {
 }
 
 type LogConfig struct {
-	IsFile     bool
+	LogFile    string
 	WebhookURL string
 }
 
@@ -70,14 +72,14 @@ type LogFields struct {
 }
 
 func InitLogger(cfg *LogConfig) (AuditLogger, error) {
-	if cfg.WebhookURL != "" && cfg.IsFile {
+	if cfg.WebhookURL != "" && cfg.LogFile != "" {
 		return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
 	}
 	if cfg.WebhookURL != "" {
 		return InitWebhookLogger(cfg.WebhookURL)
 	}
-	if cfg.IsFile {
-		return InitFileLogger()
+	if cfg.LogFile != "" {
+		return InitFileLogger(cfg.LogFile)
 	}
 
 	return nil, nil

s3log/file.go

@@ -16,9 +16,7 @@ package s3log
 
 import (
 	"crypto/tls"
-	"errors"
 	"fmt"
-	"io/fs"
 	"os"
 	"strings"
 	"sync"
@@ -29,38 +27,45 @@ import (
 )
 
 const (
-	logFile     = "access.log"
 	logFileMode = 0600
 	timeFormat  = "02/January/2006:15:04:05 -0700"
 )
 
+// FileLogger is a local file audit log
 type FileLogger struct {
-	LogFields
-	mu sync.Mutex
+	logfile string
+	f       *os.File
+	gotErr  bool
+	mu      sync.Mutex
 }
 
 var _ AuditLogger = &FileLogger{}
 
-func InitFileLogger() (AuditLogger, error) {
-	_, err := os.ReadFile(logFile)
-	if err != nil && errors.Is(err, fs.ErrNotExist) {
-		err := os.WriteFile(logFile, []byte{}, logFileMode)
-		if err != nil {
-			return nil, err
-		} else {
-			return nil, err
-		}
+// InitFileLogger initializes audit logs to local file
+func InitFileLogger(logname string) (AuditLogger, error) {
+	f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("open log: %w", err)
 	}
 
-	return &FileLogger{}, nil
+	f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return &FileLogger{logfile: logname, f: f}, nil
 }
 
+// Log sends log message to file logger
 func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
+	if f.gotErr {
+		return
+	}
+
+	lf := LogFields{}
+
 	access := "-"
-	reqURI := ctx.Request().URI().String()
+	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
 	bucket, object := path[1], strings.Join(path[2:], "/")
 	errorCode := ""
@@ -68,8 +73,8 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 	startTime := ctx.Locals("startTime").(time.Time)
 	tlsConnState := ctx.Context().TLSConnectionState()
 	if tlsConnState != nil {
-		f.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
-		f.TLSVersion = getTLSVersionName(tlsConnState.Version)
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
 	}
 
 	if err != nil {
@@ -88,117 +93,138 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 		access = ctx.Locals("access").(string)
 	}
 
-	f.BucketOwner = meta.BucketOwner
-	f.Bucket = bucket
-	f.Time = time.Now()
-	f.RemoteIP = ctx.IP()
-	f.Requester = access
-	f.RequestID = genID()
-	f.Operation = meta.Action
-	f.Key = object
-	f.RequestURI = reqURI
-	f.HttpStatus = httpStatus
-	f.ErrorCode = errorCode
-	f.BytesSent = len(body)
-	f.ObjectSize = meta.ObjectSize
-	f.TotalTime = time.Since(startTime).Milliseconds()
-	f.TurnAroundTime = time.Since(startTime).Milliseconds()
-	f.Referer = ctx.Get("Referer")
-	f.UserAgent = ctx.Get("User-Agent")
-	f.VersionID = ctx.Query("versionId")
-	f.HostID = ctx.Get("X-Amz-Id-2")
-	f.SignatureVersion = "SigV4"
-	f.AuthenticationType = "AuthHeader"
-	f.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
-	f.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
-	f.AclRequired = "Yes"
+	lf.BucketOwner = meta.BucketOwner
+	lf.Bucket = bucket
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.Key = object
+	lf.RequestURI = reqURI
+	lf.HttpStatus = httpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.ObjectSize = meta.ObjectSize
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.VersionID = ctx.Query("versionId")
+	lf.HostID = ctx.Get("X-Amz-Id-2")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
+	lf.AclRequired = "Yes"
 
-	f.writeLog()
+	f.writeLog(lf)
 }
 
-func (fl *FileLogger) writeLog() {
-	if fl.BucketOwner == "" {
-		fl.BucketOwner = "-"
+func (f *FileLogger) writeLog(lf LogFields) {
+	if lf.BucketOwner == "" {
+		lf.BucketOwner = "-"
 	}
-	if fl.Bucket == "" {
-		fl.Bucket = "-"
+	if lf.Bucket == "" {
+		lf.Bucket = "-"
 	}
-	if fl.RemoteIP == "" {
-		fl.RemoteIP = "-"
+	if lf.RemoteIP == "" {
+		lf.RemoteIP = "-"
 	}
-	if fl.Requester == "" {
-		fl.Requester = "-"
+	if lf.Requester == "" {
+		lf.Requester = "-"
 	}
-	if fl.Operation == "" {
-		fl.Operation = "-"
+	if lf.Operation == "" {
+		lf.Operation = "-"
 	}
-	if fl.Key == "" {
-		fl.Key = "-"
+	if lf.Key == "" {
+		lf.Key = "-"
 	}
-	if fl.RequestURI == "" {
-		fl.RequestURI = "-"
+	if lf.RequestURI == "" {
+		lf.RequestURI = "-"
 	}
-	if fl.ErrorCode == "" {
-		fl.ErrorCode = "-"
+	if lf.ErrorCode == "" {
+		lf.ErrorCode = "-"
 	}
-	if fl.Referer == "" {
-		fl.Referer = "-"
+	if lf.Referer == "" {
+		lf.Referer = "-"
 	}
-	if fl.UserAgent == "" {
-		fl.UserAgent = "-"
+	if lf.UserAgent == "" {
+		lf.UserAgent = "-"
 	}
-	if fl.VersionID == "" {
-		fl.VersionID = "-"
+	if lf.VersionID == "" {
+		lf.VersionID = "-"
 	}
-	if fl.HostID == "" {
-		fl.HostID = "-"
+	if lf.HostID == "" {
+		lf.HostID = "-"
 	}
-	if fl.CipherSuite == "" {
-		fl.CipherSuite = "-"
+	if lf.CipherSuite == "" {
+		lf.CipherSuite = "-"
 	}
-	if fl.HostHeader == "" {
-		fl.HostHeader = "-"
+	if lf.HostHeader == "" {
+		lf.HostHeader = "-"
 	}
-	if fl.TLSVersion == "" {
-		fl.TLSVersion = "-"
+	if lf.TLSVersion == "" {
+		lf.TLSVersion = "-"
 	}
 
-	log := fmt.Sprintf("\n%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v",
-		fl.BucketOwner,
-		fl.Bucket,
-		fmt.Sprintf("[%v]", fl.Time.Format(timeFormat)),
-		fl.RemoteIP,
-		fl.Requester,
-		fl.RequestID,
-		fl.Operation,
-		fl.Key,
-		fl.RequestURI,
-		fl.HttpStatus,
-		fl.ErrorCode,
-		fl.BytesSent,
-		fl.ObjectSize,
-		fl.TotalTime,
-		fl.TurnAroundTime,
-		fl.Referer,
-		fl.UserAgent,
-		fl.VersionID,
-		fl.HostID,
-		fl.SignatureVersion,
-		fl.CipherSuite,
-		fl.AuthenticationType,
-		fl.HostHeader,
-		fl.TLSVersion,
-		fl.AccessPointARN,
-		fl.AclRequired,
+	log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
+		lf.BucketOwner,
+		lf.Bucket,
+		fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
+		lf.RemoteIP,
+		lf.Requester,
+		lf.RequestID,
+		lf.Operation,
+		lf.Key,
+		lf.RequestURI,
+		lf.HttpStatus,
+		lf.ErrorCode,
+		lf.BytesSent,
+		lf.ObjectSize,
+		lf.TotalTime,
+		lf.TurnAroundTime,
+		lf.Referer,
+		lf.UserAgent,
+		lf.VersionID,
+		lf.HostID,
+		lf.SignatureVersion,
+		lf.CipherSuite,
+		lf.AuthenticationType,
+		lf.HostHeader,
+		lf.TLSVersion,
+		lf.AccessPointARN,
+		lf.AclRequired,
 	)
 
-	file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, logFileMode)
+	_, err := f.f.WriteString(log)
 	if err != nil {
-		fmt.Printf("error opening the log file: %v", err.Error())
-	}
-	defer file.Close()
-	_, err = file.WriteString(log)
-	if err != nil {
-		fmt.Printf("error writing in log file: %v", err.Error())
+		fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
+		// TODO: do we need to terminate on log error?
+		// set err for now so that we don't spew errors
+		f.gotErr = true
 	}
 }
+
+// HangUp closes current logfile handle and opens a new one
+// typically needed for log rotations
+func (f *FileLogger) HangUp() error {
+	err := f.f.Close()
+	if err != nil {
+		return fmt.Errorf("close log: %w", err)
+	}
+
+	f.f, err = os.OpenFile(f.logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("open log: %w", err)
+	}
+
+	f.f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
+
+	return nil
+}
+
+// Shutdown closes logfile handle
+func (f *FileLogger) Shutdown() error {
+	return f.f.Close()
+}

s3log/webhook.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -29,14 +30,15 @@ import (
 	"github.com/versity/versitygw/s3err"
 )
 
+// WebhookLogger is a webhook URL audit log
 type WebhookLogger struct {
-	LogFields
 	mu  sync.Mutex
 	url string
 }
 
 var _ AuditLogger = &WebhookLogger{}
 
+// InitWebhookLogger initializes audit logs to webhook URL
 func InitWebhookLogger(url string) (AuditLogger, error) {
 	client := &http.Client{
 		Timeout: 3 * time.Second,
@@ -44,7 +46,7 @@ func InitWebhookLogger(url string) (AuditLogger, error) {
 	_, err := client.Post(url, "application/json", nil)
 	if err != nil {
 		if err, ok := err.(net.Error); ok && !err.Timeout() {
-			return nil, fmt.Errorf("unreachable webhook url")
+			return nil, fmt.Errorf("unreachable webhook url: %w", err)
 		}
 	}
 	return &WebhookLogger{
@@ -52,12 +54,15 @@ func InitWebhookLogger(url string) (AuditLogger, error) {
 	}, nil
 }
 
+// Log sends log message to webhook
 func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 	wl.mu.Lock()
 	defer wl.mu.Unlock()
 
+	lf := LogFields{}
+
 	access := "-"
-	reqURI := ctx.Request().URI().String()
+	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
 	bucket, object := path[1], strings.Join(path[2:], "/")
 	errorCode := ""
@@ -65,8 +70,8 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 	startTime := ctx.Locals("startTime").(time.Time)
 	tlsConnState := ctx.Context().TLSConnectionState()
 	if tlsConnState != nil {
-		wl.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
-		wl.TLSVersion = getTLSVersionName(tlsConnState.Version)
+		lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
+		lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
 	}
 
 	if err != nil {
@@ -85,43 +90,43 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 		access = ctx.Locals("access").(string)
 	}
 
-	wl.BucketOwner = meta.BucketOwner
-	wl.Bucket = bucket
-	wl.Time = time.Now()
-	wl.RemoteIP = ctx.IP()
-	wl.Requester = access
-	wl.RequestID = genID()
-	wl.Operation = meta.Action
-	wl.Key = object
-	wl.RequestURI = reqURI
-	wl.HttpStatus = httpStatus
-	wl.ErrorCode = errorCode
-	wl.BytesSent = len(body)
-	wl.ObjectSize = meta.ObjectSize
-	wl.TotalTime = time.Since(startTime).Milliseconds()
-	wl.TurnAroundTime = time.Since(startTime).Milliseconds()
-	wl.Referer = ctx.Get("Referer")
-	wl.UserAgent = ctx.Get("User-Agent")
-	wl.VersionID = ctx.Query("versionId")
-	wl.HostID = ctx.Get("X-Amz-Id-2")
-	wl.SignatureVersion = "SigV4"
-	wl.AuthenticationType = "AuthHeader"
-	wl.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
-	wl.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
-	wl.AclRequired = "Yes"
+	lf.BucketOwner = meta.BucketOwner
+	lf.Bucket = bucket
+	lf.Time = time.Now()
+	lf.RemoteIP = ctx.IP()
+	lf.Requester = access
+	lf.RequestID = genID()
+	lf.Operation = meta.Action
+	lf.Key = object
+	lf.RequestURI = reqURI
+	lf.HttpStatus = httpStatus
+	lf.ErrorCode = errorCode
+	lf.BytesSent = len(body)
+	lf.ObjectSize = meta.ObjectSize
+	lf.TotalTime = time.Since(startTime).Milliseconds()
+	lf.TurnAroundTime = time.Since(startTime).Milliseconds()
+	lf.Referer = ctx.Get("Referer")
+	lf.UserAgent = ctx.Get("User-Agent")
+	lf.VersionID = ctx.Query("versionId")
+	lf.HostID = ctx.Get("X-Amz-Id-2")
+	lf.SignatureVersion = "SigV4"
+	lf.AuthenticationType = "AuthHeader"
+	lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
+	lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
+	lf.AclRequired = "Yes"
 
-	wl.sendLog()
+	wl.sendLog(lf)
 }
 
-func (wl *WebhookLogger) sendLog() {
-	jsonLog, err := json.Marshal(wl)
+func (wl *WebhookLogger) sendLog(lf LogFields) {
+	jsonLog, err := json.Marshal(lf)
 	if err != nil {
-		fmt.Printf("\n failed to parse the log data: %v", err.Error())
+		fmt.Fprintf(os.Stderr, "failed to parse the log data: %v\n", err.Error())
 	}
 
 	req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
 	if err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 	}
 	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 
@@ -135,7 +140,17 @@ func makeRequest(req *http.Request) {
 	_, err := client.Do(req)
 	if err != nil {
 		if err, ok := err.(net.Error); ok && !err.Timeout() {
-			fmt.Println("error sending the log to the specified url")
+			fmt.Fprintf(os.Stderr, "error sending webhook log: %v\n", err)
 		}
 	}
 }
+
+// HangUp does nothing for webhooks
+func (wl *WebhookLogger) HangUp() error {
+	return nil
+}
+
+// Shutdown does nothing for webhooks
+func (wl *WebhookLogger) Shutdown() error {
+	return nil
+}

s3response/s3response.go

@@ -16,6 +16,8 @@ package s3response
 
 import (
 	"encoding/xml"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 // Part describes part metadata.
@@ -27,7 +29,7 @@ type Part struct {
 }
 
 // ListPartsResponse - s3 api list parts response.
-type ListPartsResponse struct {
+type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
 	Bucket   string
@@ -50,7 +52,7 @@ type ListPartsResponse struct {
 }
 
 // ListMultipartUploadsResponse - s3 api list multipart uploads response.
-type ListMultipartUploadsResponse struct {
+type ListMultipartUploadsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
 
 	Bucket             string
@@ -107,3 +109,33 @@ type TagSet struct {
 type Tagging struct {
 	TagSet TagSet `xml:"TagSet"`
 }
+
+type DeleteObjects struct {
+	Objects []types.ObjectIdentifier `xml:"Object"`
+}
+
+type DeleteObjectsResult struct {
+	Deleted []types.DeletedObject
+	Error   []types.Error
+}
+type SelectObjectContentPayload struct {
+	Expression          *string
+	ExpressionType      types.ExpressionType
+	RequestProgress     *types.RequestProgress
+	InputSerialization  *types.InputSerialization
+	OutputSerialization *types.OutputSerialization
+	ScanRange           *types.ScanRange
+}
+
+type SelectObjectContentResult struct {
+	Records  *types.RecordsEvent
+	Stats    *types.StatsEvent
+	Progress *types.ProgressEvent
+	Cont     *types.ContinuationEvent
+	End      *types.EndEvent
+}
+
+type Bucket struct {
+	Name  string `json:"name"`
+	Owner string `json:"owner"`
+}