feat: Added s3 proxy comparison to upload, download and throughput benchmark tests

Throughput performance test

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      dev-dependencies:
+        patterns:
+          - "*"

.github/workflows/functional.yml

@@ -7,15 +7,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-
     - name: Get Dependencies
       run: |
         go get -v -t -d ./...

.github/workflows/go.yml

@@ -7,15 +7,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-
     - name: Verify all files pass gofmt formatting
       run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
 

.github/workflows/goreleaser.yml

@@ -0,0 +1,31 @@
+name: goreleaser
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+  # packages: write
+  # issues: write
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}

.github/workflows/static.yml

@@ -1,23 +1,22 @@
 name: staticcheck
 on: pull_request
 jobs: 
+
   build: 
     name: Check
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: "1.20"
-        id: go
-      - 
-        name: "Set up repo"
-        uses: actions/checkout@v1
-        with: 
-          fetch-depth: 1
-      - 
-        name: "staticcheck"
-        uses: dominikh/staticcheck-action@v1.3.0
-        with: 
-          install-go: false
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 'stable'
+      id: go
+ 
+    - name: "staticcheck"
+      uses: dominikh/staticcheck-action@v1.3.0
+      with: 
+        install-go: false

.gitignore

@@ -34,3 +34,5 @@ VERSION
 *.tar.gz
 **/rand.data
 /profile.txt
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,51 @@
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - goos:
+      - linux
+      - darwin
+      # windows is untested, we can start doing windows releases
+      # if someone is interested in taking on testing
+      # - windows
+    main: ./cmd/versitygw
+    binary: ./cmd/versitygw
+    id: versitygw
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge '
+
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

README.md

@@ -1,4 +1,4 @@
-# The Versity Gateway:<br/>A High-Performance S3 to Storage System Translation Service
+# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
 
 <picture>
   <source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
@@ -8,10 +8,14 @@
 
  [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-**Current status:** Alpha, in development not yet suited for production use
+**Current status:** Beta: Most clients functional, work in progress for more test coverage. Issue reports welcome.
 
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
+* Share filesystem directory via S3 protocol
+* Simple to deploy S3 server with a single command
+* Protocol compatibility allows common access to files via posix or S3 
+
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.

auth/acl.go

@@ -17,7 +17,6 @@ package auth
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -105,7 +104,6 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 
 			if *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
-				fmt.Println(fullControlList)
 				for _, str := range fullControlList {
 					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
 				}
@@ -148,7 +146,7 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		}
 
 		// Check if the specified accounts exist
-		accList, err := checkIfAccountsExist(accs, iam)
+		accList, err := CheckIfAccountsExist(accs, iam)
 		if err != nil {
 			return nil, err
 		}
@@ -168,17 +166,21 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 	return result, nil
 }
 
-func checkIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
+func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
 	result := []string{}
 
 	for _, acc := range accs {
 		_, err := iam.GetUserAccount(acc)
-		if err != nil && err != ErrNoSuchUser {
+		if err != nil {
+			if err == ErrNoSuchUser {
+				result = append(result, acc)
+				continue
+			}
+			if err == ErrNotSupported {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+			}
 			return nil, fmt.Errorf("check user account: %w", err)
 		}
-		if err == ErrNoSuchUser {
-			result = append(result, acc)
-		}
 	}
 	return result, nil
 }
@@ -198,7 +200,7 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRoot bool) error {
+func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
@@ -237,29 +239,14 @@ func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRo
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
-func IsAdmin(access string, isRoot bool) error {
-	var data IAMConfig
-
+func IsAdmin(acct Account, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
 
-	file, err := os.ReadFile("users.json")
-	if err != nil {
-		return fmt.Errorf("unable to read config file: %w", err)
-	}
-
-	if err := json.Unmarshal(file, &data); err != nil {
-		return err
-	}
-
-	acc, ok := data.AccessAccounts[access]
-	if !ok {
-		return fmt.Errorf("user does not exist")
-	}
-
-	if acc.Role == "admin" {
+	if acct.Role == "admin" {
 		return nil
 	}
+
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }

auth/iam.go

@@ -16,21 +16,72 @@ package auth
 
 import (
 	"errors"
+	"time"
 )
 
 // Account is a gateway IAM account
 type Account struct {
-	Secret string `json:"secret"`
-	Role   string `json:"role"`
+	Access    string `json:"access"`
+	Secret    string `json:"secret"`
+	Role      string `json:"role"`
+	UserID    int    `json:"userID"`
+	GroupID   int    `json:"groupID"`
+	ProjectID int    `json:"projectID"`
 }
 
 // IAMService is the interface for all IAM service implementations
 //
 //go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
 type IAMService interface {
-	CreateAccount(access string, account Account) error
+	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
 	DeleteUserAccount(access string) error
+	ListUserAccounts() ([]Account, error)
+	Shutdown() error
 }
 
 var ErrNoSuchUser = errors.New("user not found")
+
+type Opts struct {
+	Dir            string
+	LDAPServerURL  string
+	LDAPBindDN     string
+	LDAPPassword   string
+	LDAPQueryBase  string
+	LDAPObjClasses string
+	LDAPAccessAtr  string
+	LDAPSecretAtr  string
+	LDAPRoleAtr    string
+	CacheDisable   bool
+	CacheTTL       int
+	CachePrune     int
+}
+
+func New(o *Opts) (IAMService, error) {
+	var svc IAMService
+	var err error
+
+	switch {
+	case o.Dir != "":
+		svc, err = NewInternal(o.Dir)
+	case o.LDAPServerURL != "":
+		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
+			o.LDAPObjClasses)
+	default:
+		// if no iam options selected, default to the single user mode
+		return IAMServiceSingle{}, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if o.CacheDisable {
+		return svc, nil
+	}
+
+	return NewCache(svc,
+		time.Duration(o.CacheTTL)*time.Second,
+		time.Duration(o.CachePrune)*time.Second), nil
+}

auth/iam_cache.go

@@ -0,0 +1,179 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+)
+
+// IAMCache is an in memory cache of the IAM accounts
+// with expiration. This helps to alleviate the load on
+// the real IAM service if the gateway is handling
+// many requests. This forwards account updates to the
+// underlying service, and returns cached results while
+// the in memory account is not expired.
+type IAMCache struct {
+	service  IAMService
+	iamcache *icache
+	cancel   context.CancelFunc
+}
+
+var _ IAMService = &IAMCache{}
+
+type item struct {
+	value Account
+	exp   time.Time
+}
+
+type icache struct {
+	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
+}
+
+func (i *icache) set(k string, v Account) {
+	cpy := v
+	i.Lock()
+	i.items[k] = item{
+		exp:   time.Now().Add(i.expire),
+		value: cpy,
+	}
+	i.Unlock()
+}
+
+func (i *icache) get(k string) (Account, bool) {
+	i.RLock()
+	v, ok := i.items[k]
+	i.RUnlock()
+	if !ok || !v.exp.After(time.Now()) {
+		return Account{}, false
+	}
+	return v.value, true
+}
+
+func (i *icache) Delete(k string) {
+	i.Lock()
+	delete(i.items, k)
+	i.Unlock()
+}
+
+func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
+	for {
+		if ctx.Err() != nil {
+			break
+		}
+
+		now := time.Now()
+
+		i.Lock()
+		// prune expired entries
+		for k, v := range i.items {
+			if now.After(v.exp) {
+				delete(i.items, k)
+			}
+		}
+		i.Unlock()
+
+		// sleep for the clean interval or context cancelation,
+		// whichever comes first
+		select {
+		case <-ctx.Done():
+		case <-time.After(interval):
+		}
+	}
+}
+
+// NewCache initializes an IAM cache for the provided service. The expireTime
+// is the duration a cache entry can be valid, and the cleanupInterval is
+// how often to scan cache and cleanup expired entries.
+func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
+	i := &IAMCache{
+		service: service,
+		iamcache: &icache{
+			items:  make(map[string]item),
+			expire: expireTime,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go i.iamcache.gcCache(ctx, cleanupInterval)
+	i.cancel = cancel
+
+	return i
+}
+
+// CreateAccount send create to IAM service and creates an account cache entry
+func (c *IAMCache) CreateAccount(account Account) error {
+	err := c.service.CreateAccount(account)
+	if err != nil {
+		return err
+	}
+
+	// we need a copy of account to be able to store beyond the
+	// lifetime of the request, otherwise Fiber will reuse and corrupt
+	// these entries
+	acct := Account{
+		Access: strings.Clone(account.Access),
+		Secret: strings.Clone(account.Secret),
+		Role:   strings.Clone(account.Role),
+	}
+
+	c.iamcache.set(acct.Access, acct)
+	return nil
+}
+
+// GetUserAccount retrieves the cache account if it is in the cache and not
+// expired. Otherwise retrieves from underlying IAM service and caches
+// result for the expire duration.
+func (c *IAMCache) GetUserAccount(access string) (Account, error) {
+	acct, found := c.iamcache.get(access)
+	if found {
+		return acct, nil
+	}
+
+	a, err := c.service.GetUserAccount(access)
+	if err != nil {
+		return Account{}, err
+	}
+
+	c.iamcache.set(access, a)
+	return a, nil
+}
+
+// DeleteUserAccount deletes account from IAM service and cache
+func (c *IAMCache) DeleteUserAccount(access string) error {
+	err := c.service.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.Delete(access)
+	return nil
+}
+
+// ListUserAccounts is a passthrough to the underlying service and
+// does not make use of the cache
+func (c *IAMCache) ListUserAccounts() ([]Account, error) {
+	return c.service.ListUserAccounts()
+}
+
+// Shutdown graceful termination of service
+func (c *IAMCache) Shutdown() error {
+	c.cancel()
+	return nil
+}

auth/iam_internal.go

@@ -16,47 +16,44 @@ package auth
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
-	"hash/crc32"
-	"sync"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+const (
+	iamFile       = "users.json"
+	iamBackupFile = "users.json.backup"
 )
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	storer Storer
-
-	mu     sync.RWMutex
-	accts  IAMConfig
-	serial uint32
+	dir string
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
 type UpdateAcctFunc func([]byte) ([]byte, error)
 
-// Storer is the interface to manage the peristent IAM data for the internal
-// IAM service
-type Storer interface {
-	InitIAM() error
-	GetIAM() ([]byte, error)
-	StoreIAM(UpdateAcctFunc) error
-}
-
-// IAMConfig stores all internal IAM accounts
-type IAMConfig struct {
+// iAMConfig stores all internal IAM accounts
+type iAMConfig struct {
 	AccessAccounts map[string]Account `json:"accessAccounts"`
 }
 
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(s Storer) (*IAMServiceInternal, error) {
+func NewInternal(dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		storer: s,
+		dir: dir,
 	}
 
-	err := i.updateCache()
+	err := i.initIAM()
 	if err != nil {
-		return nil, fmt.Errorf("refresh iam cache: %w", err)
+		return nil, fmt.Errorf("init iam: %w", err)
 	}
 
 	return i, nil
@@ -64,32 +61,23 @@ func NewInternal(s Storer) (*IAMServiceInternal, error) {
 
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
-func (s *IAMServiceInternal) CreateAccount(access string, account Account) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
-		var conf IAMConfig
-
-		if len(data) > 0 {
-			if err := json.Unmarshal(data, &conf); err != nil {
-				return nil, fmt.Errorf("failed to parse iam: %w", err)
-			}
-		} else {
-			conf = IAMConfig{AccessAccounts: map[string]Account{}}
+func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
 		}
 
-		_, ok := conf.AccessAccounts[access]
+		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
 			return nil, fmt.Errorf("account already exists")
 		}
-		conf.AccessAccounts[access] = account
+		conf.AccessAccounts[account.Access] = account
 
 		b, err := json.Marshal(conf)
 		if err != nil {
 			return nil, fmt.Errorf("failed to serialize iam: %w", err)
 		}
-		s.accts = conf
 
 		return b, nil
 	})
@@ -98,25 +86,12 @@ func (s *IAMServiceInternal) CreateAccount(access string, account Account) error
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	data, err := s.storer.GetIAM()
+	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
 	}
 
-	serial := crc32.ChecksumIEEE(data)
-	if serial != s.serial {
-		s.mu.RUnlock()
-		err := s.updateCache()
-		s.mu.RLock()
-		if err != nil {
-			return Account{}, fmt.Errorf("refresh iam cache: %w", err)
-		}
-	}
-
-	acct, ok := s.accts.AccessAccounts[access]
+	acct, ok := conf.AccessAccounts[access]
 	if !ok {
 		return Account{}, ErrNoSuchUser
 	}
@@ -124,47 +99,13 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
-// updateCache must be called with no locks held
-func (s *IAMServiceInternal) updateCache() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	data, err := s.storer.GetIAM()
-	if err != nil {
-		return fmt.Errorf("get iam data: %w", err)
-	}
-
-	serial := crc32.ChecksumIEEE(data)
-
-	if len(data) > 0 {
-		if err := json.Unmarshal(data, &s.accts); err != nil {
-			return fmt.Errorf("failed to parse the config file: %w", err)
-		}
-	} else {
-		s.accts.AccessAccounts = make(map[string]Account)
-	}
-
-	s.serial = serial
-
-	return nil
-}
-
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
-		if len(data) == 0 {
-			// empty config, do nothing
-			return data, nil
-		}
-
-		var conf IAMConfig
-
-		if err := json.Unmarshal(data, &conf); err != nil {
-			return nil, fmt.Errorf("failed to parse iam: %w", err)
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
 		}
 
 		delete(conf.AccessAccounts, access)
@@ -174,8 +115,221 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 			return nil, fmt.Errorf("failed to serialize iam: %w", err)
 		}
 
-		s.accts = conf
-
 		return b, nil
 	})
 }
+
+// ListUserAccounts lists all the user accounts stored.
+func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getIAM()
+	if err != nil {
+		return []Account{}, fmt.Errorf("get iam data: %w", err)
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// Shutdown graceful termination of service
+func (s *IAMServiceInternal) Shutdown() error {
+	return nil
+}
+
+const (
+	iamMode = 0600
+)
+
+func (s *IAMServiceInternal) initIAM() error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	_, err := os.ReadFile(fname)
+	if errors.Is(err, fs.ErrNotExist) {
+		b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
+		if err != nil {
+			return fmt.Errorf("marshal default iam: %w", err)
+		}
+		err = os.WriteFile(fname, b, iamMode)
+		if err != nil {
+			return fmt.Errorf("write default iam: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
+	b, err := s.readIAMData()
+	if err != nil {
+		return iAMConfig{}, err
+	}
+
+	return parseIAM(b)
+}
+
+func parseIAM(b []byte) (iAMConfig, error) {
+	var conf iAMConfig
+	if err := json.Unmarshal(b, &conf); err != nil {
+		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
+	}
+
+	return conf, nil
+}
+
+const (
+	backoff  = 100 * time.Millisecond
+	maxretry = 300
+)
+
+func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
+	// We are going to be racing with other running gateways without any
+	// coordination. So we might find the file does not exist at times.
+	// For this case we need to retry for a while assuming the other gateway
+	// will eventually write the file. If it doesn't after the max retries,
+	// then we will return the error.
+
+	retries := 0
+
+	for {
+		b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+			return nil, fmt.Errorf("read iam file: %w", err)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		return b, nil
+	}
+}
+
+func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
+	// We are going to be racing with other running gateways without any
+	// coordination. So the strategy here is to read the current file data.
+	// If the file doesn't exist, then we assume someone else is currently
+	// updating the file. So we just need to keep retrying. We also need
+	// to make sure the data is consistent within a single update. So racing
+	// writes to a file would possibly leave this in some invalid state.
+	// We can get atomic updates with rename. If we read the data, update
+	// the data, write to a temp file, then rename the tempfile back to the
+	// data file. This should always result in a complete data image.
+
+	// There is at least one unsolved failure mode here.
+	// If a gateway removes the data file and then crashes, all other
+	// gateways will retry forever thinking that the original will eventually
+	// write the file.
+
+	retries := 0
+	fname := filepath.Join(s.dir, iamFile)
+
+	for {
+		b, err := os.ReadFile(fname)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			retries++
+			if retries < maxretry {
+				time.Sleep(backoff)
+				continue
+			}
+
+			// we have been unsuccessful trying to read the iam file
+			// so this must be the case where something happened and
+			// the file did not get updated successfully, and probably
+			// isn't going to be. The recovery procedure would be to
+			// copy the backup file into place of the original.
+			return fmt.Errorf("no iam file, needs backup recovery")
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("read iam file: %w", err)
+		}
+
+		// reset retries on successful read
+		retries = 0
+
+		err = os.Remove(iamFile)
+		if errors.Is(err, fs.ErrNotExist) {
+			// racing with someone else updating
+			// keep retrying after backoff
+			time.Sleep(backoff)
+			continue
+		}
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("remove old iam file: %w", err)
+		}
+
+		// save copy of data
+		datacopy := make([]byte, len(b))
+		copy(datacopy, b)
+
+		// make a backup copy in case we crash before update
+		// this is after remove, so there is a small window something
+		// can go wrong, but the remove should barrier other gateways
+		// from trying to write backup at the same time. Only one
+		// gateway will successfully remove the file.
+		os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
+
+		b, err = update(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return fmt.Errorf("update iam data: %w", err)
+		}
+
+		err = s.writeTempFile(b)
+		if err != nil {
+			// update failed, try to write old data back out
+			os.WriteFile(fname, datacopy, iamMode)
+			return err
+		}
+
+		break
+	}
+
+	return nil
+}
+
+func (s *IAMServiceInternal) writeTempFile(b []byte) error {
+	fname := filepath.Join(s.dir, iamFile)
+
+	f, err := os.CreateTemp(s.dir, iamFile)
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	_, err = f.Write(b)
+	if err != nil {
+		return fmt.Errorf("write temp file: %w", err)
+	}
+
+	err = os.Rename(f.Name(), fname)
+	if err != nil {
+		return fmt.Errorf("rename temp file: %w", err)
+	}
+
+	return nil
+}

auth/iam_ldap.go

@@ -0,0 +1,133 @@
+package auth
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-ldap/ldap/v3"
+)
+
+type LdapIAMService struct {
+	conn       *ldap.Conn
+	queryBase  string
+	objClasses []string
+	accessAtr  string
+	secretAtr  string
+	roleAtr    string
+}
+
+var _ IAMService = &LdapIAMService{}
+
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+		return nil, fmt.Errorf("required parameters list not fully provided")
+	}
+	conn, err := ldap.Dial("tcp", url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
+	}
+
+	err = conn.Bind(bindDN, pass)
+	if err != nil {
+		return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
+	}
+	return &LdapIAMService{
+		conn:       conn,
+		queryBase:  queryBase,
+		objClasses: strings.Split(objClasses, ","),
+		accessAtr:  accAtr,
+		secretAtr:  secAtr,
+		roleAtr:    roleAtr,
+	}, nil
+}
+
+func (ld *LdapIAMService) CreateAccount(account Account) error {
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry.Attribute("objectClass", ld.objClasses)
+	userEntry.Attribute(ld.accessAtr, []string{account.Access})
+	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
+	userEntry.Attribute(ld.roleAtr, []string{account.Role})
+
+	err := ld.conn.Add(userEntry)
+	if err != nil {
+		return fmt.Errorf("error adding an entry: %w", err)
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	result, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return Account{}, err
+	}
+
+	entry := result.Entries[0]
+	return Account{
+		Access: entry.GetAttributeValue(ld.accessAtr),
+		Secret: entry.GetAttributeValue(ld.secretAtr),
+		Role:   entry.GetAttributeValue(ld.roleAtr),
+	}, nil
+}
+
+func (ld *LdapIAMService) DeleteUserAccount(access string) error {
+	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+
+	err := ld.conn.Del(delReq)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
+	searchFilter := ""
+	for _, el := range ld.objClasses {
+		searchFilter += fmt.Sprintf("(objectClass=%v)", el)
+	}
+	searchRequest := ldap.NewSearchRequest(
+		ld.queryBase,
+		ldap.ScopeWholeSubtree,
+		ldap.NeverDerefAliases,
+		0,
+		0,
+		false,
+		fmt.Sprintf("(&%v)", searchFilter),
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		nil,
+	)
+
+	resp, err := ld.conn.Search(searchRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	result := []Account{}
+	for _, el := range resp.Entries {
+		result = append(result, Account{
+			Access: el.GetAttributeValue(ld.accessAtr),
+			Secret: el.GetAttributeValue(ld.secretAtr),
+			Role:   el.GetAttributeValue(ld.roleAtr),
+		})
+	}
+
+	return result, nil
+}
+
+// Shutdown graceful termination of service
+func (ld *LdapIAMService) Shutdown() error {
+	return ld.conn.Close()
+}

auth/iam_single.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"errors"
+)
+
+// IAMServiceSingle manages the single tenant (root-only) IAM service
+type IAMServiceSingle struct{}
+
+var _ IAMService = &IAMServiceSingle{}
+
+var ErrNotSupported = errors.New("method is not supported")
+
+// CreateAccount not valid in single tenant mode
+func (IAMServiceSingle) CreateAccount(account Account) error {
+	return ErrNotSupported
+}
+
+// GetUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	return Account{}, ErrNotSupported
+}
+
+// DeleteUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) DeleteUserAccount(access string) error {
+	return ErrNotSupported
+}
+
+// ListUserAccounts no accounts in single tenant mode
+func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
+	return []Account{}, nil
+}
+
+// Shutdown graceful termination of service
+func (IAMServiceSingle) Shutdown() error {
+	return nil
+}

backend/backend.go

@@ -29,13 +29,15 @@ type Backend interface {
 	fmt.Stringer
 	Shutdown()
 
-	ListBuckets(_ context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error)
+	// bucket operations
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
 	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
 	CreateBucket(context.Context, *s3.CreateBucketInput) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
 	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
 
+	// multipart operations
 	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
@@ -43,8 +45,8 @@ type Backend interface {
 	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
 	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
 	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
-	SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
 
+	// standard object operations
 	PutObject(context.Context, *s3.PutObjectInput) (string, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
@@ -56,11 +58,19 @@ type Backend interface {
 	DeleteObject(context.Context, *s3.DeleteObjectInput) error
 	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
-	RestoreObject(context.Context, *s3.RestoreObjectInput) error
 
-	GetTags(_ context.Context, bucket, object string) (map[string]string, error)
-	SetTags(_ context.Context, bucket, object string, tags map[string]string) error
-	RemoveTags(_ context.Context, bucket, object string) error
+	// special case object operations
+	RestoreObject(context.Context, *s3.RestoreObjectInput) error
+	SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
+
+	// object tags operations
+	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
+	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
+	DeleteObjectTagging(_ context.Context, bucket, object string) error
+
+	// non AWS actions
+	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
+	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
 type BackendUnsupported struct{}
@@ -77,32 +87,20 @@ func (BackendUnsupported) String() string {
 func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
+func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
-	return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
@@ -123,20 +121,17 @@ func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3resp
 func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
 	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
 func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
 	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
-	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
@@ -154,13 +149,36 @@ func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s
 func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
-func (BackendUnsupported) GetTags(_ context.Context, bucket, object string) (map[string]string, error) {
+func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
+	return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) SetTags(_ context.Context, bucket, object string, tags map[string]string) error {
+func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) RemoveTags(_ context.Context, bucket, object string) error {
+func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
+	return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+}

backend/common.go

@@ -61,9 +61,9 @@ var (
 
 // ParseRange parses input range header and returns startoffset, length, and
 // error. If no endoffset specified, then length is set to -1.
-func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
+func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, file.Size(), nil
+		return 0, fi.Size(), nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")

backend/posix/posix.go

@@ -29,9 +29,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -48,29 +46,19 @@ type Posix struct {
 
 	rootfd  *os.File
 	rootdir string
-
-	mu        sync.RWMutex
-	iamcache  []byte
-	iamvalid  bool
-	iamexpire time.Time
 }
 
 var _ backend.Backend = &Posix{}
 
-var (
-	cacheDuration = 5 * time.Minute
-)
-
 const (
 	metaTmpDir          = ".sgwtmp"
 	metaTmpMultipartDir = metaTmpDir + "/multipart"
 	onameAttr           = "user.objname"
 	tagHdr              = "X-Amz-Tagging"
+	metaHdr             = "X-Amz-Meta"
 	contentTypeHdr      = "content-type"
 	contentEncHdr       = "content-encoding"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
-	iamFile             = "users.json"
-	iamBackupFile       = "users.json.backup"
 	aclkey              = "user.acl"
 	etagkey             = "user.etag"
 )
@@ -97,7 +85,7 @@ func (p *Posix) String() string {
 	return "Posix Gateway"
 }
 
-func (p *Posix) ListBuckets(_ context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) {
+func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
 	entries, err := os.ReadDir(".")
 	if err != nil {
 		return s3response.ListAllMyBucketsResult{},
@@ -117,10 +105,32 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isRoot bool) (s3res
 			continue
 		}
 
-		buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-			Name:         entry.Name(),
-			CreationDate: fi.ModTime(),
-		})
+		// return all the buckets for admin users
+		if isAdmin {
+			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+				Name:         entry.Name(),
+				CreationDate: fi.ModTime(),
+			})
+			continue
+		}
+
+		aclTag, err := xattr.Get(entry.Name(), aclkey)
+		if err != nil {
+			return s3response.ListAllMyBucketsResult{}, fmt.Errorf("get acl tag: %w", err)
+		}
+
+		var acl auth.ACL
+		err = json.Unmarshal(aclTag, &acl)
+		if err != nil {
+			return s3response.ListAllMyBucketsResult{}, fmt.Errorf("parse acl tag: %w", err)
+		}
+
+		if acl.Owner == owner {
+			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+				Name:         entry.Name(),
+				CreationDate: fi.ModTime(),
+			})
+		}
 	}
 
 	sort.Sort(backend.ByBucketName(buckets))
@@ -129,6 +139,9 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isRoot bool) (s3res
 		Buckets: s3response.ListAllMyBucketsList{
 			Bucket: buckets,
 		},
+		Owner: s3response.CanonicalUser{
+			ID: owner,
+		},
 	}, nil
 }
 
@@ -210,6 +223,12 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
 		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
+	if strings.HasSuffix(*mpu.Key, "/") {
+		// directory objects can't be uploaded with mutlipart uploads
+		// because posix directories can't contain data
+		return nil, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData)
+	}
+
 	// generate random uuid for upload id
 	uploadID := uuid.New().String()
 	// hash object name for multipart container
@@ -393,13 +412,13 @@ func loadUserMetaData(path string, m map[string]string) (contentType, contentEnc
 		}
 		b, err := xattr.Get(path, e)
 		if err == syscall.ENODATA {
-			m[strings.TrimPrefix(e, "user.")] = ""
+			m[strings.TrimPrefix(e, fmt.Sprintf("user.%v.", metaHdr))] = ""
 			continue
 		}
 		if err != nil {
 			continue
 		}
-		m[strings.TrimPrefix(e, "user.")] = string(b)
+		m[strings.TrimPrefix(e, fmt.Sprintf("user.%v.", metaHdr))] = string(b)
 	}
 
 	b, err := xattr.Get(path, "user."+contentTypeHdr)
@@ -423,8 +442,22 @@ func loadUserMetaData(path string, m map[string]string) (contentType, contentEnc
 	return
 }
 
+func compareUserMetadata(meta1, meta2 map[string]string) bool {
+	if len(meta1) != len(meta2) {
+		return false
+	}
+
+	for key, val := range meta1 {
+		if meta2[key] != val {
+			return false
+		}
+	}
+
+	return true
+}
+
 func isValidMeta(val string) bool {
-	if strings.HasPrefix(val, "user.X-Amz-Meta") {
+	if strings.HasPrefix(val, "user."+metaHdr) {
 		return true
 	}
 	if strings.EqualFold(val, "user.Expires") {
@@ -533,6 +566,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 	objs, _ := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir))
 
 	var uploads []s3response.Upload
+	var resultUpds []s3response.Upload
 
 	var keyMarker string
 	if mpu.KeyMarker != nil {
@@ -542,12 +576,9 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 	if mpu.UploadIdMarker != nil {
 		uploadIDMarker = *mpu.UploadIdMarker
 	}
-	var pastMarker bool
-	if keyMarker == "" && uploadIDMarker == "" {
-		pastMarker = true
-	}
+	keyMarkerInd, uploadIdMarkerFound := -1, false
 
-	for i, obj := range objs {
+	for _, obj := range objs {
 		if !obj.IsDir() {
 			continue
 		}
@@ -566,22 +597,14 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 			continue
 		}
 
-		for j, upid := range upids {
+		for _, upid := range upids {
 			if !upid.IsDir() {
 				continue
 			}
 
-			if objectName == keyMarker || upid.Name() == uploadIDMarker {
-				pastMarker = true
-				continue
-			}
-			if keyMarker != "" && uploadIDMarker != "" && !pastMarker {
-				continue
-			}
-
-			userMetaData := make(map[string]string)
-			upiddir := filepath.Join(bucket, metaTmpMultipartDir, obj.Name(), upid.Name())
-			loadUserMetaData(upiddir, userMetaData)
+			// userMetaData := make(map[string]string)
+			// upiddir := filepath.Join(bucket, metaTmpMultipartDir, obj.Name(), upid.Name())
+			// loadUserMetaData(upiddir, userMetaData)
 
 			fi, err := upid.Info()
 			if err != nil {
@@ -589,28 +612,61 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 			}
 
 			uploadID := upid.Name()
+			if !uploadIdMarkerFound && uploadIDMarker == uploadID {
+				uploadIdMarkerFound = true
+			}
+			if keyMarkerInd == -1 && objectName == keyMarker {
+				keyMarkerInd = len(uploads)
+			}
 			uploads = append(uploads, s3response.Upload{
 				Key:       objectName,
 				UploadID:  uploadID,
 				Initiated: fi.ModTime().Format(backend.RFC3339TimeFormat),
 			})
-			if len(uploads) == int(mpu.MaxUploads) {
-				return s3response.ListMultipartUploadsResult{
-					Bucket:             bucket,
-					Delimiter:          delimiter,
-					IsTruncated:        i != len(objs) || j != len(upids),
-					KeyMarker:          keyMarker,
-					MaxUploads:         int(mpu.MaxUploads),
-					NextKeyMarker:      objectName,
-					NextUploadIDMarker: uploadID,
-					Prefix:             prefix,
-					UploadIDMarker:     uploadIDMarker,
-					Uploads:            uploads,
-				}, nil
-			}
 		}
 	}
 
+	if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) {
+		return s3response.ListMultipartUploadsResult{
+			Bucket:         bucket,
+			Delimiter:      delimiter,
+			KeyMarker:      keyMarker,
+			MaxUploads:     int(mpu.MaxUploads),
+			Prefix:         prefix,
+			UploadIDMarker: uploadIDMarker,
+			Uploads:        []s3response.Upload{},
+		}, nil
+	}
+
+	sort.SliceStable(uploads, func(i, j int) bool {
+		return uploads[i].Key < uploads[j].Key
+	})
+
+	for i := keyMarkerInd + 1; i < len(uploads); i++ {
+		if mpu.MaxUploads == 0 {
+			break
+		}
+		if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker {
+			continue
+		}
+		if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) {
+			return s3response.ListMultipartUploadsResult{
+				Bucket:             bucket,
+				Delimiter:          delimiter,
+				KeyMarker:          keyMarker,
+				MaxUploads:         int(mpu.MaxUploads),
+				NextKeyMarker:      resultUpds[i-1].Key,
+				NextUploadIDMarker: resultUpds[i-1].UploadID,
+				IsTruncated:        true,
+				Prefix:             prefix,
+				UploadIDMarker:     uploadIDMarker,
+				Uploads:            resultUpds,
+			}, nil
+		}
+
+		resultUpds = append(resultUpds, uploads[i])
+	}
+
 	return s3response.ListMultipartUploadsResult{
 		Bucket:         bucket,
 		Delimiter:      delimiter,
@@ -618,7 +674,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 		MaxUploads:     int(mpu.MaxUploads),
 		Prefix:         prefix,
 		UploadIDMarker: uploadIDMarker,
-		Uploads:        uploads,
+		Uploads:        resultUpds,
 	}, nil
 }
 
@@ -833,7 +889,7 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 	}
 
 	if startOffset+length > fi.Size()+1 {
-		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest)
+		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRange)
 	}
 
 	f, err := openTmpFile(filepath.Join(*upi.Bucket, objdir),
@@ -899,6 +955,9 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 			if len(p) != 2 {
 				return "", s3err.GetAPIError(s3err.ErrInvalidTag)
 			}
+			if len(p[0]) > 128 || len(p[1]) > 256 {
+				return "", s3err.GetAPIError(s3err.ErrInvalidTag)
+			}
 			tags[p[0]] = p[1]
 		}
 	}
@@ -907,13 +966,20 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 
 	if strings.HasSuffix(*po.Key, "/") {
 		// object is directory
+		if po.ContentLength != 0 {
+			// posix directories can't contain data, send error
+			// if reuests has a data payload associated with a
+			// directory object
+			return "", s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData)
+		}
+
 		err = mkdirAll(name, os.FileMode(0755), *po.Bucket, *po.Key)
 		if err != nil {
 			return "", err
 		}
 
 		for k, v := range po.Metadata {
-			xattr.Set(name, "user."+k, []byte(v))
+			xattr.Set(name, fmt.Sprintf("user.%v.%v", metaHdr, k), []byte(v))
 		}
 
 		// set etag attribute to signify this dir was specifically put
@@ -923,6 +989,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 	}
 
 	// object is file
+	d, err := os.Stat(name)
+	if err == nil && d.IsDir() {
+		return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+	}
+
 	f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
 		*po.Bucket, *po.Key, po.ContentLength)
 	if err != nil {
@@ -950,11 +1021,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 	}
 
 	for k, v := range po.Metadata {
-		xattr.Set(name, "user."+k, []byte(v))
+		xattr.Set(name, fmt.Sprintf("user.%v.%v", metaHdr, k), []byte(v))
 	}
 
 	if tagsStr != "" {
-		err := p.SetTags(ctx, *po.Bucket, *po.Key, tags)
+		err := p.PutObjectTagging(ctx, *po.Bucket, *po.Key, tags)
 		if err != nil {
 			return "", err
 		}
@@ -1055,16 +1126,12 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 
 	return s3response.DeleteObjectsResult{
 		Deleted: delResult,
-		Errors:  errs,
+		Error:   errs,
 	}, nil
 }
 
 func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
-	object := *input.Key
-	acceptRange := *input.Range
-	var contentRange string
-
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1073,6 +1140,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
+	object := *input.Key
 	objPath := filepath.Join(bucket, object)
 	fi, err := os.Stat(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1082,21 +1150,59 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
 
+	acceptRange := *input.Range
 	startOffset, length, err := backend.ParseRange(fi, acceptRange)
 	if err != nil {
 		return nil, err
 	}
 
+	objSize := fi.Size()
+	if fi.IsDir() {
+		// directory objects are always 0 len
+		objSize = 0
+		length = 0
+	}
+
 	if length == -1 {
-		length = fi.Size() - startOffset + 1
+		length = objSize - startOffset + 1
 	}
 
-	if startOffset+length > fi.Size()+1 {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	if startOffset+length > objSize+1 {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRange)
 	}
 
+	var contentRange string
 	if acceptRange != "" {
-		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, fi.Size())
+		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
+	}
+
+	if fi.IsDir() {
+		userMetaData := make(map[string]string)
+
+		contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
+
+		b, err := xattr.Get(objPath, etagkey)
+		etag := string(b)
+		if err != nil {
+			etag = ""
+		}
+
+		tags, err := p.getXattrTags(bucket, object)
+		if err != nil {
+			return nil, fmt.Errorf("get object tags: %w", err)
+		}
+
+		return &s3.GetObjectOutput{
+			AcceptRanges:    &acceptRange,
+			ContentLength:   length,
+			ContentEncoding: &contentEncoding,
+			ContentType:     &contentType,
+			ETag:            &etag,
+			LastModified:    backend.GetTimePtr(fi.ModTime()),
+			Metadata:        userMetaData,
+			TagCount:        int32(len(tags)),
+			ContentRange:    &contentRange,
+		}, nil
 	}
 
 	f, err := os.Open(objPath)
@@ -1189,6 +1295,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 	}
 	dstBucket := *input.Bucket
 	dstObject := *input.Key
+	owner := *input.ExpectedBucketOwner
 
 	_, err := os.Stat(srcBucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1206,6 +1313,22 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
+	dstBucketACLBytes, err := xattr.Get(dstBucket, aclkey)
+	if err != nil {
+		return nil, fmt.Errorf("get dst bucket acl tag: %w", err)
+	}
+
+	var dstBucketACL auth.ACL
+	err = json.Unmarshal(dstBucketACLBytes, &dstBucketACL)
+	if err != nil {
+		return nil, fmt.Errorf("parse dst bucket acl: %w", err)
+	}
+
+	err = auth.VerifyACL(dstBucketACL, owner, types.PermissionWrite, false)
+	if err != nil {
+		return nil, err
+	}
+
 	objPath := filepath.Join(srcBucket, srcObject)
 	f, err := os.Open(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1221,12 +1344,29 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
 
-	etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size()})
+	meta := make(map[string]string)
+	loadUserMetaData(objPath, meta)
+
+	dstObjdPath := filepath.Join(dstBucket, dstObject)
+	if dstObjdPath == objPath {
+		if compareUserMetadata(meta, input.Metadata) {
+			return &s3.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+		} else {
+			for key := range meta {
+				xattr.Remove(dstObjdPath, key)
+			}
+			for k, v := range input.Metadata {
+				xattr.Set(dstObjdPath, fmt.Sprintf("user.%v.%v", metaHdr, k), []byte(v))
+			}
+		}
+	}
+
+	etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size(), Metadata: meta})
 	if err != nil {
 		return nil, err
 	}
 
-	fi, err := os.Stat(filepath.Join(dstBucket, dstObject))
+	fi, err := os.Stat(dstObjdPath)
 	if err != nil {
 		return nil, fmt.Errorf("stat dst object: %w", err)
 	}
@@ -1406,7 +1546,7 @@ func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]
 	return b, nil
 }
 
-func (p *Posix) GetTags(_ context.Context, bucket, object string) (map[string]string, error) {
+func (p *Posix) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1439,7 +1579,7 @@ func (p *Posix) getXattrTags(bucket, object string) (map[string]string, error) {
 	return tags, nil
 }
 
-func (p *Posix) SetTags(_ context.Context, bucket, object string, tags map[string]string) error {
+func (p *Posix) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1475,200 +1615,83 @@ func (p *Posix) SetTags(_ context.Context, bucket, object string, tags map[strin
 	return nil
 }
 
-func (p *Posix) RemoveTags(ctx context.Context, bucket, object string) error {
-	return p.SetTags(ctx, bucket, object, nil)
+func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	return p.PutObjectTagging(ctx, bucket, object, nil)
 }
 
-const (
-	iamMode = 0600
-)
-
-func (p *Posix) InitIAM() error {
-	p.mu.RLock()
-	defer p.mu.RUnlock()
-
-	_, err := os.ReadFile(iamFile)
+func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
+	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		b, err := json.Marshal(auth.IAMConfig{AccessAccounts: map[string]auth.Account{}})
-		if err != nil {
-			return fmt.Errorf("marshal default iam: %w", err)
-		}
-		err = os.WriteFile(iamFile, b, iamMode)
-		if err != nil {
-			return fmt.Errorf("write default iam: %w", err)
-		}
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	aclTag, err := xattr.Get(bucket, aclkey)
+	if err != nil {
+		return fmt.Errorf("get acl: %w", err)
+	}
+
+	var acl auth.ACL
+	err = json.Unmarshal(aclTag, &acl)
+	if err != nil {
+		return fmt.Errorf("unmarshal acl: %w", err)
+	}
+
+	acl.Owner = newOwner
+
+	newAcl, err := json.Marshal(acl)
+	if err != nil {
+		return fmt.Errorf("marshal acl: %w", err)
+	}
+
+	err = xattr.Set(bucket, aclkey, newAcl)
+	if err != nil {
+		return fmt.Errorf("set acl: %w", err)
 	}
 
 	return nil
 }
 
-func (p *Posix) GetIAM() ([]byte, error) {
-	p.mu.RLock()
-	defer p.mu.RUnlock()
-
-	if !p.iamvalid || !p.iamexpire.After(time.Now()) {
-		p.mu.RUnlock()
-		err := p.refreshIAM()
-		p.mu.RLock()
-		if err != nil {
-			return nil, err
-		}
+func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
+	entries, err := os.ReadDir(".")
+	if err != nil {
+		return buckets, fmt.Errorf("readdir buckets: %w", err)
 	}
 
-	return p.iamcache, nil
-}
-
-const (
-	backoff  = 100 * time.Millisecond
-	maxretry = 300
-)
-
-func (p *Posix) refreshIAM() error {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	// We are going to be racing with other running gateways without any
-	// coordination. So we might find the file does not exist at times.
-	// For this case we need to retry for a while assuming the other gateway
-	// will eventually write the file. If it doesn't after the max retries,
-	// then we will return the error.
-
-	retries := 0
-
-	for {
-		b, err := os.ReadFile(iamFile)
-		if errors.Is(err, fs.ErrNotExist) {
-			// racing with someone else updating
-			// keep retrying after backoff
-			retries++
-			if retries < maxretry {
-				time.Sleep(backoff)
-				continue
-			}
-			return fmt.Errorf("read iam file: %w", err)
-		}
-		if err != nil {
-			return err
-		}
-
-		p.iamcache = b
-		p.iamvalid = true
-		p.iamexpire = time.Now().Add(cacheDuration)
-		break
-	}
-
-	return nil
-}
-
-func (p *Posix) StoreIAM(update auth.UpdateAcctFunc) error {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	// We are going to be racing with other running gateways without any
-	// coordination. So the strategy here is to read the current file data.
-	// If the file doesn't exist, then we assume someone else is currently
-	// updating the file. So we just need to keep retrying. We also need
-	// to make sure the data is consistent within a single update. So racing
-	// writes to a file would possibly leave this in some invalid state.
-	// We can get atomic updates with rename. If we read the data, update
-	// the data, write to a temp file, then rename the tempfile back to the
-	// data file. This should always result in a complete data image.
-
-	// There is at least one unsolved failure mode here.
-	// If a gateway removes the data file and then crashes, all other
-	// gateways will retry forever thinking that the original will eventually
-	// write the file.
-
-	retries := 0
-
-	for {
-		b, err := os.ReadFile(iamFile)
-		if errors.Is(err, fs.ErrNotExist) {
-			// racing with someone else updating
-			// keep retrying after backoff
-			retries++
-			if retries < maxretry {
-				time.Sleep(backoff)
-				continue
-			}
-
-			// we have been unsuccessful trying to read the iam file
-			// so this must be the case where something happened and
-			// the file did not get updated successfully, and probably
-			// isn't going to be. The recovery procedure would be to
-			// copy the backup file into place of the original.
-			return fmt.Errorf("no iam file, needs backup recovery")
-		}
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("read iam file: %w", err)
-		}
-
-		// reset retries on successful read
-		retries = 0
-
-		err = os.Remove(iamFile)
-		if errors.Is(err, fs.ErrNotExist) {
-			// racing with someone else updating
-			// keep retrying after backoff
-			time.Sleep(backoff)
+	for _, entry := range entries {
+		if !entry.IsDir() {
 			continue
 		}
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("remove old iam file: %w", err)
-		}
 
-		// save copy of data
-		datacopy := make([]byte, len(b))
-		copy(datacopy, b)
-
-		// make a backup copy in case we crash before update
-		// this is after remove, so there is a small window something
-		// can go wrong, but the remove should barrier other gateways
-		// from trying to write backup at the same time. Only one
-		// gateway will successfully remove the file.
-		os.WriteFile(iamBackupFile, b, iamMode)
-
-		b, err = update(b)
+		fi, err := entry.Info()
 		if err != nil {
-			// update failed, try to write old data back out
-			os.WriteFile(iamFile, datacopy, iamMode)
-			return fmt.Errorf("update iam data: %w", err)
+			continue
 		}
 
-		err = writeTempFile(b)
+		aclTag, err := xattr.Get(entry.Name(), aclkey)
 		if err != nil {
-			// update failed, try to write old data back out
-			os.WriteFile(iamFile, datacopy, iamMode)
-			return err
+			return buckets, fmt.Errorf("get acl tag: %w", err)
 		}
 
-		p.iamcache = b
-		p.iamvalid = true
-		p.iamexpire = time.Now().Add(cacheDuration)
-		break
+		var acl auth.ACL
+		err = json.Unmarshal(aclTag, &acl)
+		if err != nil {
+			return buckets, fmt.Errorf("parse acl tag: %w", err)
+		}
+
+		buckets = append(buckets, s3response.Bucket{
+			Name:  fi.Name(),
+			Owner: acl.Owner,
+		})
 	}
 
-	return nil
-}
+	sort.SliceStable(buckets, func(i, j int) bool {
+		return buckets[i].Name < buckets[j].Name
+	})
 
-func writeTempFile(b []byte) error {
-	f, err := os.CreateTemp(".", iamFile)
-	if err != nil {
-		return fmt.Errorf("create temp file: %w", err)
-	}
-	defer os.Remove(f.Name())
-
-	_, err = f.Write(b)
-	if err != nil {
-		return fmt.Errorf("write temp file: %w", err)
-	}
-
-	err = os.Rename(f.Name(), iamFile)
-	if err != nil {
-		return fmt.Errorf("rename temp file: %w", err)
-	}
-
-	return nil
+	return buckets, nil
 }
 
 func isNoAttr(err error) bool {

backend/s3proxy/client.go

@@ -0,0 +1,86 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+	"github.com/versity/versitygw/auth"
+)
+
+func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		return nil, fmt.Errorf("invalid account in context")
+	}
+
+	cfg, err := s.getConfig(ctx, acct.Access, acct.Secret)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg), nil
+}
+
+func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.disableChecksum {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(ctx, opts...)
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *S3be) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}

backend/s3proxy/s3.go

@@ -0,0 +1,500 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3proxy
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3response"
+)
+
+type S3be struct {
+	backend.BackendUnsupported
+
+	endpoint        string
+	awsRegion       string
+	disableChecksum bool
+	sslSkipVerify   bool
+	debug           bool
+}
+
+func New(endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
+	return &S3be{
+		endpoint:        endpoint,
+		awsRegion:       region,
+		disableChecksum: disableChecksum,
+		sslSkipVerify:   sslSkipVerify,
+		debug:           debug,
+	}
+}
+
+func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, err
+	}
+
+	output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
+	if err != nil {
+		return s3response.ListAllMyBucketsResult{}, err
+	}
+
+	var buckets []s3response.ListAllMyBucketsEntry
+	for _, b := range output.Buckets {
+		buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+			Name:         *b.Name,
+			CreationDate: *b.CreationDate,
+		})
+	}
+
+	return s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: buckets,
+		},
+	}, nil
+}
+
+func (s *S3be) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.HeadBucket(ctx, input)
+}
+
+func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.CreateBucket(ctx, input)
+	return err
+}
+
+func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteBucket(ctx, input)
+	return err
+}
+
+func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.CreateMultipartUpload(ctx, input)
+}
+
+func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.CompleteMultipartUpload(ctx, input)
+}
+
+func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.AbortMultipartUpload(ctx, input)
+	return err
+}
+
+const (
+	iso8601Format = "20060102T150405Z"
+)
+
+func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, err
+	}
+
+	output, err := client.ListMultipartUploads(ctx, input)
+	if err != nil {
+		return s3response.ListMultipartUploadsResult{}, err
+	}
+
+	var uploads []s3response.Upload
+	for _, u := range output.Uploads {
+		uploads = append(uploads, s3response.Upload{
+			Key:      *u.Key,
+			UploadID: *u.UploadId,
+			Initiator: s3response.Initiator{
+				ID:          *u.Initiator.ID,
+				DisplayName: *u.Initiator.DisplayName,
+			},
+			Owner: s3response.Owner{
+				ID:          *u.Owner.ID,
+				DisplayName: *u.Owner.DisplayName,
+			},
+			StorageClass: string(u.StorageClass),
+			Initiated:    u.Initiated.Format(iso8601Format),
+		})
+	}
+
+	var cps []s3response.CommonPrefix
+	for _, c := range output.CommonPrefixes {
+		cps = append(cps, s3response.CommonPrefix{
+			Prefix: *c.Prefix,
+		})
+	}
+
+	return s3response.ListMultipartUploadsResult{
+		Bucket:             *output.Bucket,
+		KeyMarker:          *output.KeyMarker,
+		UploadIDMarker:     *output.UploadIdMarker,
+		NextKeyMarker:      *output.NextKeyMarker,
+		NextUploadIDMarker: *output.NextUploadIdMarker,
+		Delimiter:          *output.Delimiter,
+		Prefix:             *output.Prefix,
+		EncodingType:       string(output.EncodingType),
+		MaxUploads:         int(output.MaxUploads),
+		IsTruncated:        output.IsTruncated,
+		Uploads:            uploads,
+		CommonPrefixes:     cps,
+	}, nil
+}
+
+func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.ListPartsResult{}, err
+	}
+
+	output, err := client.ListParts(ctx, input)
+	if err != nil {
+		return s3response.ListPartsResult{}, err
+	}
+
+	var parts []s3response.Part
+	for _, p := range output.Parts {
+		parts = append(parts, s3response.Part{
+			PartNumber:   int(p.PartNumber),
+			LastModified: p.LastModified.Format(iso8601Format),
+			ETag:         *p.ETag,
+			Size:         p.Size,
+		})
+	}
+	pnm, err := strconv.Atoi(*output.PartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse part number marker: %w", err)
+	}
+
+	npmn, err := strconv.Atoi(*output.NextPartNumberMarker)
+	if err != nil {
+		return s3response.ListPartsResult{},
+			fmt.Errorf("parse next part number marker: %w", err)
+	}
+
+	return s3response.ListPartsResult{
+		Bucket:   *output.Bucket,
+		Key:      *output.Key,
+		UploadID: *output.UploadId,
+		Initiator: s3response.Initiator{
+			ID:          *output.Initiator.ID,
+			DisplayName: *output.Initiator.DisplayName,
+		},
+		Owner: s3response.Owner{
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
+		},
+		StorageClass:         string(output.StorageClass),
+		PartNumberMarker:     pnm,
+		NextPartNumberMarker: npmn,
+		MaxParts:             int(output.MaxParts),
+		IsTruncated:          output.IsTruncated,
+		Parts:                parts,
+	}, nil
+}
+
+func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	output, err := client.UploadPart(ctx, input)
+	if err != nil {
+		return "", err
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.CopyObjectResult{}, err
+	}
+
+	output, err := client.UploadPartCopy(ctx, input)
+	if err != nil {
+		return s3response.CopyObjectResult{}, err
+	}
+
+	return s3response.CopyObjectResult{
+		LastModified: *output.CopyPartResult.LastModified,
+		ETag:         *output.CopyPartResult.ETag,
+	}, nil
+}
+
+func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	output, err := client.PutObject(ctx, input)
+	if err != nil {
+		return "", err
+	}
+
+	return *output.ETag, nil
+}
+
+func (s *S3be) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.HeadObject(ctx, input)
+}
+
+func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetObject(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+	defer output.Body.Close()
+
+	_, err = io.Copy(w, output.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}
+
+func (s *S3be) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.GetObjectAttributes(ctx, input)
+}
+
+func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.CopyObject(ctx, input)
+}
+
+func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.ListObjects(ctx, input)
+}
+
+func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.ListObjectsV2(ctx, input)
+}
+
+func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteObject(ctx, input)
+	return err
+}
+
+func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, err
+	}
+
+	output, err := client.DeleteObjects(ctx, input)
+	if err != nil {
+		return s3response.DeleteObjectsResult{}, err
+	}
+
+	return s3response.DeleteObjectsResult{
+		Deleted: output.Deleted,
+		Error:   output.Errors,
+	}, nil
+}
+
+func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetBucketAcl(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	var acl auth.ACL
+
+	acl.Owner = *output.Owner.ID
+	for _, el := range output.Grants {
+		acl.Grantees = append(acl.Grantees, auth.Grantee{
+			Permission: el.Permission,
+			Access:     *el.Grantee.ID,
+		})
+	}
+
+	return json.Marshal(acl)
+}
+
+func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	acl, err := auth.ParseACL(data)
+	if err != nil {
+		return err
+	}
+
+	input := &s3.PutBucketAclInput{
+		Bucket: &bucket,
+		ACL:    acl.ACL,
+		AccessControlPolicy: &types.AccessControlPolicy{
+			Owner: &types.Owner{
+				ID: &acl.Owner,
+			},
+		},
+	}
+
+	for _, el := range acl.Grantees {
+		input.AccessControlPolicy.Grants = append(input.AccessControlPolicy.Grants, types.Grant{
+			Permission: el.Permission,
+			Grantee: &types.Grantee{
+				ID:   &el.Access,
+				Type: types.TypeCanonicalUser,
+			},
+		})
+	}
+
+	_, err = client.PutBucketAcl(ctx, input)
+	return err
+}
+
+func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	tagging := &types.Tagging{
+		TagSet: []types.Tag{},
+	}
+	for key, val := range tags {
+		tagging.TagSet = append(tagging.TagSet, types.Tag{
+			Key:   &key,
+			Value: &val,
+		})
+	}
+
+	_, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
+		Bucket:  &bucket,
+		Key:     &object,
+		Tagging: tagging,
+	})
+	return err
+}
+
+func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	tags := make(map[string]string)
+	for _, el := range output.TagSet {
+		tags[*el.Key] = *el.Value
+	}
+
+	return tags, nil
+}
+
+func (s *S3be) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	client, err := s.getClientFromCtx(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
+		Bucket: &bucket,
+		Key:    &object,
+	})
+	return err
+}

backend/scoutfs/scoutfs.go

@@ -30,7 +30,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
-	"github.com/versity/scoutfs-go"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
@@ -188,7 +187,7 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = scoutfs.MoveData(pf, f.f)
+		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
 			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
@@ -392,7 +391,7 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 
 		// Check if there are any offline exents associated with this file.
 		// If so, we will set storage class to glacier.
-		st, err := scoutfs.StatMore(objPath)
+		st, err := statMore(objPath)
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		}
@@ -466,7 +465,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
-		st, err := scoutfs.StatMore(objPath)
+		st, err := statMore(objPath)
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		}
@@ -666,7 +665,7 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 		if s.glaciermode {
 			// Check if there are any offline exents associated with this file.
 			// If so, we will return the InvalidObjectState error.
-			st, err := scoutfs.StatMore(objPath)
+			st, err := statMore(objPath)
 			if errors.Is(err, fs.ErrNotExist) {
 				return types.Object{}, backend.ErrSkipObj
 			}

backend/scoutfs/scoutfs_compat.go

@@ -12,6 +12,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build linux && amd64
+
 package scoutfs
 
 import (
@@ -26,6 +28,7 @@ import (
 
 	"golang.org/x/sys/unix"
 
+	"github.com/versity/scoutfs-go"
 	"github.com/versity/versitygw/backend/posix"
 )
 
@@ -182,3 +185,25 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
+
+func moveData(from *os.File, to *os.File) error {
+	return scoutfs.MoveData(from, to)
+}
+
+func statMore(path string) (stat, error) {
+	st, err := scoutfs.StatMore(path)
+	if err != nil {
+		return stat{}, err
+	}
+	var s stat
+
+	s.Meta_seq = st.Meta_seq
+	s.Data_seq = st.Data_seq
+	s.Data_version = st.Data_version
+	s.Online_blocks = st.Online_blocks
+	s.Offline_blocks = st.Offline_blocks
+	s.Crtime_sec = st.Crtime_sec
+	s.Crtime_nsec = st.Crtime_nsec
+
+	return s, nil
+}

backend/scoutfs/scoutfs_incompat.go

@@ -12,6 +12,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build !(linux && amd64)
+
 package scoutfs
 
 import (
@@ -46,3 +48,11 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 
 func (tmp *tmpfile) cleanup() {
 }
+
+func moveData(from *os.File, to *os.File) error {
+	return errNotSupported
+}
+
+func statMore(path string) (stat, error) {
+	return stat{}, errNotSupported
+}

backend/scoutfs/stat.go

@@ -14,35 +14,12 @@
 
 package scoutfs
 
-import (
-	"errors"
-	"fmt"
-	"os"
-)
-
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
-	return nil, fmt.Errorf("scoutfs only available on linux")
-}
-
-type tmpfile struct {
-	f *os.File
-}
-
-var (
-	errNotSupported = errors.New("not supported")
-)
-
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
-	return nil, errNotSupported
-}
-
-func (tmp *tmpfile) link() error {
-	return errNotSupported
-}
-
-func (tmp *tmpfile) Write(b []byte) (int, error) {
-	return 0, errNotSupported
-}
-
-func (tmp *tmpfile) cleanup() {
+type stat struct {
+	Meta_seq       uint64
+	Data_seq       uint64
+	Data_version   uint64
+	Online_blocks  uint64
+	Offline_blocks uint64
+	Crtime_sec     uint64
+	Crtime_nsec    uint32
 }

backend/walk.go

@@ -47,7 +47,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		pastMarker = true
 	}
 
-	var pastMax bool
+	pastMax := max == 0
 	var newMarker string
 	var truncated bool
 
@@ -55,30 +55,30 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		if err != nil {
 			return err
 		}
+		// Ignore the root directory
+		if path == "." {
+			return nil
+		}
+		if contains(d.Name(), skipdirs) {
+			return fs.SkipDir
+		}
 
 		if pastMax {
-			newMarker = path
-			truncated = true
+			if len(objects) != 0 {
+				newMarker = *objects[len(objects)-1].Key
+				truncated = true
+			}
 			return fs.SkipAll
 		}
 
 		if d.IsDir() {
-			// Ignore the root directory
-			if path == "." {
-				return nil
-			}
-
-			if contains(d.Name(), skipdirs) {
-				return fs.SkipDir
-			}
-
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
 			// match this prefix. Make sure to append the / at the end of
 			// directories since this is implied as a directory path name.
 			// If path is a prefix of prefix, then path could still be
-			// building to match. So only skip if path isnt a prefix of prefix
-			// and prefix isnt a prefix of path.
+			// building to match. So only skip if path isn't a prefix of prefix
+			// and prefix isn't a prefix of path.
 			if prefix != "" &&
 				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
 				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
@@ -106,10 +106,13 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj
 		}
 
 		if !pastMarker {
-			if path != marker {
+			if path == marker {
+				pastMarker = true
+				return nil
+			}
+			if path < marker {
 				return nil
 			}
-			pastMarker = true
 		}
 
 		// If object doesn't have prefix, don't include in results.

cmd/versitygw/admin.go

@@ -15,31 +15,35 @@
 package main
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"text/tabwriter"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
 )
 
 var (
-	adminAccess string
-	adminSecret string
+	adminAccess   string
+	adminSecret   string
+	adminEndpoint string
 )
 
 func adminCommand() *cli.Command {
 	return &cli.Command{
-		Name:  "admin",
-		Usage: "admin CLI tool",
-		Description: `admin CLI tool for interacting with admin api.
-		Here is the available api list:
-		create-user
-		`,
+		Name:        "admin",
+		Usage:       "admin CLI tool",
+		Description: `Admin CLI tool for interacting with admin APIs.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:   "create-user",
@@ -48,13 +52,13 @@ func adminCommand() *cli.Command {
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:     "access",
-						Usage:    "access value for the new user",
+						Usage:    "access key id for the new user",
 						Required: true,
 						Aliases:  []string{"a"},
 					},
 					&cli.StringFlag{
 						Name:     "secret",
-						Usage:    "secret value for the new user",
+						Usage:    "secret access key for the new user",
 						Required: true,
 						Aliases:  []string{"s"},
 					},
@@ -64,6 +68,21 @@ func adminCommand() *cli.Command {
 						Required: true,
 						Aliases:  []string{"r"},
 					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+					&cli.IntFlag{
+						Name:    "project-id",
+						Usage:   "projectID for the new user",
+						Aliases: []string{"pi"},
+					},
 				},
 			},
 			{
@@ -73,20 +92,50 @@ func adminCommand() *cli.Command {
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:     "access",
-						Usage:    "access value for the user to be deleted",
+						Usage:    "access key id of the user to be deleted",
 						Required: true,
 						Aliases:  []string{"a"},
 					},
 				},
 			},
+			{
+				Name:   "list-users",
+				Usage:  "List all the gateway users",
+				Action: listUsers,
+			},
+			{
+				Name:  "change-bucket-owner",
+				Usage: "Changes the bucket owner",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "bucket",
+						Usage:    "the bucket name to change the owner",
+						Required: true,
+						Aliases:  []string{"b"},
+					},
+					&cli.StringFlag{
+						Name:     "owner",
+						Usage:    "the user access key id, who should be the bucket owner",
+						Required: true,
+						Aliases:  []string{"o"},
+					},
+				},
+				Action: changeBucketOwner,
+			},
+			{
+				Name:   "list-buckets",
+				Usage:  "Lists all the gateway buckets and owners.",
+				Action: listBuckets,
+			},
 		},
 		Flags: []cli.Flag{
 			// TODO: create a configuration file for this
 			&cli.StringFlag{
 				Name:        "access",
-				Usage:       "admin access account",
+				Usage:       "admin access key id",
 				EnvVars:     []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
 				Aliases:     []string{"a"},
+				Required:    true,
 				Destination: &adminAccess,
 			},
 			&cli.StringFlag{
@@ -94,14 +143,24 @@ func adminCommand() *cli.Command {
 				Usage:       "admin secret access key",
 				EnvVars:     []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
 				Aliases:     []string{"s"},
+				Required:    true,
 				Destination: &adminSecret,
 			},
+			&cli.StringFlag{
+				Name:        "endpoint-url",
+				Usage:       "admin apis endpoint url",
+				EnvVars:     []string{"ADMIN_ENDPOINT_URL"},
+				Aliases:     []string{"er"},
+				Required:    true,
+				Destination: &adminEndpoint,
+			},
 		},
 	}
 }
 
 func createUser(ctx *cli.Context) error {
 	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
+	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
 	if access == "" || secret == "" {
 		return fmt.Errorf("invalid input parameters for the new user")
 	}
@@ -109,14 +168,28 @@ func createUser(ctx *cli.Context) error {
 		return fmt.Errorf("invalid input parameter for role")
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:7070/create-user?access=%v&secret=%v&role=%v", access, secret, role), nil)
+	acc := auth.Account{
+		Access:    access,
+		Secret:    secret,
+		Role:      role,
+		UserID:    userID,
+		GroupID:   groupID,
+		ProjectID: projectID,
+	}
+
+	accJson, err := json.Marshal(acc)
+	if err != nil {
+		return fmt.Errorf("failed to parse user data: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256([]byte{})
+	hashedPayload := sha256.Sum256(accJson)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
@@ -137,6 +210,7 @@ func createUser(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	fmt.Printf("%s\n", body)
 
@@ -149,7 +223,7 @@ func deleteUser(ctx *cli.Context) error {
 		return fmt.Errorf("invalid input parameter for the new user")
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:7070/delete-user?access=%v", access), nil)
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
@@ -177,8 +251,166 @@ func deleteUser(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	fmt.Printf("%s\n", body)
 
 	return nil
 }
+
+func listUsers(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
+		return err
+	}
+	fmt.Println(accs)
+
+	printAcctTable(accs)
+
+	return nil
+}
+
+const (
+	// account table formatting
+	minwidth int  = 2   // minimal cell width including any padding
+	tabwidth int  = 0   // width of tab characters (equivalent number of spaces)
+	padding  int  = 2   // padding added to a cell before computing its width
+	padchar  byte = ' ' // ASCII char used for padding
+	flags    uint = 0   // formatting control flags
+)
+
+func printAcctTable(accs []auth.Account) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
+	for _, acc := range accs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func changeBucketOwner(ctx *cli.Context) error {
+	bucket, owner := ctx.String("bucket"), ctx.String("owner")
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fmt.Println(string(body))
+
+	return nil
+}
+
+func printBuckets(buckets []s3response.Bucket) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	fmt.Fprintln(w, "Bucket\tOwner")
+	fmt.Fprintln(w, "-------\t----")
+	for _, acc := range buckets {
+		fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func listBuckets(ctx *cli.Context) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf(string(body))
+	}
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return err
+	}
+
+	printBuckets(buckets)
+
+	return nil
+}

cmd/versitygw/main.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strconv"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
@@ -32,16 +33,24 @@ import (
 )
 
 var (
-	port                           string
-	rootUserAccess                 string
-	rootUserSecret                 string
-	region                         string
-	certFile, keyFile              string
-	kafkaURL, kafkaTopic, kafkaKey string
-	natsURL, natsTopic             string
-	logWebhookURL                  string
-	accessLog                      string
-	debug                          bool
+	port, admPort                          string
+	rootUserAccess                         string
+	rootUserSecret                         string
+	region                                 string
+	admCertFile, admKeyFile                string
+	certFile, keyFile                      string
+	kafkaURL, kafkaTopic, kafkaKey         string
+	natsURL, natsTopic                     string
+	logWebhookURL                          string
+	accessLog                              string
+	debug                                  bool
+	iamDir                                 string
+	ldapURL, ldapBindDN, ldapPassword      string
+	ldapQueryBase, ldapObjClasses          string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	iamCacheDisable                        bool
+	iamCacheTTL                            int
+	iamCachePrune                          int
 )
 
 var (
@@ -61,6 +70,7 @@ func main() {
 	app.Commands = []*cli.Command{
 		posixCommand(),
 		scoutfsCommand(),
+		s3Command(),
 		adminCommand(),
 		testCommand(),
 	}
@@ -142,6 +152,22 @@ func initFlags() []cli.Flag {
 			Usage:       "TLS key file",
 			Destination: &keyFile,
 		},
+		&cli.StringFlag{
+			Name:        "admin-port",
+			Usage:       "gateway admin server listen address <ip>:<port> or :<port>",
+			Destination: &admPort,
+			Aliases:     []string{"ap"},
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert",
+			Usage:       "TLS cert file for admin server",
+			Destination: &admCertFile,
+		},
+		&cli.StringFlag{
+			Name:        "admin-cert-key",
+			Usage:       "TLS key file for admin server",
+			Destination: &admKeyFile,
+		},
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
@@ -189,14 +215,83 @@ func initFlags() []cli.Flag {
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
+		&cli.StringFlag{
+			Name:        "iam-dir",
+			Usage:       "if defined, run internal iam service within this directory",
+			Destination: &iamDir,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-url",
+			Usage:       "ldap server url to store iam data",
+			Destination: &ldapURL,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-dn",
+			Usage:       "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
+			Destination: &ldapBindDN,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-bind-pass",
+			Usage:       "ldap server user password",
+			Destination: &ldapPassword,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-query-base",
+			Usage:       "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
+			Destination: &ldapQueryBase,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-object-classes",
+			Usage:       "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
+			Destination: &ldapObjClasses,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-access-atr",
+			Usage:       "ldap server user access key id attribute name",
+			Destination: &ldapAccessAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-secret-atr",
+			Usage:       "ldap server user secret access key attribute name",
+			Destination: &ldapSecAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-role-atr",
+			Usage:       "ldap server user role attribute name",
+			Destination: &ldapRoleAtr,
+		},
+		&cli.BoolFlag{
+			Name:        "iam-cache-disable",
+			Usage:       "disable local iam cache",
+			Destination: &iamCacheDisable,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-ttl",
+			Usage:       "local iam cache entry ttl (seconds)",
+			Value:       120,
+			Destination: &iamCacheTTL,
+		},
+		&cli.IntFlag{
+			Name:        "iam-cache-prune",
+			Usage:       "local iam cache cleanup interval (seconds)",
+			Value:       3600,
+			Destination: &iamCachePrune,
+		},
 	}
 }
 
-func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
+func runGateway(ctx *cli.Context, be backend.Backend) error {
+	// int32 max for 32 bit arch
+	blimit := int64(2*1024*1024*1024 - 1)
+	if strconv.IntSize > 32 {
+		// 5GB max for 64 bit arch
+		blimit = int64(5 * 1024 * 1024 * 1024)
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:      "versitygw",
 		ServerHeader: "VERSITYGW",
-		BodyLimit:    5 * 1024 * 1024 * 1024,
+		BodyLimit:    int(blimit),
 	})
 
 	var opts []s3api.Option
@@ -215,19 +310,51 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
 		}
 		opts = append(opts, s3api.WithTLS(cert))
 	}
-
 	if debug {
 		opts = append(opts, s3api.WithDebug())
 	}
-
-	err := s.InitIAM()
-	if err != nil {
-		return fmt.Errorf("init iam: %w", err)
+	if admPort == "" {
+		opts = append(opts, s3api.WithAdminServer())
 	}
 
-	iam, err := auth.NewInternal(s)
+	admApp := fiber.New(fiber.Config{
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+	})
+
+	var admOpts []s3api.AdminOpt
+
+	if admCertFile != "" || admKeyFile != "" {
+		if admCertFile == "" {
+			return fmt.Errorf("TLS key specified without cert file")
+		}
+		if admKeyFile == "" {
+			return fmt.Errorf("TLS cert specified without key file")
+		}
+
+		cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile)
+		if err != nil {
+			return fmt.Errorf("tls: load certs: %v", err)
+		}
+		admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert))
+	}
+
+	iam, err := auth.New(&auth.Opts{
+		Dir:            iamDir,
+		LDAPServerURL:  ldapURL,
+		LDAPBindDN:     ldapBindDN,
+		LDAPPassword:   ldapPassword,
+		LDAPQueryBase:  ldapQueryBase,
+		LDAPObjClasses: ldapObjClasses,
+		LDAPAccessAtr:  ldapAccessAtr,
+		LDAPSecretAtr:  ldapSecAtr,
+		LDAPRoleAtr:    ldapRoleAtr,
+		CacheDisable:   iamCacheDisable,
+		CacheTTL:       iamCacheTTL,
+		CachePrune:     iamCachePrune,
+	})
 	if err != nil {
-		return fmt.Errorf("setup internal iam service: %w", err)
+		return fmt.Errorf("setup iam: %w", err)
 	}
 
 	logger, err := s3log.InitLogger(&s3log.LogConfig{
@@ -257,8 +384,13 @@ func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	c := make(chan error, 1)
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
+
+	c := make(chan error, 2)
 	go func() { c <- srv.Serve() }()
+	if admPort != "" {
+		go func() { c <- admSrv.Serve() }()
+	}
 
 	// for/select blocks until shutdown
 Loop:
@@ -279,13 +411,21 @@ Loop:
 			}
 		}
 	}
+	saveErr := err
 
 	be.Shutdown()
+
+	err = iam.Shutdown()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
+	}
+
 	if logger != nil {
-		lerr := logger.Shutdown()
-		if lerr != nil {
-			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", lerr)
+		err := logger.Shutdown()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
 		}
 	}
-	return err
+
+	return saveErr
 }

cmd/versitygw/posix.go

@@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("init posix: %v", err)
 	}
 
-	return runGateway(ctx, be, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/s3.go

@@ -0,0 +1,76 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/s3proxy"
+)
+
+var (
+	s3proxyEndpoint        string
+	s3proxyRegion          string
+	s3proxyDisableChecksum bool
+	s3proxySslSkipVerify   bool
+	s3proxyDebug           bool
+)
+
+func s3Command() *cli.Command {
+	return &cli.Command{
+		Name:  "s3",
+		Usage: "s3 storage backend",
+		Description: `This runs the gateway like an s3 proxy redirecting requests
+to an s3 storage backend service.`,
+		Action: runS3,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "endpoint",
+				Usage:       "s3 service endpoint, default AWS if not specified",
+				Value:       "",
+				Destination: &s3proxyEndpoint,
+			},
+			&cli.StringFlag{
+				Name:        "region",
+				Usage:       "s3 service region, default 'us-east-1' if not specified",
+				Value:       "us-east-1",
+				Destination: &s3proxyRegion,
+			},
+			&cli.BoolFlag{
+				Name:        "disable-checksum",
+				Usage:       "disable gateway to server object checksums",
+				Value:       false,
+				Destination: &s3proxyDisableChecksum,
+			},
+			&cli.BoolFlag{
+				Name:        "ssl-skip-verify",
+				Usage:       "skip ssl cert verification for s3 service",
+				Value:       false,
+				Destination: &s3proxySslSkipVerify,
+			},
+			&cli.BoolFlag{
+				Name:        "debug",
+				Usage:       "output extra debug tracing",
+				Value:       false,
+				Destination: &s3proxyDebug,
+			},
+		},
+	}
+}
+
+func runS3(ctx *cli.Context) error {
+	be := s3proxy.New(s3proxyEndpoint, s3proxyRegion,
+		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	return runGateway(ctx, be)
+}

cmd/versitygw/scoutfs.go

@@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}
 
-	return runGateway(ctx, be, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/test.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"fmt"
+	"math"
+	"os"
+	"text/tabwriter"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/integration"
@@ -13,10 +16,12 @@ var (
 	endpoint        string
 	prefix          string
 	dstBucket       string
+	proxyURL        string
 	partSize        int64
 	objSize         int64
 	concurrency     int
 	files           int
+	totalReqs       int
 	upload          bool
 	download        bool
 	pathStyle       bool
@@ -67,120 +72,17 @@ func initTestFlags() []cli.Flag {
 
 func initTestCommands() []*cli.Command {
 	return []*cli.Command{
-		{
-			Name:  "bucket-actions",
-			Usage: "Test bucket creation, checking the existence, deletes it.",
-			Description: `Calls s3 gateway create-bucket action to create a new bucket,
-		calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`,
-			Action: getAction(integration.TestMakeBucket),
-		},
-		{
-			Name:  "object-actions",
-			Usage: "Test put/get/delete/copy objects.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying,
-			copies the object, get's the object to check the length and content, 
-			get's the copied object to check the length and content, deletes all the objects inside the source bucket, 
-			deletes both the objects and buckets.`,
-			Action: getAction(integration.TestPutGetObject),
-		},
-		{
-			Name:  "put-get-mp-object",
-			Usage: "Test put & get multipart object.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload,
-			gets the object from the bucket, deletes both the object and bucket.`,
-			Action: getAction(integration.TestPutGetMPObject),
-		},
-		{
-			Name:  "put-dir-object",
-			Usage: "Test put directory object.",
-			Description: `Creates a bucket with s3 gateway action, puts a directory object in it,
-			lists the bucket's objects, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestPutDirObject),
-		},
-		{
-			Name:  "list-objects",
-			Usage: "Test list-objects action.",
-			Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it,
-			lists the bucket's objects, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListObject),
-		},
-		{
-			Name:  "abort-mp",
-			Usage: "Tests abort-multipart-upload action.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			lists the multipart upload, aborts the multipart upload, lists the multipart upload again,
-			deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListAbortMultiPartObject),
-		},
-		{
-			Name:  "list-parts",
-			Usage: "Tests list-parts action.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			lists the upload parts, deletes both the objects and bucket.`,
-			Action: getAction(integration.TestListMultiParts),
-		},
-		{
-			Name:  "incorrect-mp",
-			Usage: "Tests incorrect multipart case.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			uploads different parts, completes the multipart upload with incorrect part numbers,
-			calls the head-object action, compares the content length, removes both the object and bucket`,
-			Action: getAction(integration.TestIncorrectMultiParts),
-		},
-		{
-			Name:  "incomplete-mp",
-			Usage: "Tests incomplete multi parts.",
-			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
-			upload a part, lists the parts, checks if the uploaded part is in the list, 
-			removes both the object and the bucket`,
-			Action: getAction(integration.TestIncompleteMultiParts),
-		},
-		{
-			Name:  "incomplete-put-object",
-			Usage: "Tests incomplete put objects case.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object with head-object action, expects the object to be got, 
-			removes both the object and bucket`,
-			Action: getAction(integration.TestIncompletePutObject),
-		},
-		{
-			Name:  "get-range",
-			Usage: "Tests get object by range.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object by specifying the object range, compares the range with the original one,
-			removes both the object and the bucket`,
-			Action: getAction(integration.TestRangeGet),
-		},
-		{
-			Name:  "invalid-mp",
-			Usage: "Tests invalid multi part case.",
-			Description: `Creates a bucket with s3 gateway action, creates a multi part upload,
-			uploads an invalid part, gets the object with head-object action, expects to get error,
-			removes both the object and bucket`,
-			Action: getAction(integration.TestInvalidMultiParts),
-		},
-		{
-			Name:  "object-tag-actions",
-			Usage: "Tests get/put/delete object tag actions.",
-			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			puts some tags for the object, gets the tags, compares the results, removes the tags,
-			gets the tags again, checks it to be empty, then removes both the object and bucket`,
-			Action: getAction(integration.TestPutGetRemoveTags),
-		},
-		{
-			Name:  "bucket-acl-actions",
-			Usage: "Tests put/get bucket actions.",
-			Description: `Creates a bucket with s3 gateway action, puts some bucket acls
-			gets the acl, verifies it, then removes the bucket`,
-			Action: getAction(integration.TestAclActions),
-		},
 		{
 			Name:        "full-flow",
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
 		},
+		{
+			Name:   "posix",
+			Usage:  "Tests posix specific features",
+			Action: getAction(integration.TestPosix),
+		},
 		{
 			Name:  "bench",
 			Usage: "Runs download/upload performance test on the gateway",
@@ -220,6 +122,7 @@ func initTestCommands() []*cli.Command {
 					Name:        "bucket",
 					Usage:       "Destination bucket name to read/write data",
 					Destination: &dstBucket,
+					Required:    true,
 				},
 				&cli.Int64Flag{
 					Name:        "partSize",
@@ -245,6 +148,11 @@ func initTestCommands() []*cli.Command {
 					Value:       false,
 					Destination: &checksumDisable,
 				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
 			},
 			Action: func(ctx *cli.Context) error {
 				if upload && download {
@@ -254,10 +162,6 @@ func initTestCommands() []*cli.Command {
 					return fmt.Errorf("must specify one of upload or download")
 				}
 
-				if dstBucket == "" {
-					return fmt.Errorf("must specify bucket")
-				}
-
 				opts := []integration.Option{
 					integration.WithAccess(awsID),
 					integration.WithSecret(awsSecret),
@@ -278,7 +182,123 @@ func initTestCommands() []*cli.Command {
 
 				s3conf := integration.NewS3Conf(opts...)
 
-				return integration.TestPerformance(s3conf, upload, download, files, objSize, dstBucket, prefix)
+				if upload {
+					if proxyURL == "" {
+						integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
+				} else {
+					if proxyURL == "" {
+						integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
+				}
+			},
+		},
+		{
+			Name:        "throughput",
+			Usage:       "Runs throughput performance test on the gateway",
+			Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "reqs",
+					Usage:       "Total number of requests to send.",
+					Value:       1000,
+					Destination: &totalReqs,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to make the requests",
+					Destination: &dstBucket,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "threads per request",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				if proxyURL == "" {
+					_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					return err
+				} else {
+					elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					opts = append(opts, integration.WithEndpoint(proxyURL))
+					s3proxy := integration.NewS3Conf(opts...)
+					proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
+					if err != nil || proxyErr != nil {
+						return nil
+					}
+
+					printProxyResultsTable([][4]string{
+						{"    #    ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
+						{"---------", "--------------", "----------", "----------------------------"},
+						{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
+						{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
+					})
+
+					return nil
+				}
 			},
 		},
 	}
@@ -309,3 +329,13 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		return nil
 	}
 }
+
+func printProxyResultsTable(stats [][4]string) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	for _, elem := range stats {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}

go.mod

@@ -3,52 +3,52 @@ module github.com/versity/versitygw
 go 1.20
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.20.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1
-	github.com/aws/smithy-go v1.14.0
-	github.com/gofiber/fiber/v2 v2.48.0
-	github.com/google/uuid v1.3.0
-	github.com/nats-io/nats.go v1.28.0
+	github.com/aws/aws-sdk-go-v2 v1.22.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
+	github.com/aws/smithy-go v1.16.0
+	github.com/go-ldap/ldap/v3 v3.4.6
+	github.com/gofiber/fiber/v2 v2.50.0
+	github.com/google/uuid v1.4.0
+	github.com/nats-io/nats.go v1.31.0
 	github.com/pkg/xattr v0.4.9
-	github.com/segmentio/kafka-go v0.4.42
+	github.com/segmentio/kafka-go v0.4.44
 	github.com/urfave/cli/v2 v2.25.7
-	github.com/valyala/fasthttp v1.48.0
+	github.com/valyala/fasthttp v1.50.0
 	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.10.0
+	golang.org/x/sys v0.14.0
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/nats-io/nats-server/v2 v2.9.20 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.32
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.31
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.24.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.15.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect

go.sum

@@ -1,65 +1,69 @@
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go-v2 v1.20.0 h1:INUDpYLt4oiPOJl0XwZDK2OVAVf0Rzo+MGVTv9f+gy8=
-github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 h1:/MS8AzqYNAhhRNalOmxUvYs8VEbNGifTnzhPFdcRQkQ=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11/go.mod h1:va22++AdXht4ccO3kH2SHkHHYvZ2G9Utz+CXKmm2CaU=
-github.com/aws/aws-sdk-go-v2/config v1.18.32 h1:tqEOvkbTxwEV7hToRcJ1xZRjcATqwDVsWbAscgRKyNI=
-github.com/aws/aws-sdk-go-v2/config v1.18.32/go.mod h1:U3ZF0fQRRA4gnbn9GGvOWLoT2EzzZfAWeKwnVrm1rDc=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.31 h1:vJyON3lG7R8VOErpJJBclBADiWTwzcwdkQpTKx8D2sk=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.31/go.mod h1:T4sESjBtY2lNxLgkIASmeP57b5j7hTQqCbqG0tWnxC4=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 h1:X3H6+SU21x+76LRglk21dFRgMTJMa5QcpW+SqUf5BBg=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7/go.mod h1:3we0V09SwcJBzNlnyovrR2wWJhWmVdqAsmVs4uronv8=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76 h1:DJ1kHj0GI9BbX+XhF0kHxlzOVjcncmDUXmCvXdbfdAE=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76/go.mod h1:/AZCdswMSgwpB2yMSFfY5H4pVeBLnCuPehdmO/r3xSM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 h1:zr/gxAZkMcvP71ZhQOcvdm8ReLjFgIXnIn0fw5AM7mo=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 h1:0HCMIkAkVY9KMgueD8tf4bRTUanzEYvhw7KkPXIMpO0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 h1:+i1DOFrW3YZ3apE45tCal9+aDKK6kNEbW6Ib7e1nFxE=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38/go.mod h1:1/jLp0OgOaWIetycOmycW+vYTYgTZFPttJQRgsI1PoU=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0/go.mod h1:EhC/83j8/hL/UB1WmExo3gkElaja/KlmZM/gl1rTfjM=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 h1:uAiiHnWihGP2rVp64fHwzLDrswGjEjsPszwRYMiYQPU=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12/go.mod h1:fUTHpOXqRQpXvEpDPSa3zxCc2fnpW6YnBoba+eQr+Bg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 h1:kvN1jPHr9UffqqG3bSgZ8tx4+1zKVHz/Ktw/BwW6hX8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32/go.mod h1:QmMEM7es84EUkbYWcpnkx8i5EW2uERPfrTFeOch128Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 h1:auGDJ0aLZahF5SPvkJ6WcUuX7iQ7kyl2MamV7Tm8QBk=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31/go.mod h1:3+lloe3sZuBQw1aBc5MyndvodzQlyqCZ7x1QPDHaWP4=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 h1:Wgjft9X4W5pMeuqgPCHIQtbZ87wsgom7S5F8obreg+c=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0/go.mod h1:FWNzS4+zcWAP05IF7TDYTY1ysZAzIvogxWaDT9p8fsA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 h1:mTgFVlfQT8gikc5+/HwD8UL9jnUro5MGv8n/VEYF12I=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1/go.mod h1:6SOWLiobcZZshbmECRTADIRYliPL0etqFSigauQEeT0=
-github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 h1:DSNpSbfEgFXRV+IfEcKE5kTbqxm+MeF5WgyeRlsLnHY=
-github.com/aws/aws-sdk-go-v2/service/sso v1.13.1/go.mod h1:TC9BubuFMVScIU+TLKamO6VZiYTkYoEHqlSQwAe2omw=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 h1:pAOJj+80tC8sPVgSDHzMYD6KLWsaLQ1kZw31PTeORbs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.21.1/go.mod h1:G8SbvL0rFk4WOJroU8tKBczhsbhj2p/YY7qeJezJ3CI=
-github.com/aws/smithy-go v1.14.0 h1:+X90sB94fizKjDmwb4vyl2cTTPXTE5E2G/1mjByb0io=
-github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
+github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
+github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
+github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
+github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
+github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0=
-github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
+github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
+github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
+github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
+github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -67,14 +71,10 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
-github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
-github.com/nats-io/nats-server/v2 v2.9.20 h1:bt1dW6xsL1hWWwv7Hovm+EJt5L6iplyqlgEFkoEUk0k=
-github.com/nats-io/nats-server/v2 v2.9.20/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw=
-github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
-github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
-github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
-github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
+github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
+github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
+github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
+github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -89,8 +89,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU=
-github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg=
+github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4=
+github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -102,8 +102,8 @@ github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
 github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc=
-github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
+github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
@@ -119,16 +119,21 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -138,27 +143,30 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

integration/bench.go

@@ -0,0 +1,142 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type prefResult struct {
+	elapsed time.Duration
+	size    int64
+	err     error
+}
+
+func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+	if objSize == 0 {
+		return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
+	}
+
+	if objSize > (int64(10000) * s.PartSize) {
+		return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
+	}
+
+	runF("performance test: upload objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			var r io.Reader = NewDataReader(int(objSize), int(s.PartSize))
+
+			start := time.Now()
+			err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = objSize
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed = time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			return 0, time.Since(start), res.err
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run upload: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return tot, time.Since(start), nil
+}
+
+func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+
+	runF("performance test: download objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			nw := NewNullWriter()
+			start := time.Now()
+			n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = n
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed = time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			return 0, elapsed, err
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run download: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return tot, elapsed, nil
+}
+
+func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
+	client := s3.NewFromConfig(s.Config())
+	var wg sync.WaitGroup
+	var resErr error
+
+	// Record the start time
+	startTime := time.Now()
+	runF("performance test: measuring request per second")
+
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < totalReqs/s.Concurrency; i++ {
+				_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket})
+				if err != nil && resErr != nil {
+					resErr = err
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	if resErr != nil {
+		failF("performance test failed with error: %w", resErr)
+		return time.Since(startTime), 0, resErr
+	}
+	elapsedTime := time.Since(startTime)
+	rps := int(float64(totalReqs) / elapsedTime.Seconds())
+
+	passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
+	return elapsedTime, rps, nil
+}

integration/group-tests.go

@@ -0,0 +1,224 @@
+package integration
+
+func TestAuthentication(s *S3Conf) {
+	Authentication_empty_auth_header(s)
+	Authentication_invalid_auth_header(s)
+	Authentication_unsupported_signature_version(s)
+	Authentication_malformed_credentials(s)
+	Authentication_malformed_credentials_invalid_parts(s)
+	Authentication_credentials_terminated_string(s)
+	Authentication_credentials_incorrect_service(s)
+	Authentication_credentials_incorrect_region(s)
+	Authentication_credentials_invalid_date(s)
+	Authentication_credentials_future_date(s)
+	Authentication_credentials_past_date(s)
+	Authentication_credentials_non_existing_access_key(s)
+	Authentication_invalid_signed_headers(s)
+	Authentication_missing_date_header(s)
+	Authentication_invalid_date_header(s)
+	Authentication_date_mismatch(s)
+	Authentication_incorrect_payload_hash(s)
+	Authentication_incorrect_md5(s)
+	Authentication_signature_error_incorrect_secret_key(s)
+}
+
+func TestCreateBucket(s *S3Conf) {
+	CreateBucket_invalid_bucket_name(s)
+	CreateBucket_existing_bucket(s)
+	CreateBucket_as_user(s)
+	CreateDeleteBucket_success(s)
+}
+
+func TestHeadBucket(s *S3Conf) {
+	HeadBucket_non_existing_bucket(s)
+	HeadBucket_success(s)
+}
+
+func TestListBuckets(s *S3Conf) {
+	ListBuckets_as_user(s)
+	ListBuckets_as_admin(s)
+	ListBuckets_success(s)
+}
+
+func TestDeleteBucket(s *S3Conf) {
+	DeleteBucket_non_existing_bucket(s)
+	DeleteBucket_non_empty_bucket(s)
+	DeleteBucket_success_status_code(s)
+}
+
+func TestPutObject(s *S3Conf) {
+	PutObject_non_existing_bucket(s)
+	PutObject_special_chars(s)
+	PutObject_invalid_long_tags(s)
+	PutObject_success(s)
+}
+
+func TestHeadObject(s *S3Conf) {
+	HeadObject_non_existing_object(s)
+	HeadObject_success(s)
+}
+
+func TestGetObject(s *S3Conf) {
+	GetObject_non_existing_key(s)
+	GetObject_invalid_ranges(s)
+	GetObject_with_meta(s)
+	GetObject_success(s)
+	GetObject_by_range_success(s)
+}
+
+func TestListObjects(s *S3Conf) {
+	ListObjects_non_existing_bucket(s)
+	ListObjects_with_prefix(s)
+	ListObject_truncated(s)
+	ListObjects_invalid_max_keys(s)
+	ListObjects_max_keys_0(s)
+	ListObjects_delimiter(s)
+	ListObjects_max_keys_none(s)
+	ListObjects_marker_not_from_obj_list(s)
+}
+
+func TestDeleteObject(s *S3Conf) {
+	DeleteObject_non_existing_object(s)
+	DeleteObject_success(s)
+	DeleteObject_success_status_code(s)
+}
+
+func TestDeleteObjects(s *S3Conf) {
+	DeleteObjects_empty_input(s)
+	DeleteObjects_non_existing_objects(s)
+	DeleteObjects_success(s)
+}
+
+func TestCopyObject(s *S3Conf) {
+	CopyObject_non_existing_dst_bucket(s)
+	CopyObject_not_owned_source_bucket(s)
+	CopyObject_copy_to_itself(s)
+	CopyObject_to_itself_with_new_metadata(s)
+	CopyObject_success(s)
+}
+
+func TestPutObjectTagging(s *S3Conf) {
+	PutObjectTagging_non_existing_object(s)
+	PutObjectTagging_long_tags(s)
+	PutObjectTagging_success(s)
+}
+
+func TestGetObjectTagging(s *S3Conf) {
+	GetObjectTagging_non_existing_object(s)
+	GetObjectTagging_success(s)
+}
+
+func TestDeleteObjectTagging(s *S3Conf) {
+	DeleteObjectTagging_non_existing_object(s)
+	DeleteObjectTagging_success_status(s)
+	DeleteObjectTagging_success(s)
+}
+
+func TestCreateMultipartUpload(s *S3Conf) {
+	CreateMultipartUpload_non_existing_bucket(s)
+	CreateMultipartUpload_success(s)
+}
+
+func TestUploadPart(s *S3Conf) {
+	UploadPart_non_existing_bucket(s)
+	UploadPart_invalid_part_number(s)
+	UploadPart_non_existing_key(s)
+	UploadPart_non_existing_mp_upload(s)
+	UploadPart_success(s)
+}
+
+func TestUploadPartCopy(s *S3Conf) {
+	UploadPartCopy_non_existing_bucket(s)
+	UploadPartCopy_incorrect_uploadId(s)
+	UploadPartCopy_incorrect_object_key(s)
+	UploadPartCopy_invalid_part_number(s)
+	UploadPartCopy_invalid_copy_source(s)
+	UploadPartCopy_non_existing_source_bucket(s)
+	UploadPartCopy_non_existing_source_object_key(s)
+	UploadPartCopy_success(s)
+	UploadPartCopy_by_range_invalid_range(s)
+	UploadPartCopy_greater_range_than_obj_size(s)
+	UploadPartCopy_by_range_success(s)
+}
+
+func TestListParts(s *S3Conf) {
+	ListParts_incorrect_uploadId(s)
+	ListParts_incorrect_object_key(s)
+	ListParts_success(s)
+}
+
+func TestListMultipartUploads(s *S3Conf) {
+	ListMultipartUploads_non_existing_bucket(s)
+	ListMultipartUploads_empty_result(s)
+	ListMultipartUploads_invalid_max_uploads(s)
+	ListMultipartUploads_max_uploads(s)
+	ListMultipartUploads_incorrect_next_key_marker(s)
+	ListMultipartUploads_ignore_upload_id_marker(s)
+	ListMultipartUploads_success(s)
+}
+
+func TestAbortMultipartUpload(s *S3Conf) {
+	AbortMultipartUpload_non_existing_bucket(s)
+	AbortMultipartUpload_incorrect_uploadId(s)
+	AbortMultipartUpload_incorrect_object_key(s)
+	AbortMultipartUpload_success(s)
+	AbortMultipartUpload_success_status_code(s)
+}
+
+func TestCompleteMultipartUpload(s *S3Conf) {
+	CompletedMultipartUpload_non_existing_bucket(s)
+	CompleteMultipartUpload_invalid_part_number(s)
+	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_success(s)
+}
+
+func TestPutBucketAcl(s *S3Conf) {
+	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_invalid_acl_canned_and_acp(s)
+	PutBucketAcl_invalid_acl_canned_and_grants(s)
+	PutBucketAcl_invalid_acl_acp_and_grants(s)
+	PutBucketAcl_invalid_owner(s)
+	PutBucketAcl_success_access_denied(s)
+	PutBucketAcl_success_grants(s)
+	PutBucketAcl_success_canned_acl(s)
+	PutBucketAcl_success_acp(s)
+}
+
+func TestGetBucketAcl(s *S3Conf) {
+	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_access_denied(s)
+	GetBucketAcl_success(s)
+}
+
+func TestFullFlow(s *S3Conf) {
+	TestAuthentication(s)
+	TestCreateBucket(s)
+	TestHeadBucket(s)
+	TestListBuckets(s)
+	TestDeleteBucket(s)
+	TestPutObject(s)
+	TestHeadObject(s)
+	TestGetObject(s)
+	TestListObjects(s)
+	TestDeleteObject(s)
+	TestDeleteObjects(s)
+	TestCopyObject(s)
+	TestPutObjectTagging(s)
+	TestDeleteObjectTagging(s)
+	TestCreateMultipartUpload(s)
+	TestUploadPart(s)
+	TestUploadPartCopy(s)
+	TestListParts(s)
+	TestListMultipartUploads(s)
+	TestAbortMultipartUpload(s)
+	TestCompleteMultipartUpload(s)
+	TestPutBucketAcl(s)
+	TestGetBucketAcl(s)
+}
+
+func TestPosix(s *S3Conf) {
+	PutObject_overwrite_dir_obj(s)
+	PutObject_overwrite_file_obj(s)
+	PutObject_dir_obj_with_data(s)
+	CreateMultipartUpload_dir_obj(s)
+}

integration/utils.go

@@ -1,15 +1,42 @@
 package integration
 
 import (
+	"bytes"
 	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
 	"fmt"
+	"io"
+	rnd "math/rand"
+	"net/http"
+	"os"
 	"os/exec"
 	"strings"
+	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
 )
 
+var (
+	bcktCount  = 0
+	succUsrCrt = "The user has been created successfully"
+	failUsrCrt = "failed to create a user: update iam data: account already exists"
+)
+
+func getBucketName() string {
+	bcktCount++
+	return fmt.Sprintf("test-bucket-%v", bcktCount)
+}
+
 func setup(s *S3Conf, bucket string) error {
 	s3client := s3.NewFromConfig(s.Config())
 
@@ -69,6 +96,170 @@ func teardown(s *S3Conf, bucket string) error {
 	return err
 }
 
+func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
+	runF(testName)
+	bucketName := getBucketName()
+	err := setup(s, bucketName)
+	if err != nil {
+		failF("%v: failed to create a bucket: %v", testName, err.Error())
+		return
+	}
+	client := s3.NewFromConfig(s.Config())
+	handlerErr := handler(client, bucketName)
+	if handlerErr != nil {
+		failF("%v: %v", testName, handlerErr.Error())
+	}
+
+	err = teardown(s, bucketName)
+	if err != nil {
+		if handlerErr == nil {
+			failF("%v: failed to delete the bucket: %v", testName, err.Error())
+		} else {
+			fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
+		}
+	}
+	if handlerErr == nil {
+		passF(testName)
+	}
+}
+
+type authConfig struct {
+	testName string
+	path     string
+	method   string
+	body     []byte
+	service  string
+	date     time.Time
+}
+
+func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
+	runF(cfg.testName)
+	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err.Error())
+		return
+	}
+
+	err = handler(req)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err.Error())
+		return
+	}
+	passF(cfg.testName)
+}
+
+func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
+	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(body)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date)
+	if signErr != nil {
+		return nil, fmt.Errorf("failed to sign the request: %w", signErr)
+	}
+
+	return req, nil
+}
+
+func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var errResp s3err.APIErrorResponse
+	err = xml.Unmarshal(body, &errResp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != apiErr.HTTPStatusCode {
+		return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
+	}
+	if errResp.Code != apiErr.Code {
+		return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
+	}
+	if errResp.Message != apiErr.Description {
+		return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
+	}
+
+	return nil
+}
+
+func checkApiErr(err error, apiErr s3err.APIError) error {
+	if err == nil {
+		return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
+			return nil
+		}
+
+		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+	} else {
+		return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
+	}
+}
+
+func checkSdkApiErr(err error, code string) error {
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() != code {
+			return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
+		}
+		return nil
+	}
+	return err
+}
+
+func putObjects(client *s3.Client, objs []string, bucket string) error {
+	for _, key := range objs {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Key:    &key,
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
+	data = make([]byte, lgth)
+	rand.Read(data)
+	csum = sha256.Sum256(data)
+	r := bytes.NewReader(data)
+	input.Body = r
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err = client.PutObject(ctx, input)
+	cancel()
+
+	return
+}
+
+func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: &bucket,
+		Key:    &key,
+	})
+	cancel()
+	return out, err
+}
+
 func isEqual(a, b []byte) bool {
 	if len(a) != len(b) {
 		return false
@@ -83,32 +274,33 @@ func isEqual(a, b []byte) bool {
 	return true
 }
 
-func contains(name string, list []types.Object) bool {
-	for _, item := range list {
-		fmt.Println(*item.Key)
-		if strings.EqualFold(name, *item.Key) {
-			return true
+func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+	for i, item := range list1 {
+		if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
+			return false
 		}
 	}
-	return false
+
+	return true
 }
 
-func containsUID(name, id string, list []types.MultipartUpload) bool {
-	for _, item := range list {
-		if strings.EqualFold(name, *item.Key) && strings.EqualFold(id, *item.UploadId) {
-			return true
-		}
+func compareParts(parts1, parts2 []types.Part) bool {
+	if len(parts1) != len(parts2) {
+		return false
 	}
-	return false
-}
 
-func containsPart(part int32, list []types.Part) bool {
-	for _, item := range list {
-		if item.PartNumber == part {
-			return true
+	for i, prt := range parts1 {
+		if prt.PartNumber != parts2[i].PartNumber {
+			return false
+		}
+		if *prt.ETag != *parts2[i].ETag {
+			return false
 		}
 	}
-	return false
+	return true
 }
 
 func areTagsSame(tags1, tags2 []types.Tag) bool {
@@ -133,7 +325,7 @@ func containsTag(tag types.Tag, list []types.Tag) bool {
 	return false
 }
 
-func checkGrants(grts1, grts2 []types.Grant) bool {
+func compareGrants(grts1, grts2 []types.Grant) bool {
 	if len(grts1) != len(grts2) {
 		return false
 	}
@@ -161,3 +353,197 @@ func getString(str *string) string {
 	}
 	return *str
 }
+
+func getPtr(str string) *string {
+	return &str
+}
+
+func areMapsSame(mp1, mp2 map[string]string) bool {
+	if len(mp1) != len(mp2) {
+		return false
+	}
+	for key, val := range mp1 {
+		if mp2[key] != val {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[*elem.Name] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[elem.Name]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareObjects(list1 []string, list2 []types.Object) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Prefix]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
+	dr := NewDataReader(size, size)
+	datafile := "rand.data"
+	w, err := os.Create(datafile)
+	if err != nil {
+		return parts, err
+	}
+	defer w.Close()
+
+	_, err = io.Copy(w, dr)
+	if err != nil {
+		return parts, err
+	}
+
+	fileInfo, err := w.Stat()
+	if err != nil {
+		return parts, err
+	}
+
+	partSize := fileInfo.Size() / int64(partCount)
+	var offset int64
+
+	for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
+		partStart := (partNumber - 1) * partSize
+		partEnd := partStart + partSize - 1
+		if partEnd > fileInfo.Size()-1 {
+			partEnd = fileInfo.Size() - 1
+		}
+		partBuffer := make([]byte, partEnd-partStart+1)
+		_, err := w.ReadAt(partBuffer, partStart)
+		if err != nil {
+			return parts, err
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     &bucket,
+			Key:        &key,
+			UploadId:   &uploadId,
+			Body:       bytes.NewReader(partBuffer),
+			PartNumber: int32(partNumber),
+		})
+		cancel()
+		if err != nil {
+			return parts, err
+		} else {
+			parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
+			offset += partSize
+		}
+	}
+
+	return parts, err
+}
+
+type user struct {
+	access string
+	secret string
+	role   string
+}
+
+func createUsers(s *S3Conf, users []user) error {
+	for _, usr := range users {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
+			return fmt.Errorf("failed to create a user account")
+		}
+	}
+	return nil
+}
+
+func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
+	for _, bucket := range buckets {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
+			return fmt.Errorf(string(out))
+		}
+	}
+
+	return nil
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func genRandString(length int) string {
+	source := rnd.NewSource(time.Now().UnixNano())
+	random := rnd.New(source)
+	result := make([]byte, length)
+	for i := range result {
+		result[i] = charset[random.Intn(len(charset))]
+	}
+	return string(result)
+}

runtests.sh

@@ -6,7 +6,7 @@ rm -rf /tmp/covdata
 mkdir /tmp/covdata
 
 # run server in background
-GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass posix /tmp/gw &
+GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
 GW_PID=$!
 
 # wait a second for server to start up

s3api/admin-router.go

@@ -0,0 +1,43 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/controllers"
+)
+
+type S3AdminRouter struct{}
+
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
+	controller := controllers.NewAdminController(iam, be)
+
+	// CreateUser admin api
+	app.Patch("/create-user", controller.CreateUser)
+
+	// DeleteUsers admin api
+	app.Patch("/delete-user", controller.DeleteUser)
+
+	// ListUsers admin api
+	app.Patch("/list-users", controller.ListUsers)
+
+	// ChangeBucketOwner admin api
+	app.Patch("/change-bucket-owner", controller.ChangeBucketOwner)
+
+	// ListBucketsAndOwners admin api
+	app.Patch("/list-buckets", controller.ListBuckets)
+}

s3api/admin-server.go

@@ -0,0 +1,72 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3api
+
+import (
+	"crypto/tls"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3api/middlewares"
+)
+
+type S3AdminServer struct {
+	app     *fiber.App
+	backend backend.Backend
+	router  *S3AdminRouter
+	port    string
+	cert    *tls.Certificate
+}
+
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
+	server := &S3AdminServer{
+		app:     app,
+		backend: be,
+		router:  new(S3AdminRouter),
+		port:    port,
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
+
+	// Logging middlewares
+	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(nil))
+
+	// Authentication middlewares
+	app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(nil))
+	app.Use(middlewares.AclParser(be, nil))
+
+	server.router.Init(app, be, iam)
+
+	return server
+}
+
+type AdminOpt func(s *S3AdminServer)
+
+func WithAdminSrvTLS(cert tls.Certificate) AdminOpt {
+	return func(s *S3AdminServer) { s.cert = &cert }
+}
+
+func (sa *S3AdminServer) Serve() (err error) {
+	if sa.cert != nil {
+		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
+	}
+	return sa.app.Listen(sa.port)
+}

s3api/controllers/admin.go

@@ -15,30 +15,39 @@
 package controllers
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 )
 
 type AdminController struct {
-	IAMService auth.IAMService
+	iam auth.IAMService
+	be  backend.Backend
+}
+
+func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
+	return AdminController{iam: iam, be: be}
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
-	access, secret, role := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role")
-	requesterRole := ctx.Locals("role").(string)
-
-	if requesterRole != "admin" {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
 		return fmt.Errorf("access denied: only admin users have access to this resource")
 	}
-	if role != "user" && role != "admin" {
+	var usr auth.Account
+	err := json.Unmarshal(ctx.Body(), &usr)
+	if err != nil {
+		return fmt.Errorf("failed to parse request body: %w", err)
+	}
+
+	if usr.Role != "user" && usr.Role != "admin" {
 		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
 	}
 
-	user := auth.Account{Secret: secret, Role: role}
-
-	err := c.IAMService.CreateAccount(access, user)
+	err = c.iam.CreateAccount(usr)
 	if err != nil {
 		return fmt.Errorf("failed to create a user: %w", err)
 	}
@@ -48,15 +57,66 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
-	requesterRole := ctx.Locals("role").(string)
-	if requesterRole != "admin" {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
 		return fmt.Errorf("access denied: only admin users have access to this resource")
 	}
 
-	err := c.IAMService.DeleteUserAccount(access)
+	err := c.iam.DeleteUserAccount(access)
 	if err != nil {
 		return err
 	}
 
 	return ctx.SendString("The user has been deleted successfully")
 }
+
+func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	accs, err := c.iam.ListUserAccounts()
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(accs)
+}
+
+func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+	owner := ctx.Query("owner")
+	bucket := ctx.Query("bucket")
+
+	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return fmt.Errorf("user specified as the new bucket owner does not exist")
+	}
+
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
+	if err != nil {
+		return err
+	}
+
+	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+}
+
+func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+
+	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(buckets)
+}

s3api/controllers/admin_test.go

@@ -15,12 +15,17 @@
 package controllers
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3response"
 )
 
 func TestAdminController_CreateUser(t *testing.T) {
@@ -29,8 +34,8 @@ func TestAdminController_CreateUser(t *testing.T) {
 	}
 
 	adminController := AdminController{
-		IAMService: &IAMServiceMock{
-			CreateAccountFunc: func(access string, account auth.Account) error {
+		iam: &IAMServiceMock{
+			CreateAccountFunc: func(account auth.Account) error {
 				return nil
 			},
 		},
@@ -39,7 +44,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 	app := fiber.New()
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "admin")
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
 		return ctx.Next()
 	})
 
@@ -48,10 +53,22 @@ func TestAdminController_CreateUser(t *testing.T) {
 	appErr := fiber.New()
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "user")
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
 		return ctx.Next()
 	})
 
+	usr := auth.Account{
+		Access: "access",
+		Secret: "secret",
+		Role:   "invalid role",
+	}
+
+	user, _ := json.Marshal(&usr)
+
+	usr.Role = "admin"
+
+	succUsr, _ := json.Marshal(&usr)
+
 	appErr.Patch("/create-user", adminController.CreateUser)
 
 	tests := []struct {
@@ -65,7 +82,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 			name: "Admin-create-user-success",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=user", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
 			},
 			wantErr:    false,
 			statusCode: 200,
@@ -74,7 +91,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 			name: "Admin-create-user-invalid-user-role",
 			app:  app,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=invalid", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -83,7 +100,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 			name: "Admin-create-user-invalid-requester-role",
 			app:  appErr,
 			args: args{
-				req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=admin", nil),
+				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -108,7 +125,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	}
 
 	adminController := AdminController{
-		IAMService: &IAMServiceMock{
+		iam: &IAMServiceMock{
 			DeleteUserAccountFunc: func(access string) error {
 				return nil
 			},
@@ -118,7 +135,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	app := fiber.New()
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "admin")
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
 		return ctx.Next()
 	})
 
@@ -127,7 +144,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	appErr := fiber.New()
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("role", "user")
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
 		return ctx.Next()
 	})
 
@@ -171,3 +188,294 @@ func TestAdminController_DeleteUser(t *testing.T) {
 		}
 	}
 }
+
+func TestAdminController_ListUsers(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			ListUserAccountsFunc: func() ([]auth.Account, error) {
+				return []auth.Account{}, fmt.Errorf("server error")
+			},
+		},
+	}
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/list-users", adminControllerErr.ListUsers)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-users", adminController.ListUsers)
+
+	appSucc := fiber.New()
+
+	appSucc.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appSucc.Patch("/list-users", adminController.ListUsers)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-list-users-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-iam-error",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Admin-list-users-success",
+			app:  appSucc,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListUsers() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListUsers() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ChangeBucketOwner(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
+				return nil
+			},
+		},
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, nil
+			},
+		},
+	}
+
+	adminControllerIamErr := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, fmt.Errorf("unknown server error")
+			},
+		},
+	}
+
+	adminControllerIamAccDoesNotExist := AdminController{
+		iam: &IAMServiceMock{
+			GetUserAccountFunc: func(access string) (auth.Account, error) {
+				return auth.Account{}, auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+	appIamErr := fiber.New()
+
+	appIamErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
+
+	appIamNoSuchUser := fiber.New()
+
+	appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Change-bucket-owner-access-denied",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-check-account-server-error",
+			app:  appIamErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-acc-does-not-exist",
+			app:  appIamNoSuchUser,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "Change-bucket-owner-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
+			},
+			wantErr:    false,
+			statusCode: 201,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ChangeBucketOwner() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ChangeBucketOwner() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
+func TestAdminController_ListBuckets(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+	adminController := AdminController{
+		be: &BackendMock{
+			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
+				return []s3response.Bucket{}, nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/list-buckets", adminController.ListBuckets)
+
+	appRoleErr := fiber.New()
+
+	appRoleErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "List-buckets-incorrect-role",
+			app:  appRoleErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 500,
+		},
+		{
+			name: "List-buckets-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.ListBuckets() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}

s3api/controllers/backend_moq_test.go

@@ -25,6 +25,9 @@ var _ backend.Backend = &BackendMock{}
 //			AbortMultipartUploadFunc: func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error {
 //				panic("mock out the AbortMultipartUpload method")
 //			},
+//			ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, newOwner string) error {
+//				panic("mock out the ChangeBucketOwner method")
+//			},
 //			CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 //				panic("mock out the CompleteMultipartUpload method")
 //			},
@@ -43,6 +46,9 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
 //				panic("mock out the DeleteObject method")
 //			},
+//			DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error {
+//				panic("mock out the DeleteObjectTagging method")
+//			},
 //			DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
 //				panic("mock out the DeleteObjects method")
 //			},
@@ -58,8 +64,8 @@ var _ backend.Backend = &BackendMock{}
 //			GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
 //				panic("mock out the GetObjectAttributes method")
 //			},
-//			GetTagsFunc: func(contextMoqParam context.Context, bucket string, object string) (map[string]string, error) {
-//				panic("mock out the GetTags method")
+//			GetObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) (map[string]string, error) {
+//				panic("mock out the GetObjectTagging method")
 //			},
 //			HeadBucketFunc: func(contextMoqParam context.Context, headBucketInput *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
 //				panic("mock out the HeadBucket method")
@@ -67,9 +73,12 @@ var _ backend.Backend = &BackendMock{}
 //			HeadObjectFunc: func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 //				panic("mock out the HeadObject method")
 //			},
-//			ListBucketsFunc: func(contextMoqParam context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) {
+//			ListBucketsFunc: func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
 //				panic("mock out the ListBuckets method")
 //			},
+//			ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
+//				panic("mock out the ListBucketsAndOwners method")
+//			},
 //			ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 //				panic("mock out the ListMultipartUploads method")
 //			},
@@ -91,8 +100,8 @@ var _ backend.Backend = &BackendMock{}
 //			PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error {
 //				panic("mock out the PutObjectAcl method")
 //			},
-//			RemoveTagsFunc: func(contextMoqParam context.Context, bucket string, object string) error {
-//				panic("mock out the RemoveTags method")
+//			PutObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error {
+//				panic("mock out the PutObjectTagging method")
 //			},
 //			RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error {
 //				panic("mock out the RestoreObject method")
@@ -100,9 +109,6 @@ var _ backend.Backend = &BackendMock{}
 //			SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
 //				panic("mock out the SelectObjectContent method")
 //			},
-//			SetTagsFunc: func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error {
-//				panic("mock out the SetTags method")
-//			},
 //			ShutdownFunc: func()  {
 //				panic("mock out the Shutdown method")
 //			},
@@ -125,6 +131,9 @@ type BackendMock struct {
 	// AbortMultipartUploadFunc mocks the AbortMultipartUpload method.
 	AbortMultipartUploadFunc func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error
 
+	// ChangeBucketOwnerFunc mocks the ChangeBucketOwner method.
+	ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, newOwner string) error
+
 	// CompleteMultipartUploadFunc mocks the CompleteMultipartUpload method.
 	CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 
@@ -143,6 +152,9 @@ type BackendMock struct {
 	// DeleteObjectFunc mocks the DeleteObject method.
 	DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error
 
+	// DeleteObjectTaggingFunc mocks the DeleteObjectTagging method.
+	DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error
+
 	// DeleteObjectsFunc mocks the DeleteObjects method.
 	DeleteObjectsFunc func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
 
@@ -158,8 +170,8 @@ type BackendMock struct {
 	// GetObjectAttributesFunc mocks the GetObjectAttributes method.
 	GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
 
-	// GetTagsFunc mocks the GetTags method.
-	GetTagsFunc func(contextMoqParam context.Context, bucket string, object string) (map[string]string, error)
+	// GetObjectTaggingFunc mocks the GetObjectTagging method.
+	GetObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) (map[string]string, error)
 
 	// HeadBucketFunc mocks the HeadBucket method.
 	HeadBucketFunc func(contextMoqParam context.Context, headBucketInput *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
@@ -168,7 +180,10 @@ type BackendMock struct {
 	HeadObjectFunc func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
 
 	// ListBucketsFunc mocks the ListBuckets method.
-	ListBucketsFunc func(contextMoqParam context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error)
+	ListBucketsFunc func(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
+
+	// ListBucketsAndOwnersFunc mocks the ListBucketsAndOwners method.
+	ListBucketsAndOwnersFunc func(contextMoqParam context.Context) ([]s3response.Bucket, error)
 
 	// ListMultipartUploadsFunc mocks the ListMultipartUploads method.
 	ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
@@ -191,8 +206,8 @@ type BackendMock struct {
 	// PutObjectAclFunc mocks the PutObjectAcl method.
 	PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error
 
-	// RemoveTagsFunc mocks the RemoveTags method.
-	RemoveTagsFunc func(contextMoqParam context.Context, bucket string, object string) error
+	// PutObjectTaggingFunc mocks the PutObjectTagging method.
+	PutObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error
 
 	// RestoreObjectFunc mocks the RestoreObject method.
 	RestoreObjectFunc func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error
@@ -200,9 +215,6 @@ type BackendMock struct {
 	// SelectObjectContentFunc mocks the SelectObjectContent method.
 	SelectObjectContentFunc func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
 
-	// SetTagsFunc mocks the SetTags method.
-	SetTagsFunc func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error
-
 	// ShutdownFunc mocks the Shutdown method.
 	ShutdownFunc func()
 
@@ -224,6 +236,15 @@ type BackendMock struct {
 			// AbortMultipartUploadInput is the abortMultipartUploadInput argument value.
 			AbortMultipartUploadInput *s3.AbortMultipartUploadInput
 		}
+		// ChangeBucketOwner holds details about calls to the ChangeBucketOwner method.
+		ChangeBucketOwner []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// NewOwner is the newOwner argument value.
+			NewOwner string
+		}
 		// CompleteMultipartUpload holds details about calls to the CompleteMultipartUpload method.
 		CompleteMultipartUpload []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -266,6 +287,15 @@ type BackendMock struct {
 			// DeleteObjectInput is the deleteObjectInput argument value.
 			DeleteObjectInput *s3.DeleteObjectInput
 		}
+		// DeleteObjectTagging holds details about calls to the DeleteObjectTagging method.
+		DeleteObjectTagging []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Object is the object argument value.
+			Object string
+		}
 		// DeleteObjects holds details about calls to the DeleteObjects method.
 		DeleteObjects []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -303,8 +333,8 @@ type BackendMock struct {
 			// GetObjectAttributesInput is the getObjectAttributesInput argument value.
 			GetObjectAttributesInput *s3.GetObjectAttributesInput
 		}
-		// GetTags holds details about calls to the GetTags method.
-		GetTags []struct {
+		// GetObjectTagging holds details about calls to the GetObjectTagging method.
+		GetObjectTagging []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// Bucket is the bucket argument value.
@@ -332,8 +362,13 @@ type BackendMock struct {
 			ContextMoqParam context.Context
 			// Owner is the owner argument value.
 			Owner string
-			// IsRoot is the isRoot argument value.
-			IsRoot bool
+			// IsAdmin is the isAdmin argument value.
+			IsAdmin bool
+		}
+		// ListBucketsAndOwners holds details about calls to the ListBucketsAndOwners method.
+		ListBucketsAndOwners []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
 		}
 		// ListMultipartUploads holds details about calls to the ListMultipartUploads method.
 		ListMultipartUploads []struct {
@@ -386,14 +421,16 @@ type BackendMock struct {
 			// PutObjectAclInput is the putObjectAclInput argument value.
 			PutObjectAclInput *s3.PutObjectAclInput
 		}
-		// RemoveTags holds details about calls to the RemoveTags method.
-		RemoveTags []struct {
+		// PutObjectTagging holds details about calls to the PutObjectTagging method.
+		PutObjectTagging []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
 			ContextMoqParam context.Context
 			// Bucket is the bucket argument value.
 			Bucket string
 			// Object is the object argument value.
 			Object string
+			// Tags is the tags argument value.
+			Tags map[string]string
 		}
 		// RestoreObject holds details about calls to the RestoreObject method.
 		RestoreObject []struct {
@@ -409,17 +446,6 @@ type BackendMock struct {
 			// SelectObjectContentInput is the selectObjectContentInput argument value.
 			SelectObjectContentInput *s3.SelectObjectContentInput
 		}
-		// SetTags holds details about calls to the SetTags method.
-		SetTags []struct {
-			// ContextMoqParam is the contextMoqParam argument value.
-			ContextMoqParam context.Context
-			// Bucket is the bucket argument value.
-			Bucket string
-			// Object is the object argument value.
-			Object string
-			// Tags is the tags argument value.
-			Tags map[string]string
-		}
 		// Shutdown holds details about calls to the Shutdown method.
 		Shutdown []struct {
 		}
@@ -442,21 +468,24 @@ type BackendMock struct {
 		}
 	}
 	lockAbortMultipartUpload    sync.RWMutex
+	lockChangeBucketOwner       sync.RWMutex
 	lockCompleteMultipartUpload sync.RWMutex
 	lockCopyObject              sync.RWMutex
 	lockCreateBucket            sync.RWMutex
 	lockCreateMultipartUpload   sync.RWMutex
 	lockDeleteBucket            sync.RWMutex
 	lockDeleteObject            sync.RWMutex
+	lockDeleteObjectTagging     sync.RWMutex
 	lockDeleteObjects           sync.RWMutex
 	lockGetBucketAcl            sync.RWMutex
 	lockGetObject               sync.RWMutex
 	lockGetObjectAcl            sync.RWMutex
 	lockGetObjectAttributes     sync.RWMutex
-	lockGetTags                 sync.RWMutex
+	lockGetObjectTagging        sync.RWMutex
 	lockHeadBucket              sync.RWMutex
 	lockHeadObject              sync.RWMutex
 	lockListBuckets             sync.RWMutex
+	lockListBucketsAndOwners    sync.RWMutex
 	lockListMultipartUploads    sync.RWMutex
 	lockListObjects             sync.RWMutex
 	lockListObjectsV2           sync.RWMutex
@@ -464,10 +493,9 @@ type BackendMock struct {
 	lockPutBucketAcl            sync.RWMutex
 	lockPutObject               sync.RWMutex
 	lockPutObjectAcl            sync.RWMutex
-	lockRemoveTags              sync.RWMutex
+	lockPutObjectTagging        sync.RWMutex
 	lockRestoreObject           sync.RWMutex
 	lockSelectObjectContent     sync.RWMutex
-	lockSetTags                 sync.RWMutex
 	lockShutdown                sync.RWMutex
 	lockString                  sync.RWMutex
 	lockUploadPart              sync.RWMutex
@@ -510,6 +538,46 @@ func (mock *BackendMock) AbortMultipartUploadCalls() []struct {
 	return calls
 }
 
+// ChangeBucketOwner calls ChangeBucketOwnerFunc.
+func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, bucket string, newOwner string) error {
+	if mock.ChangeBucketOwnerFunc == nil {
+		panic("BackendMock.ChangeBucketOwnerFunc: method is nil but Backend.ChangeBucketOwner was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		NewOwner        string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		NewOwner:        newOwner,
+	}
+	mock.lockChangeBucketOwner.Lock()
+	mock.calls.ChangeBucketOwner = append(mock.calls.ChangeBucketOwner, callInfo)
+	mock.lockChangeBucketOwner.Unlock()
+	return mock.ChangeBucketOwnerFunc(contextMoqParam, bucket, newOwner)
+}
+
+// ChangeBucketOwnerCalls gets all the calls that were made to ChangeBucketOwner.
+// Check the length with:
+//
+//	len(mockedBackend.ChangeBucketOwnerCalls())
+func (mock *BackendMock) ChangeBucketOwnerCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	NewOwner        string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		NewOwner        string
+	}
+	mock.lockChangeBucketOwner.RLock()
+	calls = mock.calls.ChangeBucketOwner
+	mock.lockChangeBucketOwner.RUnlock()
+	return calls
+}
+
 // CompleteMultipartUpload calls CompleteMultipartUploadFunc.
 func (mock *BackendMock) CompleteMultipartUpload(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	if mock.CompleteMultipartUploadFunc == nil {
@@ -726,6 +794,46 @@ func (mock *BackendMock) DeleteObjectCalls() []struct {
 	return calls
 }
 
+// DeleteObjectTagging calls DeleteObjectTaggingFunc.
+func (mock *BackendMock) DeleteObjectTagging(contextMoqParam context.Context, bucket string, object string) error {
+	if mock.DeleteObjectTaggingFunc == nil {
+		panic("BackendMock.DeleteObjectTaggingFunc: method is nil but Backend.DeleteObjectTagging was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Object          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Object:          object,
+	}
+	mock.lockDeleteObjectTagging.Lock()
+	mock.calls.DeleteObjectTagging = append(mock.calls.DeleteObjectTagging, callInfo)
+	mock.lockDeleteObjectTagging.Unlock()
+	return mock.DeleteObjectTaggingFunc(contextMoqParam, bucket, object)
+}
+
+// DeleteObjectTaggingCalls gets all the calls that were made to DeleteObjectTagging.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteObjectTaggingCalls())
+func (mock *BackendMock) DeleteObjectTaggingCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Object          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Object          string
+	}
+	mock.lockDeleteObjectTagging.RLock()
+	calls = mock.calls.DeleteObjectTagging
+	mock.lockDeleteObjectTagging.RUnlock()
+	return calls
+}
+
 // DeleteObjects calls DeleteObjectsFunc.
 func (mock *BackendMock) DeleteObjects(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
 	if mock.DeleteObjectsFunc == nil {
@@ -910,10 +1018,10 @@ func (mock *BackendMock) GetObjectAttributesCalls() []struct {
 	return calls
 }
 
-// GetTags calls GetTagsFunc.
-func (mock *BackendMock) GetTags(contextMoqParam context.Context, bucket string, object string) (map[string]string, error) {
-	if mock.GetTagsFunc == nil {
-		panic("BackendMock.GetTagsFunc: method is nil but Backend.GetTags was just called")
+// GetObjectTagging calls GetObjectTaggingFunc.
+func (mock *BackendMock) GetObjectTagging(contextMoqParam context.Context, bucket string, object string) (map[string]string, error) {
+	if mock.GetObjectTaggingFunc == nil {
+		panic("BackendMock.GetObjectTaggingFunc: method is nil but Backend.GetObjectTagging was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
@@ -924,17 +1032,17 @@ func (mock *BackendMock) GetTags(contextMoqParam context.Context, bucket string,
 		Bucket:          bucket,
 		Object:          object,
 	}
-	mock.lockGetTags.Lock()
-	mock.calls.GetTags = append(mock.calls.GetTags, callInfo)
-	mock.lockGetTags.Unlock()
-	return mock.GetTagsFunc(contextMoqParam, bucket, object)
+	mock.lockGetObjectTagging.Lock()
+	mock.calls.GetObjectTagging = append(mock.calls.GetObjectTagging, callInfo)
+	mock.lockGetObjectTagging.Unlock()
+	return mock.GetObjectTaggingFunc(contextMoqParam, bucket, object)
 }
 
-// GetTagsCalls gets all the calls that were made to GetTags.
+// GetObjectTaggingCalls gets all the calls that were made to GetObjectTagging.
 // Check the length with:
 //
-//	len(mockedBackend.GetTagsCalls())
-func (mock *BackendMock) GetTagsCalls() []struct {
+//	len(mockedBackend.GetObjectTaggingCalls())
+func (mock *BackendMock) GetObjectTaggingCalls() []struct {
 	ContextMoqParam context.Context
 	Bucket          string
 	Object          string
@@ -944,9 +1052,9 @@ func (mock *BackendMock) GetTagsCalls() []struct {
 		Bucket          string
 		Object          string
 	}
-	mock.lockGetTags.RLock()
-	calls = mock.calls.GetTags
-	mock.lockGetTags.RUnlock()
+	mock.lockGetObjectTagging.RLock()
+	calls = mock.calls.GetObjectTagging
+	mock.lockGetObjectTagging.RUnlock()
 	return calls
 }
 
@@ -1023,23 +1131,23 @@ func (mock *BackendMock) HeadObjectCalls() []struct {
 }
 
 // ListBuckets calls ListBucketsFunc.
-func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) {
+func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
 	if mock.ListBucketsFunc == nil {
 		panic("BackendMock.ListBucketsFunc: method is nil but Backend.ListBuckets was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		Owner           string
-		IsRoot          bool
+		IsAdmin         bool
 	}{
 		ContextMoqParam: contextMoqParam,
 		Owner:           owner,
-		IsRoot:          isRoot,
+		IsAdmin:         isAdmin,
 	}
 	mock.lockListBuckets.Lock()
 	mock.calls.ListBuckets = append(mock.calls.ListBuckets, callInfo)
 	mock.lockListBuckets.Unlock()
-	return mock.ListBucketsFunc(contextMoqParam, owner, isRoot)
+	return mock.ListBucketsFunc(contextMoqParam, owner, isAdmin)
 }
 
 // ListBucketsCalls gets all the calls that were made to ListBuckets.
@@ -1049,12 +1157,12 @@ func (mock *BackendMock) ListBuckets(contextMoqParam context.Context, owner stri
 func (mock *BackendMock) ListBucketsCalls() []struct {
 	ContextMoqParam context.Context
 	Owner           string
-	IsRoot          bool
+	IsAdmin         bool
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		Owner           string
-		IsRoot          bool
+		IsAdmin         bool
 	}
 	mock.lockListBuckets.RLock()
 	calls = mock.calls.ListBuckets
@@ -1062,6 +1170,38 @@ func (mock *BackendMock) ListBucketsCalls() []struct {
 	return calls
 }
 
+// ListBucketsAndOwners calls ListBucketsAndOwnersFunc.
+func (mock *BackendMock) ListBucketsAndOwners(contextMoqParam context.Context) ([]s3response.Bucket, error) {
+	if mock.ListBucketsAndOwnersFunc == nil {
+		panic("BackendMock.ListBucketsAndOwnersFunc: method is nil but Backend.ListBucketsAndOwners was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+	}{
+		ContextMoqParam: contextMoqParam,
+	}
+	mock.lockListBucketsAndOwners.Lock()
+	mock.calls.ListBucketsAndOwners = append(mock.calls.ListBucketsAndOwners, callInfo)
+	mock.lockListBucketsAndOwners.Unlock()
+	return mock.ListBucketsAndOwnersFunc(contextMoqParam)
+}
+
+// ListBucketsAndOwnersCalls gets all the calls that were made to ListBucketsAndOwners.
+// Check the length with:
+//
+//	len(mockedBackend.ListBucketsAndOwnersCalls())
+func (mock *BackendMock) ListBucketsAndOwnersCalls() []struct {
+	ContextMoqParam context.Context
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+	}
+	mock.lockListBucketsAndOwners.RLock()
+	calls = mock.calls.ListBucketsAndOwners
+	mock.lockListBucketsAndOwners.RUnlock()
+	return calls
+}
+
 // ListMultipartUploads calls ListMultipartUploadsFunc.
 func (mock *BackendMock) ListMultipartUploads(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 	if mock.ListMultipartUploadsFunc == nil {
@@ -1318,43 +1458,47 @@ func (mock *BackendMock) PutObjectAclCalls() []struct {
 	return calls
 }
 
-// RemoveTags calls RemoveTagsFunc.
-func (mock *BackendMock) RemoveTags(contextMoqParam context.Context, bucket string, object string) error {
-	if mock.RemoveTagsFunc == nil {
-		panic("BackendMock.RemoveTagsFunc: method is nil but Backend.RemoveTags was just called")
+// PutObjectTagging calls PutObjectTaggingFunc.
+func (mock *BackendMock) PutObjectTagging(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error {
+	if mock.PutObjectTaggingFunc == nil {
+		panic("BackendMock.PutObjectTaggingFunc: method is nil but Backend.PutObjectTagging was just called")
 	}
 	callInfo := struct {
 		ContextMoqParam context.Context
 		Bucket          string
 		Object          string
+		Tags            map[string]string
 	}{
 		ContextMoqParam: contextMoqParam,
 		Bucket:          bucket,
 		Object:          object,
+		Tags:            tags,
 	}
-	mock.lockRemoveTags.Lock()
-	mock.calls.RemoveTags = append(mock.calls.RemoveTags, callInfo)
-	mock.lockRemoveTags.Unlock()
-	return mock.RemoveTagsFunc(contextMoqParam, bucket, object)
+	mock.lockPutObjectTagging.Lock()
+	mock.calls.PutObjectTagging = append(mock.calls.PutObjectTagging, callInfo)
+	mock.lockPutObjectTagging.Unlock()
+	return mock.PutObjectTaggingFunc(contextMoqParam, bucket, object, tags)
 }
 
-// RemoveTagsCalls gets all the calls that were made to RemoveTags.
+// PutObjectTaggingCalls gets all the calls that were made to PutObjectTagging.
 // Check the length with:
 //
-//	len(mockedBackend.RemoveTagsCalls())
-func (mock *BackendMock) RemoveTagsCalls() []struct {
+//	len(mockedBackend.PutObjectTaggingCalls())
+func (mock *BackendMock) PutObjectTaggingCalls() []struct {
 	ContextMoqParam context.Context
 	Bucket          string
 	Object          string
+	Tags            map[string]string
 } {
 	var calls []struct {
 		ContextMoqParam context.Context
 		Bucket          string
 		Object          string
+		Tags            map[string]string
 	}
-	mock.lockRemoveTags.RLock()
-	calls = mock.calls.RemoveTags
-	mock.lockRemoveTags.RUnlock()
+	mock.lockPutObjectTagging.RLock()
+	calls = mock.calls.PutObjectTagging
+	mock.lockPutObjectTagging.RUnlock()
 	return calls
 }
 
@@ -1430,50 +1574,6 @@ func (mock *BackendMock) SelectObjectContentCalls() []struct {
 	return calls
 }
 
-// SetTags calls SetTagsFunc.
-func (mock *BackendMock) SetTags(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error {
-	if mock.SetTagsFunc == nil {
-		panic("BackendMock.SetTagsFunc: method is nil but Backend.SetTags was just called")
-	}
-	callInfo := struct {
-		ContextMoqParam context.Context
-		Bucket          string
-		Object          string
-		Tags            map[string]string
-	}{
-		ContextMoqParam: contextMoqParam,
-		Bucket:          bucket,
-		Object:          object,
-		Tags:            tags,
-	}
-	mock.lockSetTags.Lock()
-	mock.calls.SetTags = append(mock.calls.SetTags, callInfo)
-	mock.lockSetTags.Unlock()
-	return mock.SetTagsFunc(contextMoqParam, bucket, object, tags)
-}
-
-// SetTagsCalls gets all the calls that were made to SetTags.
-// Check the length with:
-//
-//	len(mockedBackend.SetTagsCalls())
-func (mock *BackendMock) SetTagsCalls() []struct {
-	ContextMoqParam context.Context
-	Bucket          string
-	Object          string
-	Tags            map[string]string
-} {
-	var calls []struct {
-		ContextMoqParam context.Context
-		Bucket          string
-		Object          string
-		Tags            map[string]string
-	}
-	mock.lockSetTags.RLock()
-	calls = mock.calls.SetTags
-	mock.lockSetTags.RUnlock()
-	return calls
-}
-
 // Shutdown calls ShutdownFunc.
 func (mock *BackendMock) Shutdown() {
 	if mock.ShutdownFunc == nil {

s3api/controllers/base.go

@@ -25,7 +25,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
@@ -54,8 +53,8 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs
 }
 
 func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error {
-	access, isRoot := ctx.Locals("access").(string), ctx.Locals("isRoot").(bool)
-	res, err := c.be.ListBuckets(ctx.Context(), access, isRoot)
+	acct := ctx.Locals("account").(auth.Account)
+	res, err := c.be.ListBuckets(ctx.Context(), acct.Access, acct.Role == "admin")
 	return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"})
 }
 
@@ -67,7 +66,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	maxParts := ctx.QueryInt("max-parts", 0)
 	partNumberMarker := ctx.Query("part-number-marker")
 	acceptRange := ctx.Get("Range")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	if keyEnd != "" {
@@ -75,11 +74,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
 
-		tags, err := c.be.GetTags(ctx.Context(), bucket, key)
+		tags, err := c.be.GetObjectTagging(ctx.Context(), bucket, key)
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
@@ -103,7 +102,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			}
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -118,7 +117,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner})
 		}
 		res, err := c.be.GetObjectAcl(ctx.Context(), &s3.GetObjectAclInput{
@@ -129,7 +128,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner})
 		}
 		var oattrs []types.ObjectAttributes
@@ -144,7 +143,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -221,13 +220,16 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	cToken := ctx.Query("continuation-token")
 	marker := ctx.Query("marker")
 	delimiter := ctx.Query("delimiter")
-	maxkeys := ctx.QueryInt("max-keys")
-	access := ctx.Locals("access").(string)
+	maxkeysStr := ctx.Query("max-keys")
+	keyMarker := ctx.Query("key-marker")
+	maxUploadsStr := ctx.Query("max-uploads")
+	uploadIdMarker := ctx.Query("upload-id-marker")
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetBucketAcl", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -241,72 +243,103 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("uploads") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
 		}
-		res, err := c.be.ListMultipartUploads(ctx.Context(), &s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
+		maxUploads, err := utils.ParseUint(maxUploadsStr)
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err, &MetaOpts{
+				Logger:      c.logger,
+				Action:      "ListMultipartUploads",
+				BucketOwner: parsedAcl.Owner,
+			})
+		}
+		res, err := c.be.ListMultipartUploads(ctx.Context(), &s3.ListMultipartUploadsInput{
+			Bucket:         &bucket,
+			Delimiter:      &delimiter,
+			Prefix:         &prefix,
+			UploadIdMarker: &uploadIdMarker,
+			MaxUploads:     maxUploads,
+			KeyMarker:      &keyMarker,
+		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
 	}
 
 	if ctx.QueryInt("list-type") == 2 {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
 		}
+		maxkeys, err := utils.ParseUint(maxkeysStr)
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err, &MetaOpts{
+				Logger:      c.logger,
+				Action:      "ListObjectsV2",
+				BucketOwner: parsedAcl.Owner,
+			})
+		}
 		res, err := c.be.ListObjectsV2(ctx.Context(), &s3.ListObjectsV2Input{
 			Bucket:            &bucket,
 			Prefix:            &prefix,
 			ContinuationToken: &cToken,
 			Delimiter:         &delimiter,
-			MaxKeys:           int32(maxkeys),
+			MaxKeys:           maxkeys,
 		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 		return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
 	}
 
+	maxkeys, err := utils.ParseUint(maxkeysStr)
+	if err != nil {
+		return SendXMLResponse(ctx, nil, err, &MetaOpts{
+			Logger:      c.logger,
+			Action:      "ListObjects",
+			BucketOwner: parsedAcl.Owner,
+		})
+	}
+
 	res, err := c.be.ListObjects(ctx.Context(), &s3.ListObjectsInput{
 		Bucket:    &bucket,
 		Prefix:    &prefix,
 		Marker:    &marker,
 		Delimiter: &delimiter,
-		MaxKeys:   int32(maxkeys),
+		MaxKeys:   maxkeys,
 	})
 	return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
 }
 
 func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
-	bucket, bucketOwner, acl, grantFullControl, grantRead, grantReadACP, granWrite, grantWriteACP, access, isRoot :=
+	bucket, acl, grantFullControl, grantRead, grantReadACP, granWrite, grantWriteACP, acct, isRoot :=
 		ctx.Params("bucket"),
-		ctx.Get("X-Amz-Expected-Bucket-Owner"),
 		ctx.Get("X-Amz-Acl"),
 		ctx.Get("X-Amz-Grant-Full-Control"),
 		ctx.Get("X-Amz-Grant-Read"),
 		ctx.Get("X-Amz-Grant-Read-Acp"),
 		ctx.Get("X-Amz-Grant-Write"),
 		ctx.Get("X-Amz-Grant-Write-Acp"),
-		ctx.Locals("access").(string),
+		ctx.Locals("account").(auth.Account),
 		ctx.Locals("isRoot").(bool)
 
 	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 		var input *s3.PutBucketAclInput
+		var accessControlPolicy auth.AccessControlPolicy
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil {
+		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE_ACP", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 		}
 
-		if len(ctx.Body()) > 0 {
-			if grants+acl != "" {
-				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
-			}
+		err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
+		if err != nil {
+			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
+		}
 
-			var accessControlPolicy auth.AccessControlPolicy
-			err := xml.Unmarshal(ctx.Body(), &accessControlPolicy)
-			if err != nil {
+		if len(accessControlPolicy.AccessControlList.Grants) > 0 {
+			if grants+acl != "" {
 				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 			}
 
@@ -320,14 +353,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			if acl != "private" && acl != "public-read" && acl != "public-read-write" {
 				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 			}
-			if len(ctx.Body()) > 0 || grants != "" {
+			if len(accessControlPolicy.AccessControlList.Grants) > 0 || grants != "" {
 				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner})
 			}
 
 			input = &s3.PutBucketAclInput{
 				Bucket:              &bucket,
 				ACL:                 types.BucketCannedACL(acl),
-				AccessControlPolicy: &types.AccessControlPolicy{Owner: &types.Owner{ID: &bucketOwner}},
+				AccessControlPolicy: &types.AccessControlPolicy{Owner: &accessControlPolicy.Owner},
 			}
 		}
 		if grants != "" {
@@ -338,7 +371,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				GrantReadACP:        &grantReadACP,
 				GrantWrite:          &granWrite,
 				GrantWriteACP:       &grantWriteACP,
-				AccessControlPolicy: &types.AccessControlPolicy{Owner: &types.Owner{ID: &bucketOwner}},
+				AccessControlPolicy: &types.AccessControlPolicy{Owner: &accessControlPolicy.Owner},
 				ACL:                 "",
 			}
 		}
@@ -358,9 +391,9 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 
 	err := c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{
 		Bucket:          &bucket,
-		ObjectOwnership: types.ObjectOwnership(access),
+		ObjectOwnership: types.ObjectOwnership(acct.Access),
 	})
-	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "CreateBucket", BucketOwner: access})
+	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "CreateBucket", BucketOwner: acct.Access})
 }
 
 func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
@@ -368,7 +401,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	keyStart := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	tagging := ctx.Get("x-amz-tagging")
@@ -413,14 +446,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		tags := make(map[string]string, len(objTagging.TagSet.Tags))
 
 		for _, tag := range objTagging.TagSet.Tags {
+			if len(tag.Key) > 128 || len(tag.Value) > 256 {
+				return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidTag), &MetaOpts{Logger: c.logger, Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner})
+			}
 			tags[tag.Key] = tag.Value
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
 
-		err = c.be.SetTags(ctx.Context(), bucket, keyStart, tags)
+		err = c.be.PutObjectTagging(ctx.Context(), bucket, keyStart, tags)
 		return SendResponse(ctx, err, &MetaOpts{
 			Logger:      c.logger,
 			EvSender:    c.evSender,
@@ -454,7 +490,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -538,7 +574,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if copySource != "" {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -557,6 +593,9 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
 			}
 		}
+
+		metadata := utils.GetUserMetaData(&ctx.Request().Header)
+
 		res, err := c.be.CopyObject(ctx.Context(), &s3.CopyObjectInput{
 			Bucket:                      &bucket,
 			Key:                         &keyStart,
@@ -565,6 +604,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			CopySourceIfNoneMatch:       &copySrcIfNoneMatch,
 			CopySourceIfModifiedSince:   &mtime,
 			CopySourceIfUnmodifiedSince: &umtime,
+			ExpectedBucketOwner:         &acct.Access,
+			Metadata:                    metadata,
 		})
 		if err == nil {
 			return SendXMLResponse(ctx, res, err, &MetaOpts{
@@ -587,7 +628,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 
 	metadata := utils.GetUserMetaData(&ctx.Request().Header)
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -618,27 +659,27 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 }
 
 func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
-	bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
 	}
 
 	err := c.be.DeleteBucket(ctx.Context(), &s3.DeleteBucketInput{
 		Bucket: &bucket,
 	})
-	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner})
+	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner, Status: 204})
 }
 
 func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
-	bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
 	var dObj s3response.DeleteObjects
 
 	if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
 		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -656,7 +697,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
@@ -665,15 +706,16 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RemoveObjectTagging", BucketOwner: parsedAcl.Owner})
 		}
 
-		err := c.be.RemoveTags(ctx.Context(), bucket, key)
+		err := c.be.DeleteObjectTagging(ctx.Context(), bucket, key)
 		return SendResponse(ctx, err, &MetaOpts{
+			Status:      http.StatusNoContent,
 			Logger:      c.logger,
 			EvSender:    c.evSender,
-			Action:      "RemoveObjectTagging",
+			Action:      "DeleteObjectTagging",
 			BucketOwner: parsedAcl.Owner,
 			EventName:   s3event.EventObjectTaggingDelete,
 		})
@@ -682,7 +724,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	if uploadId != "" {
 		expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer")
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -693,10 +735,10 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			ExpectedBucketOwner: &expectedBucketOwner,
 			RequestPayer:        types.RequestPayer(requestPayer),
 		})
-		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner})
+		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "AbortMultipartUpload", BucketOwner: parsedAcl.Owner, Status: 204})
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObject", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -710,13 +752,14 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 		Action:      "DeleteObject",
 		BucketOwner: parsedAcl.Owner,
 		EventName:   s3event.EventObjectDelete,
+		Status:      204,
 	})
 }
 
 func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
-	bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -732,14 +775,14 @@ const (
 )
 
 func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
-	bucket, access, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
+	bucket, acct, isRoot, parsedAcl := ctx.Params("bucket"), ctx.Locals("account").(auth.Account), ctx.Locals("isRoot").(bool), ctx.Locals("parsedAcl").(auth.ACL)
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	if keyEnd != "" {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 		return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -798,7 +841,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	access := ctx.Locals("access").(string)
+	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
@@ -806,6 +849,11 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
+	path := ctx.Path()
+	if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" {
+		key = key + "/"
+	}
+
 	var restoreRequest s3.RestoreObjectInput
 	if ctx.Request().URI().QueryArgs().Has("restore") {
 		err := xml.Unmarshal(ctx.Body(), &restoreRequest)
@@ -813,7 +861,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RestoreObject", BucketOwner: parsedAcl.Owner})
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RestoreObject", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -841,7 +889,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			})
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -871,7 +919,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			})
 		}
 
-		if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+		if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner})
 		}
 
@@ -902,7 +950,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 		}
 	}
 
-	if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil {
+	if err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot); err != nil {
 		return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CreateMultipartUpload", BucketOwner: parsedAcl.Owner})
 	}
 
@@ -920,6 +968,7 @@ type MetaOpts struct {
 	EventName   s3event.EventType
 	ObjectETag  *string
 	VersionId   *string
+	Status      int
 }
 
 func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
@@ -954,9 +1003,12 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
 
 	utils.LogCtxDetails(ctx, []byte{})
 
+	if l.Status == 0 {
+		l.Status = http.StatusOK
+	}
 	// https://github.com/gofiber/fiber/issues/2080
 	// ctx.SendStatus() sets incorrect content length on HEAD request
-	ctx.Status(http.StatusOK)
+	ctx.Status(l.Status)
 	return nil
 }
 
@@ -990,6 +1042,7 @@ func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *MetaOpts) error {
 		}
 
 		if len(b) > 0 {
+			ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(b)))
 			ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
 		}
 	}

s3api/controllers/base_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
@@ -90,9 +91,6 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
-				return acldata, nil
-			},
 			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, nil
 			},
@@ -100,8 +98,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
-		ctx.Locals("isRoot", true)
+		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
 		ctx.Locals("isDebug", false)
 		return ctx.Next()
 	})
@@ -111,9 +108,6 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr := fiber.New()
 	s3ApiControllerErr := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
-				return acldata, nil
-			},
 			ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 				return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
 			},
@@ -121,8 +115,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
-		ctx.Locals("isRoot", true)
+		ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
 		ctx.Locals("isDebug", false)
 		return ctx.Next()
 	})
@@ -206,13 +199,13 @@ func TestS3ApiController_GetActions(t *testing.T) {
 					StorageClass:    "storage class",
 				}, nil
 			},
-			GetTagsFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
+			GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
 				return map[string]string{"hello": "world"}, nil
 			},
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -352,7 +345,7 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -374,7 +367,7 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	}
 	appError := fiber.New()
 	appError.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -481,6 +474,22 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</AccessControlPolicy>
 	`
 
+	invOwnerBody := `
+	<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Owner>
+			<ID>hello</ID>
+		</Owner>
+	</AccessControlPolicy>
+	`
+
+	succBody := `
+	<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<Owner>
+			<ID>valid access</ID>
+		</Owner>
+	</AccessControlPolicy>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -496,7 +505,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	}
 	// Mock ctx.Locals
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{Owner: "valid access"})
@@ -514,14 +523,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	errAclReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	// PutBucketAcl incorrect bucket owner case
-	incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
+	incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
 	incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
-	incorrectBucketOwner.Header.Set("X-Amz-Expected-Bucket-Owner", "invalid access")
 
 	// PutBucketAcl acl success
-	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
+	aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
 	aclSuccReq.Header.Set("X-Amz-Acl", "private")
-	aclSuccReq.Header.Set("X-Amz-Expected-Bucket-Owner", "valid access")
 
 	// Invalid acl body case
 	errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
@@ -662,7 +669,7 @@ func TestS3ApiController_PutActions(t *testing.T) {
 			UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
 				return "hello", nil
 			},
-			SetTagsFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
+			PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
 				return nil
 			},
 			UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
@@ -671,7 +678,7 @@ func TestS3ApiController_PutActions(t *testing.T) {
 		},
 	}
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -870,7 +877,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -893,7 +900,7 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 	}
 	for _, tt := range tests {
@@ -927,7 +934,7 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -997,14 +1004,14 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 			AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
 				return nil
 			},
-			RemoveTagsFunc: func(_ context.Context, bucket, object string) error {
+			DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error {
 				return nil
 			},
 		},
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -1025,7 +1032,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	}}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -1047,7 +1054,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key?uploadId=324234", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 		{
 			name: "Remove-object-tagging-success",
@@ -1056,7 +1063,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key/key2?tagging", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 		{
 			name: "Delete-object-success",
@@ -1065,7 +1072,7 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 				req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key", nil),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 204,
 		},
 		{
 			name: "Delete-object-error",
@@ -1108,7 +1115,7 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -1131,7 +1138,7 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -1210,7 +1217,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	}
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -1233,7 +1240,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	}
 
 	appErr.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -1313,7 +1320,7 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	`
 
 	app.Use(func(ctx *fiber.Ctx) error {
-		ctx.Locals("access", "valid access")
+		ctx.Locals("account", auth.Account{Access: "valid access"})
 		ctx.Locals("isRoot", true)
 		ctx.Locals("isDebug", false)
 		ctx.Locals("parsedAcl", auth.ACL{})
@@ -1411,16 +1418,9 @@ func Test_XMLresponse(t *testing.T) {
 		resp any
 		err  error
 	}
+
 	app := fiber.New()
-
-	var ctx fiber.Ctx
-	// Mocking the fiber ctx
-	app.Get("/:bucket/:key", func(c *fiber.Ctx) error {
-		ctx = *c
-		return nil
-	})
-
-	app.Test(httptest.NewRequest(http.MethodGet, "/my-bucket/my-key", nil))
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
 		name       string
@@ -1431,9 +1431,9 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Internal-server-error",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(16),
+				err:  s3err.GetAPIError(s3err.ErrInternalError),
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -1441,9 +1441,9 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Error-not-implemented",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(50),
+				err:  s3err.GetAPIError(s3err.ErrNotImplemented),
 			},
 			wantErr:    false,
 			statusCode: 501,
@@ -1451,7 +1451,7 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Invalid-request-body",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: make(chan int),
 				err:  nil,
 			},
@@ -1461,7 +1461,7 @@ func Test_XMLresponse(t *testing.T) {
 		{
 			name: "Successful-response",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: "Valid response",
 				err:  nil,
 			},
@@ -1491,16 +1491,11 @@ func Test_response(t *testing.T) {
 		ctx  *fiber.Ctx
 		resp any
 		err  error
+		opts *MetaOpts
 	}
-	app := fiber.New()
-	var ctx fiber.Ctx
-	// Mocking the fiber ctx
-	app.Get("/:bucket/:key", func(c *fiber.Ctx) error {
-		ctx = *c
-		return nil
-	})
 
-	app.Test(httptest.NewRequest(http.MethodGet, "/my-bucket/my-key", nil))
+	app := fiber.New()
+	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
 		name       string
@@ -1511,9 +1506,10 @@ func Test_response(t *testing.T) {
 		{
 			name: "Internal-server-error",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(16),
+				err:  s3err.GetAPIError(s3err.ErrInternalError),
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -1521,9 +1517,10 @@ func Test_response(t *testing.T) {
 		{
 			name: "Internal-server-error-not-api",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
 				err:  fmt.Errorf("custom error"),
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 500,
@@ -1531,9 +1528,10 @@ func Test_response(t *testing.T) {
 		{
 			name: "Error-not-implemented",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: nil,
-				err:  s3err.GetAPIError(50),
+				err:  s3err.GetAPIError(s3err.ErrNotImplemented),
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 501,
@@ -1541,17 +1539,31 @@ func Test_response(t *testing.T) {
 		{
 			name: "Successful-response",
 			args: args{
-				ctx:  &ctx,
+				ctx:  ctx,
 				resp: "Valid response",
 				err:  nil,
+				opts: &MetaOpts{},
 			},
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Successful-response-status-204",
+			args: args{
+				ctx:  ctx,
+				resp: "Valid response",
+				err:  nil,
+				opts: &MetaOpts{
+					Status: 204,
+				},
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := SendResponse(tt.args.ctx, tt.args.err, &MetaOpts{}); (err != nil) != tt.wantErr {
+			if err := SendResponse(tt.args.ctx, tt.args.err, tt.args.opts); (err != nil) != tt.wantErr {
 				t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr)
 			}
 

s3api/controllers/iam_moq_test.go

@@ -18,7 +18,7 @@ var _ auth.IAMService = &IAMServiceMock{}
 //
 //		// make and configure a mocked auth.IAMService
 //		mockedIAMService := &IAMServiceMock{
-//			CreateAccountFunc: func(access string, account auth.Account) error {
+//			CreateAccountFunc: func(account auth.Account) error {
 //				panic("mock out the CreateAccount method")
 //			},
 //			DeleteUserAccountFunc: func(access string) error {
@@ -27,6 +27,12 @@ var _ auth.IAMService = &IAMServiceMock{}
 //			GetUserAccountFunc: func(access string) (auth.Account, error) {
 //				panic("mock out the GetUserAccount method")
 //			},
+//			ListUserAccountsFunc: func() ([]auth.Account, error) {
+//				panic("mock out the ListUserAccounts method")
+//			},
+//			ShutdownFunc: func() error {
+//				panic("mock out the Shutdown method")
+//			},
 //		}
 //
 //		// use mockedIAMService in code that requires auth.IAMService
@@ -35,7 +41,7 @@ var _ auth.IAMService = &IAMServiceMock{}
 //	}
 type IAMServiceMock struct {
 	// CreateAccountFunc mocks the CreateAccount method.
-	CreateAccountFunc func(access string, account auth.Account) error
+	CreateAccountFunc func(account auth.Account) error
 
 	// DeleteUserAccountFunc mocks the DeleteUserAccount method.
 	DeleteUserAccountFunc func(access string) error
@@ -43,12 +49,16 @@ type IAMServiceMock struct {
 	// GetUserAccountFunc mocks the GetUserAccount method.
 	GetUserAccountFunc func(access string) (auth.Account, error)
 
+	// ListUserAccountsFunc mocks the ListUserAccounts method.
+	ListUserAccountsFunc func() ([]auth.Account, error)
+
+	// ShutdownFunc mocks the Shutdown method.
+	ShutdownFunc func() error
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// CreateAccount holds details about calls to the CreateAccount method.
 		CreateAccount []struct {
-			// Access is the access argument value.
-			Access string
 			// Account is the account argument value.
 			Account auth.Account
 		}
@@ -62,28 +72,34 @@ type IAMServiceMock struct {
 			// Access is the access argument value.
 			Access string
 		}
+		// ListUserAccounts holds details about calls to the ListUserAccounts method.
+		ListUserAccounts []struct {
+		}
+		// Shutdown holds details about calls to the Shutdown method.
+		Shutdown []struct {
+		}
 	}
 	lockCreateAccount     sync.RWMutex
 	lockDeleteUserAccount sync.RWMutex
 	lockGetUserAccount    sync.RWMutex
+	lockListUserAccounts  sync.RWMutex
+	lockShutdown          sync.RWMutex
 }
 
 // CreateAccount calls CreateAccountFunc.
-func (mock *IAMServiceMock) CreateAccount(access string, account auth.Account) error {
+func (mock *IAMServiceMock) CreateAccount(account auth.Account) error {
 	if mock.CreateAccountFunc == nil {
 		panic("IAMServiceMock.CreateAccountFunc: method is nil but IAMService.CreateAccount was just called")
 	}
 	callInfo := struct {
-		Access  string
 		Account auth.Account
 	}{
-		Access:  access,
 		Account: account,
 	}
 	mock.lockCreateAccount.Lock()
 	mock.calls.CreateAccount = append(mock.calls.CreateAccount, callInfo)
 	mock.lockCreateAccount.Unlock()
-	return mock.CreateAccountFunc(access, account)
+	return mock.CreateAccountFunc(account)
 }
 
 // CreateAccountCalls gets all the calls that were made to CreateAccount.
@@ -91,11 +107,9 @@ func (mock *IAMServiceMock) CreateAccount(access string, account auth.Account) e
 //
 //	len(mockedIAMService.CreateAccountCalls())
 func (mock *IAMServiceMock) CreateAccountCalls() []struct {
-	Access  string
 	Account auth.Account
 } {
 	var calls []struct {
-		Access  string
 		Account auth.Account
 	}
 	mock.lockCreateAccount.RLock()
@@ -167,3 +181,57 @@ func (mock *IAMServiceMock) GetUserAccountCalls() []struct {
 	mock.lockGetUserAccount.RUnlock()
 	return calls
 }
+
+// ListUserAccounts calls ListUserAccountsFunc.
+func (mock *IAMServiceMock) ListUserAccounts() ([]auth.Account, error) {
+	if mock.ListUserAccountsFunc == nil {
+		panic("IAMServiceMock.ListUserAccountsFunc: method is nil but IAMService.ListUserAccounts was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockListUserAccounts.Lock()
+	mock.calls.ListUserAccounts = append(mock.calls.ListUserAccounts, callInfo)
+	mock.lockListUserAccounts.Unlock()
+	return mock.ListUserAccountsFunc()
+}
+
+// ListUserAccountsCalls gets all the calls that were made to ListUserAccounts.
+// Check the length with:
+//
+//	len(mockedIAMService.ListUserAccountsCalls())
+func (mock *IAMServiceMock) ListUserAccountsCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockListUserAccounts.RLock()
+	calls = mock.calls.ListUserAccounts
+	mock.lockListUserAccounts.RUnlock()
+	return calls
+}
+
+// Shutdown calls ShutdownFunc.
+func (mock *IAMServiceMock) Shutdown() error {
+	if mock.ShutdownFunc == nil {
+		panic("IAMServiceMock.ShutdownFunc: method is nil but IAMService.Shutdown was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockShutdown.Lock()
+	mock.calls.Shutdown = append(mock.calls.Shutdown, callInfo)
+	mock.lockShutdown.Unlock()
+	return mock.ShutdownFunc()
+}
+
+// ShutdownCalls gets all the calls that were made to Shutdown.
+// Check the length with:
+//
+//	len(mockedIAMService.ShutdownCalls())
+func (mock *IAMServiceMock) ShutdownCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockShutdown.RLock()
+	calls = mock.calls.Shutdown
+	mock.lockShutdown.RUnlock()
+	return calls
+}

s3api/middlewares/acl-parser.go

@@ -28,7 +28,7 @@ import (
 
 func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
-		isRoot, access := ctx.Locals("isRoot").(bool), ctx.Locals("access").(string)
+		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
 		path := ctx.Path()
 		pathParts := strings.Split(path, "/")
 		bucket := pathParts[1]
@@ -39,7 +39,7 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 			return ctx.Next()
 		}
 		if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
-			if err := auth.IsAdmin(access, isRoot); err != nil {
+			if err := auth.IsAdmin(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
 			return ctx.Next()

s3api/middlewares/authentication.go

@@ -17,6 +17,9 @@ package middlewares
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"fmt"
+	"math"
+	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -34,6 +37,7 @@ import (
 
 const (
 	iso8601Format = "20060102T150405Z"
+	YYYYMMDD      = "20060102"
 )
 
 type RootUserConfig struct {
@@ -72,14 +76,32 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if len(credKv) != 2 {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
 		}
+		// Credential variables validation
 		creds := strings.Split(credKv[1], "/")
-		if len(creds) < 4 {
+		if len(creds) != 5 {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
 		}
+		if creds[4] != "aws4_request" {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureTerminationStr), &controllers.MetaOpts{Logger: logger})
+		}
+		if creds[3] != "s3" {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureIncorrService), &controllers.MetaOpts{Logger: logger})
+		}
+		if creds[2] != region {
+			return controllers.SendResponse(ctx, s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
+				HTTPStatusCode: http.StatusForbidden,
+			}, &controllers.MetaOpts{Logger: logger})
+		}
 
-		ctx.Locals("access", creds[0])
 		ctx.Locals("isRoot", creds[0] == root.Access)
 
+		_, err := time.Parse(YYYYMMDD, creds[1])
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
+		}
+
 		signHdrKv := strings.Split(authParts[1], "=")
 		if len(signHdrKv) != 2 {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
@@ -93,7 +115,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		if err != nil {
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 		}
-		ctx.Locals("role", account.Role)
+		ctx.Locals("account", account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
@@ -107,6 +129,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
 		}
 
+		if date[:8] != creds[1] {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
+		}
+
+		// Validate the dates difference
+		err = validateDate(tdate)
+		if err != nil {
+			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+		}
+
 		hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
 		ok := isSpecialPayload(hashPayloadHeader)
 
@@ -133,6 +165,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 			AccessKeyID:     creds[0],
 			SecretAccessKey: account.Secret,
 		}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
+			options.DisableURIPathEscaping = true
 			if debug {
 				options.LogSigning = true
 				options.Logger = logging.NewStandardLogger(os.Stderr)
@@ -165,6 +198,7 @@ type accounts struct {
 func (a accounts) getAccount(access string) (auth.Account, error) {
 	if access == a.root.Access {
 		return auth.Account{
+			Access: a.root.Access,
 			Secret: a.root.Secret,
 			Role:   "admin",
 		}, nil
@@ -185,3 +219,27 @@ func isSpecialPayload(str string) bool {
 
 	return specialValues[str]
 }
+
+func validateDate(date time.Time) error {
+	now := time.Now().UTC()
+	diff := date.Unix() - now.Unix()
+
+	// Checks the dates difference to be less than a minute
+	if math.Abs(float64(diff)) > 60 {
+		if diff > 0 {
+			return s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+				HTTPStatusCode: http.StatusForbidden,
+			}
+		} else {
+			return s3err.APIError{
+				Code:           "SignatureDoesNotMatch",
+				Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+				HTTPStatusCode: http.StatusForbidden,
+			}
+		}
+	}
+
+	return nil
+}

s3api/middlewares/url-decoder.go

@@ -0,0 +1,36 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"net/url"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		reqURL := ctx.Request().URI().String()
+		decoded, err := url.Parse(reqURL)
+		if err != nil {
+			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
+		}
+		ctx.Path(decoded.Path)
+		return ctx.Next()
+	}
+}

s3api/router.go

@@ -23,20 +23,36 @@ import (
 	"github.com/versity/versitygw/s3log"
 )
 
-type S3ApiRouter struct{}
+type S3ApiRouter struct {
+	WithAdmSrv bool
+}
 
 func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
 	s3ApiController := controllers.New(be, iam, logger, evs)
-	adminController := controllers.AdminController{IAMService: iam}
 
-	app.Patch("/create-user", adminController.CreateUser)
+	if sa.WithAdmSrv {
+		adminController := controllers.NewAdminController(iam, be)
+
+		// CreateUser admin api
+		app.Patch("/create-user", adminController.CreateUser)
+
+		// DeleteUsers admin api
+		app.Patch("/delete-user", adminController.DeleteUser)
+
+		// ListUsers admin api
+		app.Patch("/list-users", adminController.ListUsers)
+
+		// ChangeBucketOwner admin api
+		app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
+
+		// ListBucketsAndOwners admin api
+		app.Patch("/list-buckets", adminController.ListBuckets)
+	}
 
-	// Admin Delete api
-	app.Patch("/delete-user", adminController.DeleteUser)
 	// ListBuckets action
 	app.Get("/", s3ApiController.ListBuckets)
 
-	// PutBucket action
+	// CreateBucket action
 	// PutBucketAcl action
 	app.Put("/:bucket", s3ApiController.PutBucketActions)
 
@@ -45,6 +61,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 
 	// HeadBucket
 	app.Head("/:bucket", s3ApiController.HeadBucket)
+
 	// GetBucketAcl action
 	// ListMultipartUploads action
 	// ListObjects action
@@ -53,22 +70,34 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 
 	// HeadObject action
 	app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
+
 	// GetObjectAcl action
 	// GetObject action
 	// ListObjectParts action
+	// GetObjectTagging action
+	// ListParts action
+	// GetObjectAttributes action
 	app.Get("/:bucket/:key/*", s3ApiController.GetActions)
+
 	// DeleteObject action
 	// AbortMultipartUpload action
+	// DeleteObjectTagging action
 	app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
+
 	// DeleteObjects action
 	app.Post("/:bucket", s3ApiController.DeleteObjects)
+
 	// CompleteMultipartUpload action
 	// CreateMultipartUpload
 	// RestoreObject action
+	// SelectObjectContent action
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
+
 	// CopyObject action
 	// PutObject action
 	// UploadPart action
 	// UploadPartCopy action
+	// PutObjectTagging action
+	// PutObjectAcl action
 	app.Put("/:bucket/:key/*", s3ApiController.PutActions)
 }

s3api/server.go

@@ -49,6 +49,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 
 	// Logging middlewares
 	app.Use(logger.New())
+	app.Use(middlewares.DecodeURL(l))
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares
@@ -69,6 +70,11 @@ func WithTLS(cert tls.Certificate) Option {
 	return func(s *S3ApiServer) { s.cert = &cert }
 }
 
+// WithAdminServer runs admin endpoints with the gateway in the same network
+func WithAdminServer() Option {
+	return func(s *S3ApiServer) { s.router.WithAdmSrv = true }
+}
+
 // WithDebug sets debug output
 func WithDebug() Option {
 	return func(s *S3ApiServer) { s.debug = true }

s3api/utils/utils.go

@@ -20,10 +20,12 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3err"
 )
 
 var (
@@ -33,13 +35,16 @@ var (
 
 func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
 	metadata = make(map[string]string)
-	headers.VisitAll(func(key, value []byte) {
-		if strings.HasPrefix(string(key), "X-Amz-Meta-") {
-			trimmedKey := strings.TrimPrefix(string(key), "X-Amz-Meta-")
+	headers.DisableNormalizing()
+	headers.VisitAllInOrder(func(key, value []byte) {
+		hKey := string(key)
+		if strings.HasPrefix(strings.ToLower(hKey), "x-amz-meta-") {
+			trimmedKey := hKey[11:]
 			headerValue := string(value)
 			metadata[trimmedKey] = headerValue
 		}
 	})
+	headers.EnableNormalizing()
 
 	return
 }
@@ -47,7 +52,7 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
 func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
 	req := ctx.Request()
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
+	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body()))
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -73,9 +78,22 @@ func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Reques
 }
 
 func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
+	ctx.Response().Header.DisableNormalizing()
 	for key, val := range meta {
-		ctx.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
+		ctx.Response().Header.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
 	}
+	ctx.Response().Header.EnableNormalizing()
+}
+
+func ParseUint(str string) (int32, error) {
+	if str == "" {
+		return 1000, nil
+	}
+	num, err := strconv.ParseUint(str, 10, 16)
+	if err != nil {
+		return 1000, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
+	}
+	return int32(num), nil
 }
 
 type CustomHeader struct {

s3api/utils/utils_test.go

@@ -79,13 +79,6 @@ func TestGetUserMetaData(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 	req := ctx.Request()
 
-	// Case 2
-	ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
-	req2 := ctx2.Request()
-
-	req2.Header.Add("X-Amz-Meta-Name", "Nick")
-	req2.Header.Add("X-Amz-Meta-Age", "27")
-
 	tests := []struct {
 		name         string
 		args         args
@@ -98,16 +91,6 @@ func TestGetUserMetaData(t *testing.T) {
 			},
 			wantMetadata: map[string]string{},
 		},
-		{
-			name: "Success-non-empty-response",
-			args: args{
-				headers: &req2.Header,
-			},
-			wantMetadata: map[string]string{
-				"Age":  "27",
-				"Name": "Nick",
-			},
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -221,3 +204,60 @@ func TestIsValidBucketName(t *testing.T) {
 		})
 	}
 }
+
+func TestParseUint(t *testing.T) {
+	type args struct {
+		str string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    int32
+		wantErr bool
+	}{
+		{
+			name: "Parse-uint-empty-string",
+			args: args{
+				str: "",
+			},
+			want:    1000,
+			wantErr: false,
+		},
+		{
+			name: "Parse-uint-invalid-number-string",
+			args: args{
+				str: "bla",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-invalid-negative-number",
+			args: args{
+				str: "-5",
+			},
+			want:    1000,
+			wantErr: true,
+		},
+		{
+			name: "Parse-uint-success",
+			args: args{
+				str: "23",
+			},
+			want:    23,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseUint(tt.args.str)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseMaxKeys() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ParseMaxKeys() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

s3err/s3err.go

@@ -97,6 +97,9 @@ const (
 	ErrNegativeExpires
 	ErrMaximumExpires
 	ErrSignatureDoesNotMatch
+	ErrSignatureDateDoesNotMatch
+	ErrSignatureTerminationStr
+	ErrSignatureIncorrService
 	ErrContentSHA256Mismatch
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
@@ -107,10 +110,12 @@ const (
 	ErrPreconditionFailed
 	ErrInvalidObjectState
 	ErrInvalidRange
+	ErrInvalidURI
 
 	// Non-AWS errors
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
+	ErrDirectoryObjectContainsData
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -189,13 +194,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "We encountered an internal error, please try again.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
-
 	ErrInvalidPart: {
 		Code:           "InvalidPart",
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -286,7 +289,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Signature header missing Signature field.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
 		Description:    "There were headers present in the request which were not signed",
@@ -322,25 +324,36 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
 		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
 	ErrRequestNotReadyYet: {
 		Code:           "AccessDenied",
 		Description:    "Request is not valid yet",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
 	ErrSignatureDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
 		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-
+	ErrSignatureDateDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureTerminationStr: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped with a valid terminator: 'aws4_request'",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureIncorrService: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "Credential should be scoped to correct service: s3",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrContentSHA256Mismatch: {
 		Code:           "XAmzContentSHA256Mismatch",
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
@@ -381,6 +394,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "The requested range is not valid for the request. Try another range.",
 		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
 	},
+	ErrInvalidURI: {
+		Code:           "InvalidURI",
+		Description:    "The specified URI couldn't be parsed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrExistingObjectIsDirectory: {
 		Code:           "ExistingObjectIsDirectory",
 		Description:    "Existing Object is a directory.",
@@ -391,6 +409,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Object parent already exists as a file.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrDirectoryObjectContainsData: {
+		Code:           "DirectoryObjectContainsData",
+		Description:    "Directory object contains data payload.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3log/file.go

@@ -65,7 +65,7 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 	lf := LogFields{}
 
 	access := "-"
-	reqURI := ctx.Request().URI().String()
+	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
 	bucket, object := path[1], strings.Join(path[2:], "/")
 	errorCode := ""

s3log/webhook.go

@@ -62,7 +62,7 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 	lf := LogFields{}
 
 	access := "-"
-	reqURI := ctx.Request().URI().String()
+	reqURI := ctx.OriginalURL()
 	path := strings.Split(ctx.Path(), "/")
 	bucket, object := path[1], strings.Join(path[2:], "/")
 	errorCode := ""

s3response/s3response.go

@@ -116,7 +116,7 @@ type DeleteObjects struct {
 
 type DeleteObjectsResult struct {
 	Deleted []types.DeletedObject
-	Errors  []types.Error
+	Error   []types.Error
 }
 type SelectObjectContentPayload struct {
 	Expression          *string
@@ -131,6 +131,11 @@ type SelectObjectContentResult struct {
 	Records  *types.RecordsEvent
 	Stats    *types.StatsEvent
 	Progress *types.ProgressEvent
-	Cont     *string
-	End      *string
+	Cont     *types.ContinuationEvent
+	End      *types.EndEvent
+}
+
+type Bucket struct {
+	Name  string `json:"name"`
+	Owner string `json:"owner"`
 }