Merge pull request #520 from versity/test_cmdline_user_s3cmd

test: s3cmd user, fix for non-bucket creating testing

.github/dependabot.yml

@@ -8,3 +8,7 @@ updates:
       dev-dependencies:
         patterns:
           - "*"
+    allow:
+      # Allow both direct and indirect updates for all packages
+      - dependency-type: "all"
+

.github/workflows/docker.yaml

@@ -12,7 +12,7 @@ jobs:
       packages: write
       contents: read
     steps:
-      - name: Check out the repo
+      - name: Checkout
         uses: actions/checkout@v4
 
       - name: Log in to Docker Hub
@@ -43,3 +43,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            VERSION=${{ github.event.release.tag_name }}
+            TIME=${{ github.event.release.published_at }}
+            BUILD=${{ github.sha }}

.github/workflows/functional.yml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go

.github/workflows/go.yml

@@ -8,10 +8,10 @@ jobs:
     steps:
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go

.github/workflows/goreleaser.yml

@@ -15,14 +15,21 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - run: git fetch --force --tags
-      - uses: actions/setup-go@v4
+
+      - name: Fetch tags
+        run: git fetch --force --tags
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
         with:
           go-version: stable
-      - uses: goreleaser/goreleaser-action@v4
+
+      - name: Run Releaser
+        uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser
           version: latest

.github/workflows/static.yml

@@ -7,16 +7,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go
  
     - name: "staticcheck"
-      uses: dominikh/staticcheck-action@v1.3.0
-      with: 
-        install-go: false
+      uses: dominikh/staticcheck-action@v1
+      with:
+        version: "latest"

.github/workflows/system.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install ShellCheck
         run: sudo apt-get install shellcheck
@@ -15,7 +15,7 @@ jobs:
         run: shellcheck -S warning ./tests/*.sh
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 'stable'
         id: go
@@ -51,8 +51,8 @@ jobs:
           export WORKSPACE=$GITHUB_WORKSPACE
           openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir /tmp/cover
-          VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+          mkdir cover iam
+          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
 
       #- name: Build and run, s3 backend
       #  run: |
@@ -70,4 +70,4 @@ jobs:
 
       - name: Coverage report
         run: |
-          go tool covdata percent -i=/tmp/cover
+          go tool covdata percent -i=cover

Dockerfile

@@ -1,5 +1,15 @@
 FROM golang:latest
 
+# Set build arguments with default values
+ARG VERSION="none"
+ARG BUILD="none"
+ARG TIME="none"
+
+# Set environment variables
+ENV VERSION=${VERSION}
+ENV BUILD=${BUILD}
+ENV TIME=${TIME}
+
 WORKDIR /app
 
 COPY go.mod ./
@@ -9,7 +19,7 @@ COPY ./ ./
 
 WORKDIR /app/cmd/versitygw
 ENV CGO_ENABLED=0
-RUN go build -o versitygw
+RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
 
 FROM alpine:latest
 

Dockerfile_test_bats

@@ -61,7 +61,6 @@ USER tester
 COPY --chown=tester:tester . /home/tester
 
 WORKDIR /home/tester
-RUN cp ${CONFIG_FILE}.default $CONFIG_FILE
 #RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
 RUN cp tests/s3cfg.local.default tests/s3cfg.local
 RUN make

README.md

@@ -15,7 +15,8 @@ Download [latest release](https://github.com/versity/versitygw/releases)
  |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
  
 ### News
-* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
+* New performance (scale up) analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
+* New performance (scale out) Part 2 analysis article [https://github.com/versity/versitygw/wiki/Performance-Part-2](https://github.com/versity/versitygw/wiki/Performance-Part-2)
 
 ### Mailing List
 Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).

auth/acl.go

@@ -206,15 +206,6 @@ func splitUnique(s, divider string) []string {
 }
 
 func verifyACL(acl ACL, access string, permission types.Permission) error {
-	// Default disabled ACL case
-	if acl.ACL == "" && len(acl.Grantees) == 0 {
-		if acl.Owner == access {
-			return nil
-		}
-
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
 	if acl.ACL != "" {
 		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -225,6 +216,9 @@ func verifyACL(acl ACL, access string, permission types.Permission) error {
 
 		return nil
 	} else {
+		if len(acl.Grantees) == 0 {
+			return nil
+		}
 		grantee := Grantee{Access: access, Permission: permission}
 		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
 
@@ -298,10 +292,20 @@ func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) e
 		return nil
 	}
 
-	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+	policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
+	if err != nil {
 		return err
 	}
-	if err := verifyBucketPolicy(ctx, be, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
+
+	// If bucket policy is not set and the ACL is default, only the owner has access
+	if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
+		return err
+	}
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
 		return err
 	}
 

auth/bucket_policy.go

@@ -15,12 +15,10 @@
 package auth
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
 
-	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -41,8 +39,13 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 
 func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
 	for _, statement := range bp.Statement {
-		if statement.isAllowed(principal, action, resource) {
-			return true
+		if statement.findMatch(principal, action, resource) {
+			switch statement.Effect {
+			case BucketPolicyAccessTypeAllow:
+				return true
+			case BucketPolicyAccessTypeDeny:
+				return false
+			}
 		}
 	}
 
@@ -83,14 +86,9 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 	return nil
 }
 
-func (bpi *BucketPolicyItem) isAllowed(principal string, action Action, resource string) bool {
+func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
 	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
-		switch bpi.Effect {
-		case BucketPolicyAccessTypeAllow:
-			return true
-		case BucketPolicyAccessTypeDeny:
-			return false
-		}
+		return true
 	}
 
 	return false
@@ -117,26 +115,23 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 	return nil
 }
 
-func verifyBucketPolicy(ctx context.Context, be backend.Backend, access, bucket, object string, action Action) error {
-	policyDoc, err := be.GetBucketPolicy(ctx, bucket)
-	if err != nil {
-		return err
-	}
+func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
 	// If bucket policy is not set
-	if len(policyDoc) == 0 {
+	if len(policy) == 0 {
 		return nil
 	}
 
 	var bucketPolicy BucketPolicy
-	if err := json.Unmarshal(policyDoc, &bucketPolicy); err != nil {
+	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
 		return err
 	}
 
 	resource := bucket
 	if object != "" {
-		resource += "" + object
+		resource += "/" + object
 	}
 
+	fmt.Println(access, action, resource)
 	if !bucketPolicy.isAllowed(access, action, resource) {
 		return s3err.GetAPIError(s3err.ErrAccessDenied)
 	}

auth/iam_ldap.go

@@ -22,7 +22,7 @@ func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objCl
 	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
-	conn, err := ldap.Dial("tcp", url)
+	conn, err := ldap.DialURL(url)
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
 	}

auth/iam_single.go

@@ -32,7 +32,7 @@ func (IAMServiceSingle) CreateAccount(account Account) error {
 
 // GetUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	return Account{}, ErrNotSupported
+	return Account{}, ErrNoSuchUser
 }
 
 // DeleteUserAccount no accounts in single tenant mode

backend/meta/meta.go

@@ -0,0 +1,26 @@
+package meta
+
+// MetadataStorer defines the interface for managing metadata.
+// When object == "", the operation is on the bucket.
+type MetadataStorer interface {
+	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+	// Returns the value of the attribute, or an error if the attribute does not exist.
+	RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
+
+	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+	// If attribute already exists, new attribute should replace existing.
+	// Returns an error if the operation fails.
+	StoreAttribute(bucket, object, attribute string, value []byte) error
+
+	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+	// Returns an error if the operation fails.
+	DeleteAttribute(bucket, object, attribute string) error
+
+	// ListAttributes lists all attributes for an object or a bucket.
+	// Returns list of attribute names, or an error if the operation fails.
+	ListAttributes(bucket, object string) ([]string, error)
+
+	// DeleteAttributes removes all attributes for an object or a bucket.
+	// Returns an error if the operation fails.
+	DeleteAttributes(bucket, object string) error
+}

backend/meta/xattr.go

@@ -0,0 +1,76 @@
+package meta
+
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/xattr"
+)
+
+const (
+	xattrPrefix = "user."
+)
+
+var (
+	// ErrNoSuchKey is returned when the key does not exist.
+	ErrNoSuchKey = errors.New("no such key")
+)
+
+type XattrMeta struct{}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
+	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
+	if errors.Is(err, errNoData) {
+		return nil, ErrNoSuchKey
+	}
+	return b, err
+}
+
+// StoreAttribute stores the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
+	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
+func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
+	err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
+	if errors.Is(err, errNoData) {
+		return ErrNoSuchKey
+	}
+	return err
+}
+
+// DeleteAttributes is not implemented for xattr since xattrs
+// are automatically removed when the file is deleted.
+func (x XattrMeta) DeleteAttributes(bucket, object string) error {
+	return nil
+}
+
+// ListAttributes lists all attributes for an object in a bucket.
+func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
+	attrs, err := xattr.List(filepath.Join(bucket, object))
+	if err != nil {
+		return nil, err
+	}
+	attributes := make([]string, 0, len(attrs))
+	for _, attr := range attrs {
+		if !isUserAttr(attr) {
+			continue
+		}
+		attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
+	}
+	return attributes, nil
+}
+
+func isUserAttr(attr string) bool {
+	return strings.HasPrefix(attr, xattrPrefix)
+}
+
+// Test is a helper function to test if xattrs are supported.
+func (x XattrMeta) Test(path string) bool {
+	_, err := xattr.Get(path, "user.test")
+	return !errors.Is(err, syscall.ENOTSUP)
+}

backend/meta/xattr_with_enodata.go

@@ -15,7 +15,7 @@
 //go:build !freebsd && !openbsd && !netbsd
 // +build !freebsd,!openbsd,!netbsd
 
-package posix
+package meta
 
 import "syscall"
 

backend/meta/xattr_without_enodata.go

@@ -15,7 +15,7 @@
 //go:build freebsd || openbsd || netbsd
 // +build freebsd openbsd netbsd
 
-package posix
+package meta
 
 import "syscall"
 

backend/mkdir.go

@@ -0,0 +1,81 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// MkdirAll borrowed from stdlib to add ability to set ownership
+// as directories are created
+
+package backend
+
+import (
+	"io/fs"
+	"os"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	// TODO: make this configurable
+	defaultDirPerm fs.FileMode = 0755
+)
+
+// MkdirAll is similar to os.MkdirAll but it will return
+// ErrObjectParentIsFile when appropriate
+// MkdirAll creates a directory named path,
+// along with any necessary parents, and returns nil,
+// or else returns an error.
+// The permission bits perm (before umask) are used for all
+// directories that MkdirAll creates.
+// Any newly created directory is set to provided uid/gid ownership.
+// If path is already a directory, MkdirAll does nothing
+// and returns nil.
+// Any directoy created will be set to provided uid/gid ownership
+// if doChown is true.
+func MkdirAll(path string, uid, gid int, doChown bool) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = MkdirAll(path[:j-1], uid, gid, doChown)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, defaultDirPerm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return err
+	}
+	if doChown {
+		err = os.Chown(path, uid, gid)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

backend/posix/with_otmpfile.go

@@ -27,30 +27,42 @@ import (
 	"strconv"
 	"syscall"
 
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 	"golang.org/x/sys/unix"
 )
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	isOTmp  bool
-	size    int64
+	f          *os.File
+	bucket     string
+	objname    string
+	isOTmp     bool
+	size       int64
+	needsChown bool
+	uid        int
+	gid        int
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+var (
+	// TODO: make this configurable
+	defaultFilePerm uint32 = 0644
+)
+
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
 	// Not all filesystems support this, so fallback to CreateTemp for when
 	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err := os.MkdirAll(dir, 0700)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -59,11 +71,27 @@ func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 		if err != nil {
 			return nil, err
 		}
-		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		tmp := &tmpfile{
+			f:          f,
+			bucket:     bucket,
+			objname:    obj,
+			size:       size,
+			needsChown: doChown,
+			uid:        uid,
+			gid:        gid,
+		}
 		// falloc is best effort, its fine if this fails
 		if size > 0 {
 			tmp.falloc()
 		}
+
+		if doChown {
+			err := f.Chown(uid, gid)
+			if err != nil {
+				return nil, fmt.Errorf("set temp file ownership: %w", err)
+			}
+		}
+
 		return tmp, nil
 	}
 
@@ -71,11 +99,29 @@ func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		isOTmp:     true,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+
 	// falloc is best effort, its fine if this fails
 	if size > 0 {
 		tmp.falloc()
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return tmp, nil
 }
 
@@ -100,6 +146,13 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
+	dir := filepath.Dir(objPath)
+
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	if err != nil {
+		return fmt.Errorf("make parent dir: %w", err)
+	}
+
 	if !tmp.isOTmp {
 		// O_TMPFILE not suported, use fallback
 		return tmp.fallbackLink()
@@ -111,14 +164,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dir, err := os.Open(filepath.Dir(objPath))
+	dirf, err := os.Open(dir)
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dir.Close()
+	defer dirf.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile (%q in %q): %w",
 			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
@@ -138,6 +191,9 @@ func (tmp *tmpfile) fallbackLink() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(fs.FileMode(defaultFilePerm))
+
 	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)

backend/posix/without_otmpfile.go

@@ -24,6 +24,9 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 )
 
 type tmpfile struct {
@@ -33,20 +36,36 @@ type tmpfile struct {
 	size    int64
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	// Create a temp file for upload while in progress (see link comments below).
-	err := os.MkdirAll(dir, 0700)
+	var err error
+	err = backend.MkdirAll(dir, uid, gid, doChown)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create temp file: %w", err)
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
 }
 
+var (
+	// TODO: make this configurable
+	defaultFilePerm fs.FileMode = 0644
+)
+
 func (tmp *tmpfile) link() error {
 	tempname := tmp.f.Name()
 	// cleanup in case anything goes wrong, if rename succeeds then
@@ -64,6 +83,9 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(defaultFilePerm)
+
 	err = tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)

backend/scoutfs/scoutfs.go

@@ -25,15 +25,23 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
 )
 
+type ScoutfsOpts struct {
+	ChownUID    bool
+	ChownGID    bool
+	GlacierMode bool
+}
+
 type ScoutFS struct {
 	*posix.Posix
 	rootfd  *os.File
@@ -49,6 +57,16 @@ type ScoutFS struct {
 	// ListObjects: if file offline, set obj storage class to GLACIER
 	// RestoreObject: add batch stage request to file
 	glaciermode bool
+
+	// chownuid/gid enable chowning of files to the account uid/gid
+	// when objects are uploaded
+	chownuid bool
+	chowngid bool
+
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -92,14 +110,6 @@ const (
 	ExtCacheDone
 )
 
-// Option sets various options for scoutfs
-type Option func(s *ScoutFS)
-
-// WithGlacierEmulation sets glacier mode emulation
-func WithGlacierEmulation() Option {
-	return func(s *ScoutFS) { s.glaciermode = true }
-}
-
 func (s *ScoutFS) Shutdown() {
 	s.Posix.Shutdown()
 	s.rootfd.Close()
@@ -110,10 +120,47 @@ func (*ScoutFS) String() string {
 	return "ScoutFS Gateway"
 }
 
+// getChownIDs returns the uid and gid that should be used for chowning
+// the object to the account uid/gid. It also returns a boolean indicating
+// if chowning is needed.
+func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
+	uid := s.euid
+	gid := s.egid
+	var needsChown bool
+	if s.chownuid && acct.UserID != s.euid {
+		uid = acct.UserID
+		needsChown = true
+	}
+	if s.chowngid && acct.GroupID != s.egid {
+		gid = acct.GroupID
+		needsChown = true
+	}
+
+	return uid, gid, needsChown
+}
+
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
-func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	if input.Key == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if input.UploadId == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
+	}
+	if input.MultipartUpload == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
@@ -174,8 +221,11 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 
 	// use totalsize=0 because we wont be writing to the file, only moving
 	// extents around.  so we dont want to fallocate this.
-	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
+	f, err := s.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return nil, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -203,9 +253,10 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		uid, gid, doChown := s.getChownIDs(acct)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
-			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			return nil, err
 		}
 	}
 	err = f.link()
@@ -310,51 +361,6 @@ func isValidMeta(val string) bool {
 	return false
 }
 
-// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
-// when appropriate
-func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(path[:j-1], perm, bucket, object)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-	return nil
-}
-
 func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key

backend/scoutfs/scoutfs_compat.go

@@ -17,7 +17,6 @@
 package scoutfs
 
 import (
-	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -29,11 +28,17 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/versity/scoutfs-go"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
-	p, err := posix.New(rootdir)
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
+	p, err := posix.New(rootdir, meta.XattrMeta{}, posix.PosixOpts{
+		ChownUID: opts.ChownUID,
+		ChownGID: opts.ChownGID,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -43,60 +48,70 @@ func New(rootdir string, opts ...Option) (*ScoutFS, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
-	s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
-	for _, opt := range opts {
-		opt(s)
-	}
-
-	return s, nil
+	return &ScoutFS{
+		Posix:    p,
+		rootfd:   f,
+		rootdir:  rootdir,
+		chownuid: opts.ChownUID,
+		chowngid: opts.ChownGID,
+	}, nil
 }
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	isOTmp  bool
-	size    int64
+	f          *os.File
+	bucket     string
+	objname    string
+	size       int64
+	needsChown bool
+	uid        int
+	gid        int
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+var (
+	// TODO: make this configurable
+	defaultFilePerm uint32 = 0644
+)
+
+func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := s.getChownIDs(acct)
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
-	// Not all filesystems support this, so fallback to CreateTemp for when
-	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
-		// O_TMPFILE not supported, try fallback
-		err := os.MkdirAll(dir, 0700)
-		if err != nil {
-			return nil, fmt.Errorf("make temp dir: %w", err)
-		}
-		f, err := os.CreateTemp(dir,
-			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-		if err != nil {
-			return nil, err
-		}
-		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
-		// falloc is best effort, its fine if this fails
-		if size > 0 {
-			tmp.falloc()
-		}
-		return tmp, nil
+		return nil, err
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+
 	// falloc is best effort, its fine if this fails
 	if size > 0 {
 		tmp.falloc()
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return tmp, nil
 }
 
@@ -121,9 +136,11 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	if !tmp.isOTmp {
-		// O_TMPFILE not suported, use fallback
-		return tmp.fallbackLink()
+	dir := filepath.Dir(objPath)
+
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	if err != nil {
+		return fmt.Errorf("make parent dir: %w", err)
 	}
 
 	procdir, err := os.Open(procfddir)
@@ -132,14 +149,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dir, err := os.Open(filepath.Dir(objPath))
+	dirf, err := os.Open(dir)
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dir.Close()
+	defer dirf.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile: %w", err)
 	}
@@ -152,26 +169,6 @@ func (tmp *tmpfile) link() error {
 	return nil
 }
 
-func (tmp *tmpfile) fallbackLink() error {
-	tempname := tmp.f.Name()
-	// cleanup in case anything goes wrong, if rename succeeds then
-	// this will no longer exist
-	defer os.Remove(tempname)
-
-	err := tmp.f.Close()
-	if err != nil {
-		return fmt.Errorf("close tmpfile: %w", err)
-	}
-
-	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
-}
-
 func (tmp *tmpfile) Write(b []byte) (int, error) {
 	if int64(len(b)) > tmp.size {
 		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)

backend/scoutfs/scoutfs_incompat.go

@@ -20,9 +20,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
+
+	"github.com/versity/versitygw/auth"
 )
 
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
@@ -34,7 +36,12 @@ var (
 	errNotSupported = errors.New("not supported")
 )
 
-func openTmpFile(_, _, _ string, _ int64) (*tmpfile, error) {
+func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile, error) {
+	// make these look used for static check
+	_ = s.chownuid
+	_ = s.chowngid
+	_ = s.euid
+	_ = s.egid
 	return nil, errNotSupported
 }
 

cmd/versitygw/admin.go

@@ -17,6 +17,7 @@ package main
 import (
 	"bytes"
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -37,6 +38,7 @@ var (
 	adminAccess   string
 	adminSecret   string
 	adminEndpoint string
+	allowInsecure bool
 )
 
 func adminCommand() *cli.Command {
@@ -154,10 +156,24 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminEndpoint,
 			},
+			&cli.BoolFlag{
+				Name:        "allow-insecure",
+				Usage:       "disable tls certificate verification for the admin endpoint",
+				EnvVars:     []string{"ADMIN_ALLOW_INSECURE"},
+				Aliases:     []string{"ai"},
+				Destination: &allowInsecure,
+			},
 		},
 	}
 }
 
+func initHTTPClient() *http.Client {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
+	}
+	return &http.Client{Transport: tr}
+}
+
 func createUser(ctx *cli.Context) error {
 	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
 	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
@@ -199,18 +215,22 @@ func createUser(ctx *cli.Context) error {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := http.Client{}
+	client := initHTTPClient()
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
+	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
 
 	fmt.Printf("%s\n", body)
 
@@ -240,18 +260,22 @@ func deleteUser(ctx *cli.Context) error {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := http.Client{}
+	client := initHTTPClient()
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
+	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
 
 	fmt.Printf("%s\n", body)
 
@@ -276,18 +300,18 @@ func listUsers(ctx *cli.Context) error {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := http.Client{}
+	client := initHTTPClient()
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
+	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
 		return fmt.Errorf("%s", body)
@@ -343,18 +367,22 @@ func changeBucketOwner(ctx *cli.Context) error {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := http.Client{}
+	client := initHTTPClient()
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
+	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
 
 	fmt.Println(string(body))
 
@@ -391,18 +419,18 @@ func listBuckets(ctx *cli.Context) error {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := http.Client{}
+	client := initHTTPClient()
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
+	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
 		return fmt.Errorf("%s", body)

cmd/versitygw/gateway_test.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/tests/integration"
 )
@@ -56,7 +57,7 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir)
+	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}

cmd/versitygw/main.go

@@ -19,6 +19,8 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"github.com/gofiber/fiber/v2"
@@ -40,10 +42,13 @@ var (
 	certFile, keyFile                      string
 	kafkaURL, kafkaTopic, kafkaKey         string
 	natsURL, natsTopic                     string
+	eventWebhookURL                        string
+	eventConfigFilePath                    string
 	logWebhookURL                          string
 	accessLog                              string
 	healthPath                             string
 	debug                                  bool
+	pprof                                  string
 	quiet                                  bool
 	iamDir                                 string
 	ldapURL, ldapBindDN, ldapPassword      string
@@ -79,6 +84,7 @@ func main() {
 		azureCommand(),
 		adminCommand(),
 		testCommand(),
+		utilsCommand(),
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -187,6 +193,12 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
+		&cli.StringFlag{
+			Name:        "pprof",
+			Usage:       "enable pprof debug on specified port",
+			EnvVars:     []string{"VGW_PPROF"},
+			Destination: &pprof,
+		},
 		&cli.BoolFlag{
 			Name:        "quiet",
 			Usage:       "silence stdout request logging output",
@@ -241,6 +253,20 @@ func initFlags() []cli.Flag {
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
+		&cli.StringFlag{
+			Name:        "event-webhook-url",
+			Usage:       "webhook url to send bucket notifications",
+			EnvVars:     []string{"VGW_EVENT_WEBHOOK_URL"},
+			Destination: &eventWebhookURL,
+			Aliases:     []string{"ewu"},
+		},
+		&cli.StringFlag{
+			Name:        "event-filter",
+			Usage:       "bucket event notifications filters configuration file path",
+			EnvVars:     []string{"VGW_EVENT_FILTER"},
+			Destination: &eventConfigFilePath,
+			Aliases:     []string{"ef"},
+		},
 		&cli.StringFlag{
 			Name:        "iam-dir",
 			Usage:       "if defined, run internal iam service within this directory",
@@ -373,6 +399,14 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		return fmt.Errorf("root user access and secret key must be provided")
 	}
 
+	if pprof != "" {
+		// listen on specified port for pprof debug
+		// point browser to http://<ip:port>/debug/pprof/
+		go func() {
+			log.Fatal(http.ListenAndServe(pprof, nil))
+		}()
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:           "versitygw",
 		ServerHeader:      "VERSITYGW",
@@ -465,14 +499,16 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
-		KafkaURL:      kafkaURL,
-		KafkaTopic:    kafkaTopic,
-		KafkaTopicKey: kafkaKey,
-		NatsURL:       natsURL,
-		NatsTopic:     natsTopic,
+		KafkaURL:             kafkaURL,
+		KafkaTopic:           kafkaTopic,
+		KafkaTopicKey:        kafkaKey,
+		NatsURL:              natsURL,
+		NatsTopic:            natsTopic,
+		WebhookURL:           eventWebhookURL,
+		FilterConfigFilePath: eventConfigFilePath,
 	})
 	if err != nil {
-		return fmt.Errorf("unable to connect to the message broker: %w", err)
+		return fmt.Errorf("init bucket event notifications: %w", err)
 	}
 
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
@@ -531,5 +567,15 @@ Loop:
 		}
 	}
 
+	if evSender != nil {
+		err := evSender.Close()
+		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
+			fmt.Fprintf(os.Stderr, "close event sender: %v\n", err)
+		}
+	}
+
 	return saveErr
 }

cmd/versitygw/posix.go

@@ -18,9 +18,14 @@ import (
 	"fmt"
 
 	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
+var (
+	chownuid, chowngid bool
+)
+
 func posixCommand() *cli.Command {
 	return &cli.Command{
 		Name:  "posix",
@@ -36,6 +41,20 @@ bucket: mybucket
 object: a/b/c/myobject
 will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 		Action: runPosix,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
+		},
 	}
 }
 
@@ -44,7 +63,16 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	be, err := posix.New(ctx.Args().Get(0))
+	gwroot := (ctx.Args().Get(0))
+	ok := meta.XattrMeta{}.Test(gwroot)
+	if !ok {
+		return fmt.Errorf("posix backend requires extended attributes support")
+	}
+
+	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
+		ChownUID: chownuid,
+		ChownGID: chowngid,
+	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)
 	}

cmd/versitygw/scoutfs.go

@@ -51,6 +51,18 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
 		},
 	}
 }
@@ -60,12 +72,12 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	var opts []scoutfs.Option
-	if glacier {
-		opts = append(opts, scoutfs.WithGlacierEmulation())
-	}
+	var opts scoutfs.ScoutfsOpts
+	opts.GlacierMode = glacier
+	opts.ChownUID = chownuid
+	opts.ChownGID = chowngid
 
-	be, err := scoutfs.New(ctx.Args().Get(0), opts...)
+	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}

cmd/versitygw/test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package main
 
 import (
@@ -84,6 +98,11 @@ func initTestCommands() []*cli.Command {
 			Usage:  "Tests iam service",
 			Action: getAction(integration.TestIAM),
 		},
+		{
+			Name:   "access-control",
+			Usage:  "Tests gateway access control with bucket ACLs and Policies",
+			Action: getAction(integration.TestAccessControl),
+		},
 		{
 			Name:  "bench",
 			Usage: "Runs download/upload performance test on the gateway",

cmd/versitygw/utils.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/s3event"
+)
+
+func utilsCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "utils",
+		Usage: "utility helper CLI tool",
+		Subcommands: []*cli.Command{
+			{
+				Name:    "gen-event-filter-config",
+				Aliases: []string{"gefc"},
+				Usage:   "Create a new configuration file for bucket event notifications filter.",
+				Action:  generateEventFiltersConfig,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:    "path",
+						Usage:   "the path where the config file has to be created",
+						Aliases: []string{"p"},
+					},
+				},
+			},
+		},
+	}
+}
+
+func generateEventFiltersConfig(ctx *cli.Context) error {
+	pathFlag := ctx.String("path")
+	path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
+	if err != nil {
+		return err
+	}
+
+	config := s3event.EventFilter{
+		s3event.EventObjectCreated:           true,
+		s3event.EventObjectCreatedPut:        true,
+		s3event.EventObjectCreatedPost:       true,
+		s3event.EventObjectCreatedCopy:       true,
+		s3event.EventCompleteMultipartUpload: true,
+		s3event.EventObjectDeleted:           true,
+		s3event.EventObjectTagging:           true,
+		s3event.EventObjectTaggingPut:        true,
+		s3event.EventObjectTaggingDelete:     true,
+		s3event.EventObjectAclPut:            true,
+		s3event.EventObjectRestore:           true,
+		s3event.EventObjectRestorePost:       true,
+		s3event.EventObjectRestoreCompleted:  true,
+	}
+
+	configBytes, err := json.Marshal(config)
+	if err != nil {
+		return fmt.Errorf("parse event config: %w", err)
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create config file: %w", err)
+	}
+	defer file.Close()
+
+	_, err = file.Write(configBytes)
+	if err != nil {
+		return fmt.Errorf("write config file: %w", err)
+	}
+
+	return nil
+}

extra/example.conf

@@ -121,28 +121,55 @@ ROOT_SECRET_ACCESS_KEY=
 # Event Logs #
 ##############
 
+# The gateway events are similar to AWS S3 events, and are documented in the
+# wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+
+# The VGW_EVENT_FILTER option specifies a config file that contains the
+# event filter rules. The event filter rules are used to determine which
+# events are sent to the configured event services.
+# Use the following to generate a default rules file in /etc/versitygw.d/:
+# versitygw utils gen-event-filter-config -p /etc/versitygw.d
+# The resulting file, /etc/versitygw.d/event_config.json, can be modified and
+# specified in the VGW_EVENT_FILTER option.
+# When VGW_EVENT_FILTER is not specified, all events are sent to the configured
+# event service.
+#VGW_EVENT_FILTER=
+
 # Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
 # VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
-# bucket events will be sent to the kafka service. The gateway events are
-# similar to AWS S3 events, and are documented in the wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
+# configured bucket events will be sent to the kafka service.
 #VGW_EVENT_KAFKA_URL=
 #VGW_EVENT_KAFKA_TOPIC=
 #VGW_EVENT_KAFKA_KEY=
 
 # Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
-# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the
-# the NATS messaging service. The gateway events are  similar to AWS S3 events,
-# and are documented in the wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
+# and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be
+# sent to the the NATS messaging service.
 #VGW_EVENT_NATS_URL=
 #VGW_EVENT_NATS_TOPIC=
 
+# Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is
+# specified, all configured bucket events will be sent to the webhook.
+#VGW_EVENT_WEBHOOK_URL=
+
+#######################
+# Debug / Diagnostics #
+#######################
+
 # The VGW_DEBUG option enables verbose debug log output to stdout. This output
 # includes details for signature verification steps. This is generally only
 # useful for debugging the S3 server, and should not be used in production.
 #VGW_DEBUG=false
 
+# The VGW_PPROF option enables the pprof HTTP server for profiling the S3
+# server. See the following for more information:
+# https://pkg.go.dev/net/http/pprof
+# To enable, set the VGW_PPROF option to the listening address for the pprof
+# server. For example, to listen on localhost port 6060, set the option to
+# "localhost:6060".
+#VGW_PPROF=
+
 ################
 # IAM services #
 ################
@@ -216,20 +243,24 @@ ROOT_SECRET_ACCESS_KEY=
 # below the "bucket directory" are treated as the objects. The object
 # name is split on "/" separator to translate to posix storage.
 # For example:
-# top level: /mnt/fs/gwroot
+# top level (VGW_BACKEND_ARG): /mnt/fs/gwroot
 # bucket: mybucket
 # object: a/b/c/myobject
 # will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
 
-# There are currently no further options other than VGW_BACKEND_ARG for the
-# posix backend.
+# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
+# change the ownership of newly created files and directories to the IAM
+# account UID/GID.
+#VGW_CHOWN_UID=false
+#VGW_CHOWN_GID=false
 
 ###########
 # scoutfs #
 ###########
 
 # The scoutfs backend requires a ScoutFS filesystem type for the backend
-# path. The glacier mode functionality requires ScoutAM to be configured
+# path. The object to posix name mappings follow the same rules as posix for
+# scoutfs. The glacier mode functionality requires ScoutAM to be configured
 # for tiering data from the ScoutFS filesystem to a mass stroage system.
 # The mass storage system is often one or more tape libraries. Due to the
 # high latency of tape, the glacier mode functionality is designed to
@@ -248,6 +279,12 @@ ROOT_SECRET_ACCESS_KEY=
 # RestoreObject: add batch stage request to file
 #VGW_SCOUTFS_GLACIER=false
 
+# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
+# change the ownership of newly created files and directories to the IAM
+# account UID/GID.
+#VGW_CHOWN_UID=false
+#VGW_CHOWN_GID=false
+
 ######
 # s3 #
 ######

go.mod

@@ -3,68 +3,68 @@ module github.com/versity/versitygw
 go 1.21
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
-	github.com/aws/aws-sdk-go-v2 v1.26.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
-	github.com/aws/smithy-go v1.20.1
-	github.com/go-ldap/ldap/v3 v3.4.6
-	github.com/gofiber/fiber/v2 v2.52.3
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/aws/smithy-go v1.20.2
+	github.com/go-ldap/ldap/v3 v3.4.7
+	github.com/gofiber/fiber/v2 v2.52.4
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.34.0
+	github.com/nats-io/nats.go v1.34.1
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/urfave/cli/v2 v2.27.1
 	github.com/valyala/fasthttp v1.52.0
-	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.18.0
+	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
+	golang.org/x/sys v0.19.0
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.9
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.9
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 )

go.sum

@@ -1,61 +1,61 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
-github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
+github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
-github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
-github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg=
-github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 h1:F+PUZee9mlfpEJVZdgyewRumKekS9O3fftj8fEMt0rQ=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13/go.mod h1:Rl7i2dEWGHGsBIJCpUxlRt7VwK/HyXxICxdvIRssQHE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48=
+github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
+github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
+github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
+github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
-github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
-github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -63,24 +63,40 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
-github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
-github.com/gofiber/fiber/v2 v2.52.3 h1:bgAZwPv0aHIfRwIUdkWhg6U8D3MEYnoJjT+HfW/dDTo=
-github.com/gofiber/fiber/v2 v2.52.3/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
-github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
-github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/go-ldap/ldap/v3 v3.4.7 h1:3Hbd7mIB1qjd3Ra59fI3JYea/t5kykFu2CVHBca9koE=
+github.com/go-ldap/ldap/v3 v3.4.7/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
+github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
+github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
+github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
+github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
+github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
+github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
+github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
+github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
+github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
+github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
+github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
+github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
+github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
-github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -90,15 +106,15 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk=
-github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
+github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
-github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
@@ -106,18 +122,21 @@ github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6kt
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
 github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -126,33 +145,39 @@ github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7g
 github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
-github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
-github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
+github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
+github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -167,16 +192,18 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -192,6 +219,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

s3api/controllers/admin.go

@@ -49,7 +49,7 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		return fmt.Errorf("failed to create a user: %w", err)
+		return fmt.Errorf("failed to create user: %w", err)
 	}
 
 	return ctx.SendString("The user has been created successfully")

s3api/controllers/base_test.go

@@ -77,7 +77,8 @@ func TestNew(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := New(tt.args.be, tt.args.iam, nil, nil); !reflect.DeepEqual(got, tt.want) {
+			got := New(tt.args.be, tt.args.iam, nil, nil, false)
+			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("New() = %v, want %v", got, tt.want)
 			}
 		})

s3api/middlewares/acl-parser.go

@@ -54,7 +54,6 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 			}
 			return ctx.Next()
 		}
-		//TODO: provide correct action names for the logger, after implementing DetectAction middleware
 		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
 		if err != nil {
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})

s3api/router.go

@@ -27,8 +27,8 @@ type S3ApiRouter struct {
 	WithAdmSrv bool
 }
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
-	s3ApiController := controllers.New(be, iam, logger, evs)
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
+	s3ApiController := controllers.New(be, iam, logger, evs, debug)
 
 	if sa.WithAdmSrv {
 		adminController := controllers.NewAdminController(iam, be)

s3api/router_test.go

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
 		})
 	}
 }

s3api/server.go

@@ -70,7 +70,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l))
 
-	server.router.Init(app, be, iam, l, evs)
+	server.router.Init(app, be, iam, l, evs, server.debug)
 
 	return server, nil
 }

s3api/utils/chunk-reader.go

@@ -193,11 +193,11 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		cr.chunkHash.Write(p[:chunkSize])
 		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
 		return n + int(chunkSize), err
-	} else {
-		cr.chunkDataLeft = chunkSize - int64(n)
-		cr.chunkHash.Write(p[:n])
 	}
 
+	cr.chunkDataLeft = chunkSize - int64(n)
+	cr.chunkHash.Write(p[:n])
+
 	return n, nil
 }
 
@@ -231,6 +231,7 @@ const (
 // error if any. See the AWS documentation for the chunk header format. The
 // header[0] byte is expected to be the first byte of the chunk size here.
 func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
+	stashLen := len(cr.stash)
 	if cr.stash != nil {
 		tmp := make([]byte, maxHeaderSize)
 		copy(tmp, cr.stash)
@@ -265,5 +266,5 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
 	signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
 	dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
 
-	return chunkSize, signature, dataStartOffset, nil
+	return chunkSize, signature, dataStartOffset - stashLen, nil
 }

s3err/s3err.go

@@ -116,6 +116,7 @@ const (
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
+	ErrQuotaExceeded
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -414,6 +415,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Directory object contains data payload.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrQuotaExceeded: {
+		Code:           "QuotaExceeded",
+		Description:    "Your request was denied due to quota exceeded.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3event/event.go

@@ -15,13 +15,18 @@
 package s3event
 
 import (
+	"encoding/json"
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 )
 
 type S3EventSender interface {
 	SendEvent(ctx *fiber.Ctx, meta EventMeta)
+	Close() error
 }
 
 type EventMeta struct {
@@ -36,22 +41,6 @@ type EventFields struct {
 	Records []EventSchema
 }
 
-type EventType string
-
-const (
-	EventObjectPut               EventType = "s3:ObjectCreated:Put"
-	EventObjectCopy              EventType = "s3:ObjectCreated:Copy"
-	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
-	EventObjectDelete            EventType = "s3:ObjectRemoved:Delete"
-	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
-	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
-	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
-	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
-	// Not supported
-	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
-	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
-)
-
 type EventSchema struct {
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
@@ -78,9 +67,18 @@ type EventResponseElements struct {
 	HostId    string `json:"x-amz-id-2"`
 }
 
+type ConfigurationId string
+
+// This field will be changed after implementing per bucket notifications
+const (
+	ConfigurationIdKafka   ConfigurationId = "kafka-global"
+	ConfigurationIdNats    ConfigurationId = "nats-global"
+	ConfigurationIdWebhook ConfigurationId = "webhook-global"
+)
+
 type EventS3Data struct {
 	S3SchemaVersion string            `json:"s3SchemaVersion"`
-	ConfigurationId string            `json:"configurationId"`
+	ConfigurationId ConfigurationId   `json:"configurationId"`
 	Bucket          EventS3BucketData `json:"bucket"`
 	Object          EventObjectData   `json:"object"`
 }
@@ -109,22 +107,95 @@ type EventObjectData struct {
 }
 
 type EventConfig struct {
-	KafkaURL      string
-	KafkaTopic    string
-	KafkaTopicKey string
-	NatsURL       string
-	NatsTopic     string
+	KafkaURL             string
+	KafkaTopic           string
+	KafkaTopicKey        string
+	NatsURL              string
+	NatsTopic            string
+	WebhookURL           string
+	FilterConfigFilePath string
 }
 
 func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
-	if cfg.KafkaURL != "" && cfg.NatsURL != "" {
-		return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
+	filter, err := parseEventFilters(cfg.FilterConfigFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("parse event filter config file %w", err)
 	}
-	if cfg.NatsURL != "" {
-		return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
+	var evSender S3EventSender
+	switch {
+	case cfg.WebhookURL != "":
+		evSender, err = InitWebhookEventSender(cfg.WebhookURL, filter)
+		fmt.Printf("initializing S3 Event Notifications with webhook URL %v\n", cfg.WebhookURL)
+	case cfg.KafkaURL != "":
+		evSender, err = InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey, filter)
+		fmt.Printf("initializing S3 Event Notifications with kafka. URL: %v, topic: %v\n", cfg.WebhookURL, cfg.KafkaTopic)
+	case cfg.NatsURL != "":
+		evSender, err = InitNatsEventService(cfg.NatsURL, cfg.NatsTopic, filter)
+		fmt.Printf("initializing S3 Event Notifications with Nats. URL: %v, topic: %v\n", cfg.NatsURL, cfg.NatsTopic)
+	default:
+		return nil, nil
 	}
-	if cfg.KafkaURL != "" {
-		return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
-	}
-	return nil, nil
+
+	return evSender, err
+}
+
+func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) ([]byte, error) {
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+	acc := ctx.Locals("account").(auth.Account)
+
+	event := []EventSchema{
+		{
+			EventVersion: "2.2",
+			EventSource:  "aws:s3",
+			AwsRegion:    ctx.Locals("region").(string),
+			EventTime:    time.Now().Format(time.RFC3339),
+			EventName:    meta.EventName,
+			UserIdentity: EventUserIdentity{
+				PrincipalId: acc.Access,
+			},
+			RequestParameters: EventRequestParams{
+				SourceIPAddress: ctx.IP(),
+			},
+			ResponseElements: EventResponseElements{
+				RequestId: ctx.Get("X-Amz-Request-Id"),
+				HostId:    ctx.Get("X-Amz-Id-2"),
+			},
+			S3: EventS3Data{
+				S3SchemaVersion: "1.0",
+				ConfigurationId: configId,
+				Bucket: EventS3BucketData{
+					Name: bucket,
+					OwnerIdentity: EventUserIdentity{
+						PrincipalId: meta.BucketOwner,
+					},
+					Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+				},
+				Object: EventObjectData{
+					Key:       object,
+					Size:      meta.ObjectSize,
+					ETag:      meta.ObjectETag,
+					VersionId: meta.VersionId,
+					Sequencer: genSequencer(),
+				},
+			},
+			GlacierEventData: EventGlacierData{
+				// Not supported
+				RestoreEventData: EventRestoreData{},
+			},
+		},
+	}
+
+	return json.Marshal(event)
+}
+
+func generateTestEvent() ([]byte, error) {
+	msg := map[string]string{
+		"Service": "S3",
+		"Event":   "s3:TestEvent",
+		"Time":    time.Now().Format(time.RFC3339),
+		"Bucket":  "Test-Bucket",
+	}
+
+	return json.Marshal(msg)
 }

s3event/filter.go

@@ -0,0 +1,122 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type EventType string
+
+const (
+	EventObjectCreated           EventType = "s3:ObjectCreated:*" // ObjectCreated
+	EventObjectCreatedPut        EventType = "s3:ObjectCreated:Put"
+	EventObjectCreatedPost       EventType = "s3:ObjectCreated:Post"
+	EventObjectCreatedCopy       EventType = "s3:ObjectCreated:Copy"
+	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
+	EventObjectDeleted           EventType = "s3:ObjectRemoved:Delete" // ObjectRemoved
+	EventObjectTagging           EventType = "s3:ObjectTagging:*"      // ObjectTagging
+	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
+	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
+	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
+	EventObjectRestore           EventType = "s3:ObjectRestore:*" // ObjectRestore
+	EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
+	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
+)
+
+func (event EventType) IsValid() bool {
+	_, ok := supportedEventFilters[event]
+	return ok
+}
+
+var supportedEventFilters = map[EventType]struct{}{
+	EventObjectCreated:           {},
+	EventObjectCreatedPut:        {},
+	EventObjectCreatedPost:       {},
+	EventObjectCreatedCopy:       {},
+	EventCompleteMultipartUpload: {},
+	EventObjectDeleted:           {},
+	EventObjectTagging:           {},
+	EventObjectTaggingPut:        {},
+	EventObjectTaggingDelete:     {},
+	EventObjectAclPut:            {},
+	EventObjectRestore:           {},
+	EventObjectRestorePost:       {},
+	EventObjectRestoreCompleted:  {},
+}
+
+type EventFilter map[EventType]bool
+
+func parseEventFilters(path string) (EventFilter, error) {
+	// if no filter config file path is specified return nil map
+	if path == "" {
+		return nil, nil
+	}
+
+	configFilePath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Open the JSON file
+	file, err := os.Open(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var filter EventFilter
+	if err := json.NewDecoder(file).Decode(&filter); err != nil {
+		return nil, err
+	}
+
+	if err := filter.Validate(); err != nil {
+		return nil, err
+	}
+
+	return filter, nil
+}
+
+func (ef EventFilter) Validate() error {
+	for event := range ef {
+		if isValid := event.IsValid(); !isValid {
+			return fmt.Errorf("invalid configuration property: %v", event)
+		}
+	}
+
+	return nil
+}
+
+func (ef EventFilter) Filter(event EventType) bool {
+	ev, found := ef[event]
+	if found {
+		return ev
+	}
+
+	// check wildcard match
+	wildCardEv := EventType(string(event[strings.LastIndex(string(event), ":")+1]) + "*")
+	wildcard, found := ef[wildCardEv]
+	if found {
+		return wildcard
+	}
+
+	return false
+}

s3event/kafka.go

@@ -16,10 +16,8 @@ package s3event
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -32,10 +30,11 @@ var sequencer = 0
 type Kafka struct {
 	key    string
 	writer *kafka.Writer
+	filter EventFilter
 	mu     sync.Mutex
 }
 
-func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
+func InitKafkaEventService(url, topic, key string, filter EventFilter) (S3EventSender, error) {
 	if topic == "" {
 		return nil, fmt.Errorf("kafka message topic should be specified")
 	}
@@ -47,26 +46,19 @@ func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
 		BatchTimeout: 5 * time.Millisecond,
 	})
 
-	msg := map[string]string{
-		"Service": "S3",
-		"Event":   "s3:TestEvent",
-		"Time":    time.Now().Format(time.RFC3339),
-		"Bucket":  "Test-Bucket",
-	}
-
-	msgJSON, err := json.Marshal(msg)
+	msg, err := generateTestEvent()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("kafka generate test event: %w", err)
 	}
 
 	message := kafka.Message{
 		Key:   []byte(key),
-		Value: msgJSON,
+		Value: msg,
 	}
 
-	ctx := context.Background()
-
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	err = w.WriteMessages(ctx, message)
+	cancel()
 	if err != nil {
 		return nil, err
 	}
@@ -74,6 +66,7 @@ func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
 	return &Kafka{
 		key:    key,
 		writer: w,
+		filter: filter,
 	}, nil
 }
 
@@ -81,67 +74,31 @@ func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 	ks.mu.Lock()
 	defer ks.mu.Unlock()
 
-	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
-
-	schema := EventSchema{
-		EventVersion: "2.2",
-		EventSource:  "aws:s3",
-		AwsRegion:    ctx.Locals("region").(string),
-		EventTime:    time.Now().Format(time.RFC3339),
-		EventName:    meta.EventName,
-		UserIdentity: EventUserIdentity{
-			PrincipalId: ctx.Locals("access").(string),
-		},
-		RequestParameters: EventRequestParams{
-			SourceIPAddress: ctx.IP(),
-		},
-		ResponseElements: EventResponseElements{
-			RequestId: ctx.Get("X-Amz-Request-Id"),
-			HostId:    ctx.Get("X-Amx-Id-2"),
-		},
-		S3: EventS3Data{
-			S3SchemaVersion: "1.0",
-			// This field will come up after implementing per bucket notifications
-			ConfigurationId: "kafka-global",
-			Bucket: EventS3BucketData{
-				Name: bucket,
-				OwnerIdentity: EventUserIdentity{
-					PrincipalId: ctx.Locals("access").(string),
-				},
-				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
-			},
-			Object: EventObjectData{
-				Key:       object,
-				Size:      meta.ObjectSize,
-				ETag:      meta.ObjectETag,
-				VersionId: meta.VersionId,
-				Sequencer: genSequencer(),
-			},
-		},
-		GlacierEventData: EventGlacierData{
-			// Not supported
-			RestoreEventData: EventRestoreData{},
-		},
-	}
-
-	ks.send([]EventSchema{schema})
-}
-
-func (ks *Kafka) send(evnt []EventSchema) {
-	msg, err := json.Marshal(evnt)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
+	if ks.filter != nil && !ks.filter.Filter(meta.EventName) {
 		return
 	}
 
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdKafka)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create kafka event: %v\n", err.Error())
+		return
+	}
+
+	go ks.send(schema)
+}
+
+func (ks *Kafka) Close() error {
+	return ks.writer.Close()
+}
+
+func (ks *Kafka) send(event []byte) {
 	message := kafka.Message{
 		Key:   []byte(ks.key),
-		Value: msg,
+		Value: event,
 	}
 
 	ctx := context.Background()
-	err = ks.writer.WriteMessages(ctx, message)
+	err := ks.writer.WriteMessages(ctx, message)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
 	}

s3event/nats.go

@@ -15,12 +15,9 @@
 package s3event
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
-	"time"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/nats-io/nats.go"
@@ -30,9 +27,10 @@ type NatsEventSender struct {
 	topic  string
 	client *nats.Conn
 	mu     sync.Mutex
+	filter EventFilter
 }
 
-func InitNatsEventService(url, topic string) (S3EventSender, error) {
+func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {
 	if topic == "" {
 		return nil, fmt.Errorf("nats message topic should be specified")
 	}
@@ -42,9 +40,20 @@ func InitNatsEventService(url, topic string) (S3EventSender, error) {
 		return nil, err
 	}
 
+	msg, err := generateTestEvent()
+	if err != nil {
+		return nil, fmt.Errorf("nats generate test event: %w", err)
+	}
+
+	err = client.Publish(topic, msg)
+	if err != nil {
+		return nil, fmt.Errorf("nats publish test event: %v", err)
+	}
+
 	return &NatsEventSender{
 		topic:  topic,
 		client: client,
+		filter: filter,
 	}, nil
 }
 
@@ -52,60 +61,26 @@ func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 	ns.mu.Lock()
 	defer ns.mu.Unlock()
 
-	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
-
-	schema := EventSchema{
-		EventVersion: "2.2",
-		EventSource:  "aws:s3",
-		AwsRegion:    ctx.Locals("region").(string),
-		EventTime:    time.Now().Format(time.RFC3339),
-		EventName:    meta.EventName,
-		UserIdentity: EventUserIdentity{
-			PrincipalId: ctx.Locals("access").(string),
-		},
-		RequestParameters: EventRequestParams{
-			SourceIPAddress: ctx.IP(),
-		},
-		ResponseElements: EventResponseElements{
-			RequestId: ctx.Get("X-Amz-Request-Id"),
-			HostId:    ctx.Get("X-Amx-Id-2"),
-		},
-		S3: EventS3Data{
-			S3SchemaVersion: "1.0",
-			// This field will come up after implementing per bucket notifications
-			ConfigurationId: "nats-global",
-			Bucket: EventS3BucketData{
-				Name: bucket,
-				OwnerIdentity: EventUserIdentity{
-					PrincipalId: ctx.Locals("access").(string),
-				},
-				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
-			},
-			Object: EventObjectData{
-				Key:       object,
-				Size:      meta.ObjectSize,
-				ETag:      meta.ObjectETag,
-				VersionId: meta.VersionId,
-				Sequencer: genSequencer(),
-			},
-		},
-		GlacierEventData: EventGlacierData{
-			// Not supported
-			RestoreEventData: EventRestoreData{},
-		},
+	if ns.filter != nil && !ns.filter.Filter(meta.EventName) {
+		return
 	}
 
-	ns.send([]EventSchema{schema})
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdNats)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create nats event: %v\n", err.Error())
+		return
+	}
+
+	go ns.send(schema)
 }
 
-func (ns *NatsEventSender) send(evnt []EventSchema) {
-	msg, err := json.Marshal(evnt)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
-	}
+func (ns *NatsEventSender) Close() error {
+	ns.client.Close()
+	return nil
+}
 
-	err = ns.client.Publish(ns.topic, msg)
+func (ns *NatsEventSender) send(event []byte) {
+	err := ns.client.Publish(ns.topic, event)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
 	}

s3event/webhook.go

@@ -0,0 +1,108 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type Webhook struct {
+	url    string
+	client *http.Client
+	filter EventFilter
+	mu     sync.Mutex
+}
+
+func InitWebhookEventSender(url string, filter EventFilter) (S3EventSender, error) {
+	if url == "" {
+		return nil, fmt.Errorf("webhook url should be specified")
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 1,
+	}
+
+	testEv, err := generateTestEvent()
+	if err != nil {
+		return nil, fmt.Errorf("webhook generate test event: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(testEv))
+	if err != nil {
+		return nil, fmt.Errorf("create webhook http request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	_, err = client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			return nil, fmt.Errorf("send webhook test event: %w", err)
+		}
+	}
+
+	return &Webhook{
+		client: &http.Client{
+			Timeout: 3 * time.Second,
+		},
+		url:    url,
+		filter: filter,
+	}, nil
+}
+
+func (w *Webhook) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.filter != nil && !w.filter.Filter(meta.EventName) {
+		return
+	}
+
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create webhook event: %v\n", err.Error())
+		return
+	}
+
+	go w.send(schema)
+}
+
+func (w *Webhook) Close() error {
+	return nil
+}
+
+func (w *Webhook) send(event []byte) {
+	req, err := http.NewRequest(http.MethodPost, w.url, bytes.NewReader(event))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create webhook event request: %v\n", err.Error())
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	_, err = w.client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			fmt.Fprintf(os.Stderr, "failed to send webhook event: %v\n", err.Error())
+		}
+	}
+}

s3log/file.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -88,9 +89,9 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 		}
 	}
 
-	switch ctx.Locals("access").(type) {
-	case string:
-		access = ctx.Locals("access").(string)
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner

s3log/webhook.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -85,9 +86,9 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 		}
 	}
 
-	switch ctx.Locals("access").(type) {
-	case string:
-		access = ctx.Locals("access").(string)
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner

tests/.env.default

@@ -11,4 +11,7 @@ CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
 S3CMD_CONFIG=./tests/s3cfg.local.default
 SECRETS_FILE=./tests/.secrets
-MC_ALIAS=versity
+MC_ALIAS=versity
+LOG_LEVEL=2
+GOCOVERDIR=$PWD/cover
+USERS_FOLDER=$PWD/iam

tests/.env.s3.default

@@ -11,4 +11,4 @@ CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
 S3CMD_CONFIG=./tests/s3cfg.local.default
 SECRETS_FILE=./tests/.secrets.s3
-MC_ALIAS=versity
+MC_ALIAS=versity_s3

tests/README.md

@@ -2,6 +2,8 @@
 
 ## Instructions - Running Locally
 
+### Posix Backend
+
 1. Build the `versitygw` binary.
 2. Install the command-line interface(s) you want to test if unavailable on your machine.  
    * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
@@ -28,7 +30,18 @@
 8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
 9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
+### S3 Backend
+
+Instructions are mostly the same; however, testing with the S3 backend requires two S3 accounts.  Ideally, these are two real accounts, but one can also be a dummy account that versity uses internally.
+
+To set up the latter:
+1. Create a new AWS profile with ID and key values set to dummy 20-char allcaps and 40-char alphabetical values respectively.
+1. In the `.secrets` file being used, create the fields `AWS_ACCESS_KEY_ID_TWO` and `AWS_SECRET_ACCESS_KEY_TWO`.  Set these values to the actual AWS ID and key.  
+2. Set the values for `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` the same dummy values set in the AWS profile, and set `AWS_PROFILE` to the profile you just created.
+3. Create a new AWS profile with these dummy values.  In the `.env` file being used, set the `AWS_PROFILE` parameter to the name of this new profile, and the ID and key fields to the dummy values.  
+4. Set `BACKEND` to `s3`.  Also, change the `MC_ALIAS` value if testing **mc** in this configuration.
+
 ## Instructions - Running With Docker
 
 1.  Create a `.secrets` file in the `tests` folder, and add the `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and the `AWS_PROFILE` fields.
-2.  Build and run the `Dockerfile_test_bats` file.
+2.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to an S3-backend-friendly config.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.

tests/integration/group-tests.go

@@ -309,6 +309,7 @@ func TestFullFlow(s *S3Conf) {
 	TestPutBucketPolicy(s)
 	TestGetBucketPolicy(s)
 	TestDeleteBucketPolicy(s)
+	TestAccessControl(s)
 }
 
 func TestPosix(s *S3Conf) {
@@ -325,6 +326,16 @@ func TestIAM(s *S3Conf) {
 	IAM_admin_ChangeBucketOwner(s)
 }
 
+func TestAccessControl(s *S3Conf) {
+	AccessControl_default_ACL_user_access_denied(s)
+	AccessControl_default_ACL_userplus_access_denied(s)
+	AccessControl_default_ACL_admin_successful_access(s)
+	AccessControl_bucket_resource_single_action(s)
+	AccessControl_bucket_resource_all_action(s)
+	AccessControl_single_object_resource_actions(s)
+	AccessControl_multi_statement_policy(s)
+}
+
 type IntTests map[string]func(s *S3Conf) error
 
 func GetIntTests() IntTests {

tests/integration/tests.go

@@ -5797,6 +5797,320 @@ func DeleteBucketPolicy_success(s *S3Conf) error {
 	})
 }
 
+// Access control tests (with bucket ACLs and Policies)
+func AccessControl_default_ACL_user_access_denied(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_user_access_denied"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = usr.access
+		cfg.awsSecret = usr.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_default_ACL_userplus_access_denied(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_userplus_access_denied"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr := user{
+			access: "userplus1",
+			secret: "userplus1secret",
+			role:   "userplus",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = usr.access
+		cfg.awsSecret = usr.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_default_ACL_admin_successful_access(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_admin_successful_access"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		admin := user{
+			access: "admin1",
+			secret: "admin1secret",
+			role:   "admin",
+		}
+		err := createUsers(s, []user{admin})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = admin.access
+		cfg.awsSecret = admin.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_bucket_resource_single_action(s *S3Conf) error {
+	testName := "AccessControl_bucket_resource_single_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		usr2 := user{
+			access: "grt2",
+			secret: "grt2secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr1, usr2})
+		if err != nil {
+			return err
+		}
+
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:PutBucketTagging"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.DeleteBucketTagging(ctx, &s3.DeleteBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		user2Client := getUserS3Client(usr2, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user2Client.DeleteBucketTagging(ctx, &s3.DeleteBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_bucket_resource_all_action(s *S3Conf) error {
+	testName := "AccessControl_bucket_resource_all_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		usr2 := user{
+			access: "grt2",
+			secret: "grt2secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr1, usr2})
+		if err != nil {
+			return err
+		}
+
+		bucketResource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		objectResource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+		err = putObjects(user1Client, []string{"my-obj"}, bucket)
+		if err != nil {
+			return err
+		}
+
+		user2Client := getUserS3Client(usr2, s)
+
+		err = putObjects(user2Client, []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_single_object_resource_actions(s *S3Conf) error {
+	testName := "AccessControl_single_object_resource_actions"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		obj := "my-obj/nested-obj"
+		err := putObjects(s3client, []string{obj}, bucket)
+		if err != nil {
+			return err
+		}
+
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err = createUsers(s, []user{usr1})
+		if err != nil {
+			return err
+		}
+
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%v/%v"`, bucket, obj))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: &bucket,
+			Key:    &obj,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_multi_statement_policy(s *S3Conf) error {
+	testName := "AccessControl_multi_statement_policy"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		policy := fmt.Sprintf(`
+		{
+			"Statement": [
+				{
+					"Effect": "Deny",
+					"Principal": ["grt1"],
+					"Action":  "s3:DeleteBucket",
+					"Resource":  "arn:aws:s3:::%s"
+				},
+				{
+					"Effect": "Allow",
+					"Principal": "grt1",
+					"Action": "s3:*",
+					"Resource": ["arn:aws:s3:::%s", "arn:aws:s3:::%s/*"]
+				}
+			]
+		}	
+		`, bucket, bucket, bucket)
+
+		usr := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &policy,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		userClient := getUserS3Client(usr, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = userClient.ListObjects(ctx, &s3.ListObjectsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = userClient.DeleteBucket(ctx, &s3.DeleteBucketInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 // IAM related tests
 // multi-user iam tests
 func IAM_user_access_denied(s *S3Conf) error {
@@ -5808,22 +6122,17 @@ func IAM_user_access_denied(s *S3Conf) error {
 		secret: "grt1secret",
 		role:   "user",
 	}
-	err := deleteUser(s, usr.access)
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
 
-	err = createUsers(s, []user{usr})
+	err := createUsers(s, []user{usr})
 	if err != nil {
 		failF("%v: %v", testName, err)
 		return fmt.Errorf("%v: %w", testName, err)
 	}
 
 	out, err := execCommand("admin", "-a", usr.access, "-s", usr.secret, "-er", s.endpoint, "delete-user", "-a", "random_access")
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
+	if err == nil {
+		failF("%v: expected cmd error", testName)
+		return fmt.Errorf("%v: expected cmd error", testName)
 	}
 	if !strings.Contains(string(out), adminAccessDeniedMsg) {
 		failF("%v: expected response error message to be %v, instead got %s", testName, adminAccessDeniedMsg, out)
@@ -5844,22 +6153,17 @@ func IAM_userplus_access_denied(s *S3Conf) error {
 		secret: "grt1secret",
 		role:   "userplus",
 	}
-	err := deleteUser(s, usr.access)
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
 
-	err = createUsers(s, []user{usr})
+	err := createUsers(s, []user{usr})
 	if err != nil {
 		failF("%v: %v", testName, err)
 		return fmt.Errorf("%v: %w", testName, err)
 	}
 
 	out, err := execCommand("admin", "-a", usr.access, "-s", usr.secret, "-er", s.endpoint, "delete-user", "-a", "random_access")
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
+	if err == nil {
+		failF("%v: expected cmd error", testName)
+		return fmt.Errorf("%v: expected cmd error", testName)
 	}
 	if !strings.Contains(string(out), adminAccessDeniedMsg) {
 		failF("%v: expected response error message to be %v, instead got %s", testName, adminAccessDeniedMsg, out)
@@ -5879,12 +6183,8 @@ func IAM_userplus_CreateBucket(s *S3Conf) error {
 			secret: "grt1secret",
 			role:   "userplus",
 		}
-		err := deleteUser(s, usr.access)
-		if err != nil {
-			return err
-		}
 
-		err = createUsers(s, []user{usr})
+		err := createUsers(s, []user{usr})
 		if err != nil {
 			return err
 		}

tests/integration/utils.go

@@ -30,7 +30,7 @@ import (
 var (
 	bcktCount            = 0
 	succUsrCrt           = "The user has been created successfully"
-	failUsrCrt           = "failed to create a user: update iam data: account already exists"
+	failUsrCrt           = "failed to create user: update iam data: account already exists"
 	adminAccessDeniedMsg = "access denied: only admin users have access to this resource"
 	succDeleteUserMsg    = "The user has been deleted successfully"
 )
@@ -537,12 +537,16 @@ type user struct {
 
 func createUsers(s *S3Conf, users []user) error {
 	for _, usr := range users {
+		err := deleteUser(s, usr.access)
+		if err != nil {
+			return err
+		}
 		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
 		if err != nil {
 			return err
 		}
 		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
-			return fmt.Errorf("failed to create a user account")
+			return fmt.Errorf("failed to create user account")
 		}
 	}
 	return nil
@@ -633,3 +637,11 @@ func getMalformedPolicyError(msg string) s3err.APIError {
 		HTTPStatusCode: http.StatusBadRequest,
 	}
 }
+
+func getUserS3Client(usr user, cfg *S3Conf) *s3.Client {
+	config := *cfg
+	config.awsID = usr.access
+	config.awsSecret = usr.secret
+
+	return s3.NewFromConfig(config.Config())
+}

tests/logger.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# levels:  1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace
+
+log() {
+  if [[ $# -ne 2 ]]; then
+    echo "log function requires level, message"
+    return 1
+  fi
+  if [[ $1 -gt $LOG_LEVEL ]]; then
+    return 0
+  fi
+  echo "$2"
+}

tests/run.sh

@@ -20,7 +20,7 @@ handle_param() {
       -s|--static)
           export RECREATE_BUCKETS=false
           ;;
-      aws|aws-posix|s3cmd|mc)
+      aws|aws-posix|s3cmd|mc|user)
           set_command_type "$1"
           ;;
       *) # Handle unrecognized options or positional arguments
@@ -39,7 +39,14 @@ set_command_type() {
   export command_type
 }
 
-export RECREATE_BUCKETS=true
+if [[ -z $RECREATE_BUCKETS ]]; then
+  export RECREATE_BUCKETS=true
+elif [[ $RECREATE_BUCKETS != true ]] && [[ $RECREATE_BUCKETS != false ]]; then
+  echo "Invalid RECREATE_BUCKETS value: $RECREATE_BUCKETS"
+  exit 1
+else
+  export RECREATE_BUCKETS=$RECREATE_BUCKETS
+fi
 while [[ "$#" -gt 0 ]]; do
   handle_param "$1"
   shift # past argument or value
@@ -60,16 +67,25 @@ fi
 case $command_type in
   aws)
     "$HOME"/bin/bats ./tests/test_aws.sh || exit_code=$?
+    if [[ $exit_code -eq 0 ]]; then
+      "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
+    fi
     ;;
   aws-posix)
     "$HOME"/bin/bats ./tests/test_aws_posix.sh || exit_code=$?
     ;;
   s3cmd)
     "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
+    if [[ $exit_code -eq 0 ]]; then
+      "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
+    fi
     ;;
   mc)
     "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
     ;;
+  user)
+    "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
+    ;;
 esac
 
 exit $exit_code

tests/run_all.sh

@@ -4,6 +4,11 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
   echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
   exit 1
 fi
+
+# shellcheck source=./.env.default
+source "$VERSITYGW_TEST_ENV"
+export RECREATE_BUCKETS
+
 if ! ./tests/run.sh aws; then
   exit 1
 fi
@@ -16,16 +21,7 @@ fi
 if ! ./tests/run.sh mc; then
   exit 1
 fi
-if ! ./tests/run.sh -s aws; then
-  exit 1
-fi
-if ! ./tests/run.sh -s aws-posix; then
-  exit 1
-fi
-if ! ./tests/run.sh -s s3cmd; then
-  exit 1
-fi
-if ! ./tests/run.sh -s mc; then
+if ! ./tests/run.sh user; then
   exit 1
 fi
 exit 0

tests/setup.sh

@@ -54,6 +54,11 @@ check_params() {
     echo "RECREATE_BUCKETS must be 'true' or 'false'"
     return 1
   fi
+  if [[ -z "$LOG_LEVEL" ]]; then
+    export LOG_LEVEL=2
+  else
+    export LOG_LEVEL
+  fi
   return 0
 }
 

tests/test_aws.sh

@@ -2,6 +2,8 @@
 
 source ./tests/setup.sh
 source ./tests/util.sh
+source ./tests/util_aws.sh
+source ./tests/util_bucket_create.sh
 source ./tests/util_file.sh
 source ./tests/test_common.sh
 
@@ -10,6 +12,19 @@ source ./tests/test_common.sh
   test_common_create_delete_bucket "aws"
 }
 
+@test "test_create_bucket_invalid_name" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "aws" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"Invalid bucket name "* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
 # test adding and removing an object on versitygw
 @test "test_put_object-with-data" {
   test_common_put_object_with_data "aws"
@@ -68,7 +83,6 @@ source ./tests/test_common.sh
 
 # test ability to delete multiple objects from bucket
 @test "test_delete_objects" {
-
   local object_one="test-file-one"
   local object_two="test-file-two"
 
@@ -106,7 +120,6 @@ source ./tests/test_common.sh
 
 # test v1 s3api list objects command
 @test "test-s3api-list-objects-v1" {
-
   local object_one="test-file-one"
   local object_two="test-file-two"
   local object_two_data="test data\n"
@@ -137,7 +150,6 @@ source ./tests/test_common.sh
 
 # test v2 s3api list objects command
 @test "test-s3api-list-objects-v2" {
-
   local object_one="test-file-one"
   local object_two="test-file-two"
   local object_two_data="test data\n"
@@ -173,7 +185,6 @@ source ./tests/test_common.sh
 
 # test multi-part upload
 @test "test-multi-part-upload" {
-
   local bucket_file="bucket-file"
   bucket_file_data="test file\n"
 
@@ -196,7 +207,6 @@ source ./tests/test_common.sh
 
 # test multi-part upload abort
 @test "test-multi-part-upload-abort" {
-
   local bucket_file="bucket-file"
   bucket_file_data="test file\n"
 
@@ -218,7 +228,6 @@ source ./tests/test_common.sh
 
 # test multi-part upload list parts command
 @test "test-multipart-upload-list-parts" {
-
   local bucket_file="bucket-file"
   local bucket_file_data="test file\n"
 
@@ -266,10 +275,14 @@ source ./tests/test_common.sh
 
 # test listing of active uploads
 @test "test-multipart-upload-list-uploads" {
-
   local bucket_file_one="bucket-file-one"
   local bucket_file_two="bucket-file-two"
 
+  if [[ $RECREATE_BUCKETS == false ]]; then
+    abort_all_multipart_uploads "$BUCKET_ONE_NAME" || local abort_result=$?
+    [[ $abort_result -eq 0 ]] || fail "error aborting all uploads"
+  fi
+
   create_test_files "$bucket_file_one" "$bucket_file_two" || local created=$?
   [[ $created -eq 0 ]] || fail "Error creating test files"
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
@@ -280,6 +293,7 @@ source ./tests/test_common.sh
 
   local key_one
   local key_two
+  log 5 "$uploads"
   key_one=$(echo "$uploads" | jq '.Uploads[0].Key')
   key_two=$(echo "$uploads" | jq '.Uploads[1].Key')
   key_one=${key_one//\"/}
@@ -300,8 +314,8 @@ source ./tests/test_common.sh
   bucket_file_data="test file\n"
 
   create_test_files "$bucket_file" || local created=$?
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
   [[ $created -eq 0 ]] || fail "Error creating test files"
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
@@ -319,3 +333,88 @@ source ./tests/test_common.sh
 @test "test-presigned-url-utf8-chars" {
   test_common_presigned_url_utf8_chars "aws"
 }
+
+@test "test-list-objects-delimiter" {
+  folder_name="two"
+  object_name="three"
+  create_test_folder "$folder_name" || local created=$?
+  [[ $created -eq 0 ]] || fail "error creating folder"
+  create_test_files "$folder_name"/"$object_name" || created=$?
+  [[ $created -eq 0 ]] || fail "error creating file"
+
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  put_object "aws" "$test_file_folder"/"$folder_name"/"$object_name" "$BUCKET_ONE_NAME"/"$folder_name"/"$object_name" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"
+  prefix=$(echo "${objects[@]}" | jq ".CommonPrefixes[0].Prefix")
+  [[ $prefix == "\""$folder_name/"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"
+  key=$(echo "${objects[@]}" | jq ".Contents[0].Key")
+  [[ $key == "\""$folder_name/$object_name"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $folder_name
+}
+
+# ensure that lists of files greater than a size of 1000 (pagination) are returned properly
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "aws"
+}
+
+#@test "test_filename_length" {
+#  file_name=$(printf "%0.sa" $(seq 1 1025))
+#  echo "$file_name"
+
+#  create_test_files "$file_name" || created=$?
+#  [[ $created -eq 0 ]] || fail "error creating file"
+
+#  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+#  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+#  put_object "aws" "$test_file_folder"/"$file_name" "$BUCKET_ONE_NAME"/"$file_name" || local put_object=$?
+#  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+#}
+
+@test "test_head_bucket" {
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "aws" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_head_bucket_doesnt_exist" {
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_copy_object_aws" {
+
+  bucket_file="bucket_file"
+
+  create_test_files "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  setup_bucket "aws" "$BUCKET_TWO_NAME" || local setup_result_two=$?
+  [[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error"
+  put_object "aws" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  error=$(aws --no-verify-ssl s3api copy-object --copy-source "$BUCKET_ONE_NAME"/"$bucket_file" --key "$bucket_file" --bucket "$BUCKET_TWO_NAME" 2>&1) || local copy_result=$?
+  [[ $copy_result -eq 0 ]] || fail "Error copying file: $error"
+  copy_file "s3://$BUCKET_TWO_NAME"/"$bucket_file" "$test_file_folder/${bucket_file}_copy" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  compare_files "$test_file_folder/$bucket_file" "$test_file_folder/${bucket_file}_copy" || local compare_result=$?
+  [[ $compare_result -eq 0 ]] || file "files don't match"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "aws" "$BUCKET_TWO_NAME"
+  delete_test_files "$bucket_file"
+}

tests/test_aws_posix.sh

@@ -2,6 +2,7 @@
 
 source ./tests/setup.sh
 source ./tests/util.sh
+source ./tests/util_bucket_create.sh
 source ./tests/util_file.sh
 source ./tests/util_posix.sh
 

tests/test_common.sh

@@ -34,6 +34,7 @@ test_common_put_object_with_data() {
   create_test_files "$object_name" || local create_result=$?
   [[ $create_result -eq 0 ]] || fail "Error creating test file"
   echo "test data" > "$test_file_folder"/"$object_name"
+  test_common_put_object "$1" "$object_name"
 }
 
 test_common_put_object_no_data() {
@@ -148,7 +149,6 @@ test_common_list_objects() {
 }
 
 test_common_set_get_bucket_tags() {
-
   if [[ $# -ne 1 ]]; then
     fail "set/get bucket tags test requires command type"
   fi
@@ -164,7 +164,7 @@ test_common_set_get_bucket_tags() {
 
   if [[ $1 == 'aws' ]]; then
     if [[ $tags != "" ]]; then
-      tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet')
+      tag_set=$(echo "$tags" | jq '.TagSet')
       [[ $tag_set == "[]" ]] || fail "Error:  tags not empty: $tags"
     fi
   else
@@ -178,8 +178,9 @@ test_common_set_get_bucket_tags() {
   local tag_set_key
   local tag_set_value
   if [[ $1 == 'aws' ]]; then
-    tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key')
-    tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value')
+    log 5 "Post-export tags: $tags"
+    tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
+    tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
     [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
     [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
   else
@@ -192,7 +193,6 @@ test_common_set_get_bucket_tags() {
 }
 
 test_common_set_get_object_tags() {
-
   if [[ $# -ne 1 ]]; then
     echo "get/set object tags missing command type"
     return 1
@@ -213,7 +213,7 @@ test_common_set_get_object_tags() {
   get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
   [[ $get_result -eq 0 ]] || fail "Error getting object tags"
   if [[ $1 == 'aws' ]]; then
-    tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet')
+    tag_set=$(echo "$tags" | jq '.TagSet')
     [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
   elif [[ ! $tags == *"No tags found"* ]]; then
     fail "no tags found (tags: $tags)"
@@ -223,8 +223,8 @@ test_common_set_get_object_tags() {
   get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
   [[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
   if [[ $1 == 'aws' ]]; then
-    tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key')
-    tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value')
+    tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
+    tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
     [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
     [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
   else
@@ -293,3 +293,23 @@ test_common_presigned_url_utf8_chars() {
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
   delete_test_files "$bucket_file" "$bucket_file_copy"
 }
+
+test_common_list_objects_file_count() {
+  if [[ $# -ne 1 ]]; then
+    echo "list objects greater than 1000 missing command type"
+    return 1
+  fi
+  create_test_file_count 1001 || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "error creating test files"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  put_object_multiple "$1" "$test_file_folder/file_*" "$BUCKET_ONE_NAME" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "Failed to copy files to bucket"
+  list_objects "$1" "$BUCKET_ONE_NAME"
+  if [[ $LOG_LEVEL -ge 5 ]]; then
+    log 5 "Array: ${object_array[*]}"
+  fi
+  local file_count="${#object_array[@]}"
+  [[ $file_count == 1001 ]] || fail "file count should be 1001, is $file_count"
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+}

tests/test_mc.sh

@@ -2,6 +2,7 @@
 
 source ./tests/test_common.sh
 source ./tests/setup.sh
+source ./tests/util_bucket_create.sh
 
 export RUN_MC=true
 
@@ -41,3 +42,37 @@ export RUN_MC=true
 @test "test_presigned_url_utf8_chars_mc" {
   test_common_presigned_url_utf8_chars "mc"
 }
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "mc"
+}
+
+@test "test_create_bucket_invalid_name_mc" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "mc" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"Bucket name cannot be empty"* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_mc" {
+  setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "mc" "$BUCKET_ONE_NAME"
+  [[ $bucket_info == *"$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info"
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_doesnt_exist_mc" {
+  setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "mc" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"does not exist"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}

tests/test_s3cmd.sh

@@ -3,6 +3,7 @@
 source ./tests/setup.sh
 source ./tests/test_common.sh
 source ./tests/util.sh
+source ./tests/util_bucket_create.sh
 
 export RUN_S3CMD=true
 
@@ -35,4 +36,38 @@ export RUN_S3CMD=true
 
 #@test "test_presigned_url_utf8_chars_s3cmd" {
 #  test_common_presigned_url_utf8_chars "s3cmd"
-#}
+#}
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3cmd"
+}
+
+@test "test_create_bucket_invalid_name_s3cmd" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "s3cmd" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"just the bucket name"* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_s3cmd" {
+  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "s3cmd" "$BUCKET_ONE_NAME"
+  [[ $bucket_info == *"s3://$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info"
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_doesnt_exist_s3cmd" {
+  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "s3cmd" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}

tests/test_user_aws.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bats
+
+source ./tests/test_user_common.sh
+
+@test "test_admin_user_aws" {
+  test_admin_user "aws"
+}
+
+@test "test_create_user_already_exists_aws" {
+  test_create_user_already_exists "aws"
+}
+
+@test "test_user_user_aws" {
+  test_user_user "aws"
+}
+
+@test "test_userplus_operation_aws" {
+  test_userplus_operation "aws"
+}

tests/test_user_common.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bats
+
+source ./tests/setup.sh
+source ./tests/util_users.sh
+source ./tests/util.sh
+source ./tests/util_bucket_create.sh
+
+test_admin_user() {
+  if [[ $# -ne 1 ]]; then
+    fail "test admin user command requires command type"
+  fi
+
+  admin_username="ABCDEF"
+  user_username="GHIJKL"
+  admin_password="123456"
+  user_password="789012"
+
+  user_exists "$admin_username" || local admin_exists_result=$?
+  if [[ $admin_exists_result -eq 0 ]]; then
+    delete_user "$admin_username" || local delete_admin_result=$?
+    [[ $delete_admin_result -eq 0 ]] || fail "failed to delete admin user"
+  fi
+  create_user "$admin_username" "$admin_password" "admin" || create_admin_result=$?
+  [[ $create_admin_result -eq 0 ]] || fail "failed to create admin user"
+
+  user_exists "$user_username" || local user_exists_result=$?
+  if [[ $user_exists_result -eq 0 ]]; then
+    delete_user "$user_username" || local delete_user_result=$?
+    [[ $delete_user_result -eq 0 ]] || fail "failed to delete user user"
+  fi
+  create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user"
+
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  delete_bucket "aws" "versity-gwtest-admin-bucket" || local delete_result=$?
+  [[ $delete_result -eq 0 ]] || fail "error deleting bucket if it exists"
+  create_bucket_with_user "aws" "versity-gwtest-admin-bucket" "$admin_username" "$admin_password" || create_result_two=$?
+  [[ $create_result_two -eq 0 ]] || fail "error creating bucket with user"
+
+  bucket_one_found=false
+  bucket_two_found=false
+  list_buckets_with_user "aws" "$admin_username" "$admin_password"
+  for bucket in "${bucket_array[@]}"; do
+    if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
+      bucket_one_found=true
+    elif [ "$bucket" == "versity-gwtest-admin-bucket" ]; then
+      bucket_two_found=true
+    fi
+    if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then
+      break
+    fi
+  done
+  if [ $bucket_one_found == false ] || [ $bucket_two_found == false ]; then
+    fail "not all expected buckets listed"
+  fi
+  change_bucket_owner "$admin_username" "$admin_password" "versity-gwtest-admin-bucket" "$user_username" || local change_result=$?
+  [[ $change_result -eq 0 ]] || fail "error changing bucket owner"
+
+  delete_bucket "aws" "versity-gwtest-admin-bucket"
+  delete_user "$user_username"
+  delete_user "$admin_username"
+}
+
+test_create_user_already_exists() {
+  if [[ $# -ne 1 ]]; then
+    fail "test admin user command requires command type"
+  fi
+
+  username="ABCDEG"
+  password="123456"
+
+  user_exists "$username" || local exists_result=$?
+  if [[ $exists_result -eq 0 ]]; then
+    delete_user "$username" || local delete_result=$?
+    [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'"
+  fi
+
+  create_user "$username" "123456" "admin" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "error creating user"
+  create_user "$username" "123456" "admin" || local create_result=$?
+  [[ $create_result -eq 1 ]] || fail "'user already exists' error not returned"
+
+  delete_bucket "aws" "versity-gwtest-admin-bucket"
+  delete_user "$username"
+}
+
+test_user_user() {
+  if [[ $# -ne 1 ]]; then
+    fail "test admin user command requires command type"
+  fi
+
+  username="ABCDEG"
+  password="123456"
+
+  user_exists "$username" || local exists_result=$?
+  if [[ $exists_result -eq 0 ]]; then
+    delete_user "$username" || local delete_result=$?
+    [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'"
+  fi
+  delete_bucket "aws" "versity-gwtest-user-bucket"
+
+  create_user "$username" "123456" "user" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "error creating user"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  create_bucket_with_user "aws" "versity-gwtest-user-bucket" "$username" "$password" || create_result_two=$?
+  [[ $create_result_two -eq 1 ]] || fail "creating bucket with 'user' account failed to return error"
+  [[ $error == *"Access Denied"* ]] || fail "error message '$error' doesn't contain 'Access Denied'"
+
+  create_bucket "aws" "versity-gwtest-user-bucket" || create_result_three=$?
+  [[ $create_result_three -eq 0 ]] || fail "creating bucket account returned error"
+
+  change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "versity-gwtest-user-bucket" "$username" || local change_result=$?
+  [[ $change_result -eq 0 ]] || fail "error changing bucket owner"
+  change_bucket_owner "$username" "$password" "versity-gwtest-user-bucket" "admin" || local change_result_two=$?
+  [[ $change_result_two -eq 1 ]] || fail "user shouldn't be able to change bucket owner"
+
+  list_buckets_with_user "aws" "$username" "$password"
+  bucket_found=false
+  for bucket in "${bucket_array[@]}"; do
+    if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
+      fail "$BUCKET_ONE_NAME shouldn't show up in 'user' bucket list"
+    elif [ "$bucket" == "versity-gwtest-user-bucket" ]; then
+      bucket_found=true
+    fi
+  done
+  if [ $bucket_found == false ]; then
+    fail "user-owned bucket not found in user list"
+  fi
+
+  delete_bucket "aws" "versity-gwtest-user-bucket"
+  delete_user "$username"
+}
+
+test_userplus_operation() {
+  if [[ $# -ne 1 ]]; then
+    fail "test admin user command requires command type"
+  fi
+
+  username="ABCDEG"
+  password="123456"
+
+  user_exists "$username" || local exists_result=$?
+  if [[ $exists_result -eq 0 ]]; then
+    delete_user "$username" || local delete_result=$?
+    [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'"
+  fi
+  delete_bucket "aws" "versity-gwtest-userplus-bucket"
+
+  create_user "$username" "123456" "userplus" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "error creating user"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  create_bucket_with_user "aws" "versity-gwtest-userplus-bucket" "$username" "$password" || create_result_two=$?
+  [[ $create_result_two -eq 0 ]] || fail "error creating bucket"
+
+  list_buckets_with_user "aws" "$username" "$password"
+  bucket_found=false
+  for bucket in "${bucket_array[@]}"; do
+    if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
+      fail "$BUCKET_ONE_NAME shouldn't show up in 'userplus' bucket list"
+    elif [ "$bucket" == "versity-gwtest-userplus-bucket" ]; then
+      bucket_found=true
+    fi
+  done
+  if [ $bucket_found == false ]; then
+    fail "userplus-owned bucket not found in user list"
+  fi
+
+  change_bucket_owner "$username" "$password" "versity-gwtest-userplus-bucket" "admin" || local change_result_two=$?
+  [[ $change_result_two -eq 1 ]] || fail "userplus shouldn't be able to change bucket owner"
+
+  delete_bucket "aws" "versity-gwtest-admin-bucket"
+  delete_user "$username" || delete_result=$?
+  [[ $delete_result -eq 0 ]] || fail "error deleting user"
+}

tests/test_user_s3cmd.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bats
+
+source ./tests/test_user_common.sh
+
+@test "test_admin_user_s3cmd" {
+  test_admin_user "s3cmd"
+}
+
+@test "test_create_user_already_exists_s3cmd" {
+  test_create_user_already_exists "s3cmd"
+}
+
+@test "test_user_user_s3cmd" {
+  test_user_user "s3cmd"
+}
+
+@test "test_userplus_operation_s3cmd" {
+  test_userplus_operation "s3cmd"
+}

tests/util.sh

@@ -1,34 +1,7 @@
-#!/usr/bin/env bats
+#!/usr/bin/env bash
 
 source ./tests/util_mc.sh
-
-# create an AWS bucket
-# param:  bucket name
-# return 0 for success, 1 for failure
-create_bucket() {
-  if [ $# -ne 2 ]; then
-    echo "create bucket missing command type, bucket name"
-    return 1
-  fi
-
-  local exit_code=0
-  local error
-  if [[ $1 == "aws" ]]; then
-    error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
-  elif [[ $1 == "s3cmd" ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
-  elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
-  else
-    echo "invalid command type $1"
-    return 1
-  fi
-  if [ $exit_code -ne 0 ]; then
-    echo "error creating bucket: $error"
-    return 1
-  fi
-  return 0
-}
+source ./tests/logger.sh
 
 # delete an AWS bucket
 # param:  bucket name
@@ -274,6 +247,7 @@ put_object() {
     echo "invalid command type $1"
     return 1
   fi
+  log 5 "put object exit code: $exit_code"
   if [ $exit_code -ne 0 ]; then
     echo "error copying object to bucket: $error"
     return 1
@@ -281,6 +255,35 @@ put_object() {
   return 0
 }
 
+put_object_multiple() {
+  if [ $# -ne 3 ]; then
+    echo "put object command requires command type, source, destination"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == 'aws' ]]; then
+    # shellcheck disable=SC2086
+    error=$(aws --no-verify-ssl s3 cp "$(dirname "$2")" s3://"$3" --recursive --exclude="*" --include="$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    # shellcheck disable=SC2086
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put $2 "s3://$3/" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    # shellcheck disable=SC2086
+    error=$(mc --insecure cp $2 "$MC_ALIAS"/"$3" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error copying object to bucket: $error"
+    return 1
+  else
+    log 5 "$error"
+  fi
+  return 0
+}
+
 # add object to versitygw if it doesn't exist
 # params:  source file, destination copy location
 # return 0 for success or already exists, 1 for failure
@@ -367,6 +370,35 @@ list_buckets() {
   export bucket_array
 }
 
+list_buckets_with_user() {
+  if [[ $# -ne 3 ]]; then
+    echo "List buckets command missing format, user id, key"
+    return 1
+  fi
+
+  local exit_code=0
+  local output
+  if [[ $1 == "aws" ]]; then
+    output=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3 ls s3:// 2>&1) || exit_code=$?
+  else
+    echo "invalid format:  $1"
+    return 1
+  fi
+
+  if [ $exit_code -ne 0 ]; then
+    echo "error listing buckets: $output"
+    return 1
+  fi
+
+  bucket_array=()
+  while IFS= read -r line; do
+    bucket_name=$(echo "$line" | awk '{print $NF}')
+    bucket_array+=("${bucket_name%/}")
+  done <<< "$output"
+
+  export bucket_array
+}
+
 # list objects on versitygw, in bucket or folder
 # param:  path of bucket or folder
 # export object_array (object names) on success, return 1 for failure
@@ -394,8 +426,10 @@ list_objects() {
 
   object_array=()
   while IFS= read -r line; do
-    object_name=$(echo "$line" | awk '{print $NF}')
-    object_array+=("$object_name")
+    if [[ $line != *InsecureRequestWarning* ]]; then
+      object_name=$(echo "$line" | awk '{print $NF}')
+      object_array+=("$object_name")
+    fi
   done <<< "$output"
 
   export object_array
@@ -519,6 +553,8 @@ get_bucket_tags() {
     echo "invalid command type $1"
     return 1
   fi
+  log 5 "Tags: $tags"
+  tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
   if [[ $result -ne 0 ]]; then
     if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
       export tags=
@@ -593,6 +629,8 @@ get_object_tags() {
     echo "error getting object tags: $tags"
     return 1
   fi
+  log 5 "$tags"
+  tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
   export tags
 }
 
@@ -600,11 +638,15 @@ get_object_tags() {
 # param:  bucket
 # export objects on success, return 1 for failure
 list_objects_s3api_v1() {
-  if [ $# -ne 1 ]; then
-    echo "list objects command missing bucket"
+  if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+    echo "list objects command requires bucket, (optional) delimiter"
     return 1
   fi
-  objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+  if [ "$2" == "" ]; then
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+  else
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
+  fi
   if [[ $result -ne 0 ]]; then
     echo "error listing objects: $objects"
     return 1
@@ -926,3 +968,27 @@ create_presigned_url() {
   fi
   export presigned_url
 }
+
+head_bucket() {
+  if [ $# -ne 2 ]; then
+    echo "head bucket command missing command type, bucket name"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == "aws" ]]; then
+    bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    bucket_info=$(s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error getting bucket info: $bucket_info"
+    return 1
+  fi
+  export bucket_info
+}

tests/util_aws.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+abort_all_multipart_uploads() {
+  if [[ $# -ne 1 ]]; then
+    echo "abort all multipart uploads command missing bucket name"
+    return 1
+  fi
+
+  upload_list=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1) || list_result=$?
+  if [[ $list_result -ne 0 ]]; then
+    echo "error listing multipart uploads: $upload_list"
+    return 1
+  fi
+  log 5 "$upload_list"
+  while IFS= read -r line; do
+    if [[ $line != *"InsecureRequestWarning"* ]]; then
+      modified_upload_list+=("$line")
+    fi
+  done <<< "$upload_list"
+
+  log 5 "Modified upload list: ${modified_upload_list[*]}"
+  has_uploads=$(echo "${modified_upload_list[*]}" | jq 'has("Uploads")')
+  if [[ $has_uploads != false ]]; then
+    lines=$(echo "${modified_upload_list[*]}" | jq -r '.Uploads[] | "--key \(.Key) --upload-id \(.UploadId)"') || lines_result=$?
+    if [[ $lines_result -ne 0 ]]; then
+      echo "error getting lines for multipart upload delete: $lines"
+      return 1
+    fi
+
+    log 5 "$lines"
+    while read -r line; do
+      error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" $line 2>&1) || abort_result=$?
+      if [[ $abort_result -ne 0 ]]; then
+        echo "error aborting multipart upload: $error"
+        return 1
+      fi
+    done <<< "$lines"
+  fi
+  return 0
+}

tests/util_bucket_create.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+source ./tests/util_mc.sh
+source ./tests/logger.sh
+
+# create an AWS bucket
+# param:  bucket name
+# return 0 for success, 1 for failure
+create_bucket() {
+  if [ $# -ne 2 ]; then
+    echo "create bucket missing command type, bucket name"
+    return 1
+  fi
+
+  local exit_code=0
+  local error
+  if [[ $1 == "aws" ]]; then
+    error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "mc" ]]; then
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error creating bucket: $error"
+    return 1
+  fi
+  return 0
+}
+
+create_bucket_with_user() {
+  if [ $# -ne 4 ]; then
+    echo "create bucket missing command type, bucket name, access, secret"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == "aws" ]]; then
+    error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "mc" ]]; then
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error creating bucket: $error"
+    export error
+    return 1
+  fi
+  return 0
+}
+
+create_bucket_invalid_name() {
+  if [ $# -ne 1 ]; then
+    echo "create bucket w/invalid name missing command type"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == "aws" ]]; then
+    bucket_create_error=$(aws --no-verify-ssl s3 mb "s3://" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    bucket_create_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb "s3://" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    bucket_create_error=$(mc --insecure mb "$MC_ALIAS" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -eq 0 ]; then
+    echo "error:  bucket should have not been created but was"
+    return 1
+  fi
+  export bucket_create_error
+}

tests/util_file.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bats
 
+source ./tests/logger.sh
+
 # create a test file and export folder.  do so in temp folder
 # params:  filename
 # export test file folder on success, return 1 for error
@@ -8,7 +10,7 @@ create_test_files() {
     echo "create test files command missing filename"
     return 1
   fi
-  test_file_folder=.
+  test_file_folder=$PWD
   if [[ -z "$GITHUB_ACTIONS" ]]; then
     create_test_file_folder
   fi
@@ -21,6 +23,23 @@ create_test_files() {
   export test_file_folder
 }
 
+create_test_folder() {
+  if [ $# -lt 1 ]; then
+    echo "create test folder command missing folder name"
+    return 1
+  fi
+  test_file_folder=$PWD
+  if [[ -z "$GITHUB_ACTIONS" ]]; then
+    create_test_file_folder
+  fi
+  for name in "$@"; do
+    mkdir -p "$test_file_folder"/"$name" || local mkdir_result=$?
+    if [[ $mkdir_result -ne 0 ]]; then
+      echo "error creating file $name"
+    fi
+  done
+}
+
 # delete a test file
 # params:  filename
 # return:  0 for success, 1 for error
@@ -34,7 +53,7 @@ delete_test_files() {
     return 1
   fi
   for name in "$@"; do
-    rm "$test_file_folder"/"$name" || rm_result=$?
+    rm -rf "${test_file_folder:?}"/"${name:?}" || rm_result=$?
     if [[ $rm_result -ne 0 ]]; then
       echo "error deleting file $name"
     fi
@@ -80,7 +99,11 @@ compare_files() {
 }
 
 create_test_file_folder() {
-  test_file_folder=${TMPDIR}versity-gwtest
+  if [[ -n $TMPDIR ]]; then
+    test_file_folder=${TMPDIR}versity-gwtest
+  else
+    test_file_folder=$PWD/versity-gwtest
+  fi
   mkdir -p "$test_file_folder" || local mkdir_result=$?
   if [[ $mkdir_result -ne 0 ]]; then
     echo "error creating test file folder"
@@ -97,16 +120,40 @@ create_large_file() {
     return 1
   fi
 
-  test_file_folder=.
+  test_file_folder=$PWD
   if [[ -z "$GITHUB_ACTIONS" ]]; then
     create_test_file_folder
   fi
 
   filesize=$((160*1024*1024))
-  error=$(dd if=/dev/urandom of=$test_file_folder/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$?
+  error=$(dd if=/dev/urandom of="$test_file_folder"/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$?
   if [[ $dd_result -ne 0 ]]; then
     echo "error creating file: $error"
     return 1
   fi
   return 0
 }
+
+create_test_file_count() {
+  if [[ $# -ne 1 ]]; then
+    echo "create test file count function missing bucket name, count"
+    return 1
+  fi
+  test_file_folder=$PWD
+  if [[ -z "$GITHUB_ACTIONS" ]]; then
+    create_test_file_folder
+  fi
+  local touch_result
+  for ((i=1;i<=$1;i++)) {
+    error=$(touch "$test_file_folder/file_$i") || touch_result=$?
+    if [[ $touch_result -ne 0 ]]; then
+      echo "error creating file_$i:  $error"
+      return 1
+    fi
+  }
+  if [[ $LOG_LEVEL -ge 5 ]]; then
+    ls_result=$(ls "$test_file_folder"/file_*)
+    log 5 "$ls_result"
+  fi
+  return 0
+}

tests/util_mc.sh

@@ -21,4 +21,4 @@ delete_bucket_recursive_mc() {
     return 1
   fi
   return 0
-}
+}

tests/util_users.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+create_user() {
+  if [[ $# -ne 3 ]]; then
+    echo "create user command requires user ID, key, and role"
+    return 1
+  fi
+  create_user_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3" || create_result=$?
+  if [[ $create_result -ne 0 ]]; then
+    echo "error creating user: $error"
+    return 1
+  fi
+  return 0
+}
+
+create_user_with_user() {
+  if [[ $# -ne 5 ]]; then
+    echo "create user with user command requires creator ID, key, and new user ID, key, and role"
+    return 1
+  fi
+  error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5") || local create_result=$?
+  if [[ $create_result -ne 0 ]]; then
+    echo "error creating user: $error"
+    return 1
+  fi
+  return 0
+}
+
+list_users() {
+  users=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users) || local list_result=$?
+  if [[ $list_result -ne 0 ]]; then
+    echo "error listing users: $users"
+    return 1
+  fi
+  parsed_users=()
+  while IFS= read -r line; do
+    parsed_users+=("$line")
+  done < <(awk 'NR>2 {print $1}' <<< "$users")
+  export parsed_users
+  return 0
+}
+
+user_exists() {
+  if [[ $# -ne 1 ]]; then
+    echo "user exists command requires username"
+    return 2
+  fi
+  list_users || local list_result=$?
+  if [[ $list_result -ne 0 ]]; then
+    echo "error listing user"
+    return 2
+  fi
+  for element in "${parsed_users[@]}"; do
+    if [[ $element == "$1" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+delete_user() {
+  if [[ $# -ne 1 ]]; then
+      echo "delete user command requires user ID"
+      return 1
+    fi
+    error=$($VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access "$1") || local delete_result=$?
+    if [[ $delete_result -ne 0 ]]; then
+      echo "error deleting user: $error"
+      return 1
+    fi
+    return 0
+}
+
+change_bucket_owner() {
+  if [[ $# -ne 4 ]]; then
+      echo "change bucket owner command requires ID, key, bucket name, and new owner"
+      return 1
+    fi
+    error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$?
+    if [[ $change_result -ne 0 ]]; then
+      echo "error changing bucket owner: $error"
+      return 1
+    fi
+    return 0
+}

tests/versity.sh

@@ -44,6 +44,12 @@ check_exe_params() {
   elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then
     echo "RUN_VERSITYGW must be 'true' or 'false'"
     return 1
+  elif [ -z "$USERS_FOLDER" ]; then
+    echo "No users folder parameter set"
+    return 1
+  fi
+  if [[ -r $GOCOVERDIR ]]; then
+    export GOCOVERDIR=$GOCOVERDIR
   fi
   if [[ $RUN_VERSITYGW == "true" ]]; then
     local check_result
@@ -86,7 +92,7 @@ start_versity() {
     fi
   fi
 
-  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
+  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL VERSITY_EXE
 }
 
 start_versity_process() {
@@ -125,7 +131,7 @@ run_versity_app_posix() {
     echo "run versity app w/posix command requires access ID, secret key, process number"
     return 1
   fi
-  base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION")
+  base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION"  --iam-dir="$USERS_FOLDER")
   if [ -n "$CERT" ] && [ -n "$KEY" ]; then
     base_command+=(--cert "$CERT" --key "$KEY")
   fi