feat: implement read only option for posix/scoutfs

Fixes #484. When read only option selected, backend will return
AccessDenied for any operations that would update data or metadata
in the backend. This include creating and deleting objects and
buckets as well as updating object metadata.

This will still allow admin to change bucket owner since read only
is mainly intended for s3 api only.

.github/dependabot.yml

@@ -8,3 +8,7 @@ updates:
       dev-dependencies:
         patterns:
           - "*"
+    allow:
+      # Allow both direct and indirect updates for all packages
+      - dependency-type: "all"
+

.github/workflows/docker.yaml

@@ -12,7 +12,7 @@ jobs:
       packages: write
       contents: read
     steps:
-      - name: Check out the repo
+      - name: Checkout
         uses: actions/checkout@v4
 
       - name: Log in to Docker Hub
@@ -43,3 +43,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            VERSION=${{ github.event.release.tag_name }}
+            TIME=${{ github.event.release.published_at }}
+            BUILD=${{ github.sha }}

.github/workflows/functional.yml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go

.github/workflows/go.yml

@@ -8,10 +8,10 @@ jobs:
     steps:
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go

.github/workflows/goreleaser.yml

@@ -15,14 +15,21 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - run: git fetch --force --tags
-      - uses: actions/setup-go@v4
+
+      - name: Fetch tags
+        run: git fetch --force --tags
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
         with:
           go-version: stable
-      - uses: goreleaser/goreleaser-action@v4
+
+      - name: Run Releaser
+        uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser
           version: latest

.github/workflows/static.yml

@@ -7,16 +7,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go
  
     - name: "staticcheck"
-      uses: dominikh/staticcheck-action@v1.3.0
-      with: 
-        install-go: false
+      uses: dominikh/staticcheck-action@v1
+      with:
+        version: "latest"

.github/workflows/system.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install ShellCheck
         run: sudo apt-get install shellcheck
@@ -15,7 +15,7 @@ jobs:
         run: shellcheck -S warning ./tests/*.sh
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 'stable'
         id: go
@@ -51,8 +51,8 @@ jobs:
           export WORKSPACE=$GITHUB_WORKSPACE
           openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir /tmp/cover
-          VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+          mkdir cover
+          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
 
       #- name: Build and run, s3 backend
       #  run: |
@@ -70,4 +70,4 @@ jobs:
 
       - name: Coverage report
         run: |
-          go tool covdata percent -i=/tmp/cover
+          go tool covdata percent -i=cover

Dockerfile

@@ -1,5 +1,15 @@
 FROM golang:latest
 
+# Set build arguments with default values
+ARG VERSION="none"
+ARG BUILD="none"
+ARG TIME="none"
+
+# Set environment variables
+ENV VERSION=${VERSION}
+ENV BUILD=${BUILD}
+ENV TIME=${TIME}
+
 WORKDIR /app
 
 COPY go.mod ./
@@ -9,7 +19,7 @@ COPY ./ ./
 
 WORKDIR /app/cmd/versitygw
 ENV CGO_ENABLED=0
-RUN go build -o versitygw
+RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
 
 FROM alpine:latest
 

Dockerfile_test_bats

@@ -61,7 +61,6 @@ USER tester
 COPY --chown=tester:tester . /home/tester
 
 WORKDIR /home/tester
-RUN cp ${CONFIG_FILE}.default $CONFIG_FILE
 #RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
 RUN cp tests/s3cfg.local.default tests/s3cfg.local
 RUN make

auth/acl.go

@@ -206,15 +206,6 @@ func splitUnique(s, divider string) []string {
 }
 
 func verifyACL(acl ACL, access string, permission types.Permission) error {
-	// Default disabled ACL case
-	if acl.ACL == "" && len(acl.Grantees) == 0 {
-		if acl.Owner == access {
-			return nil
-		}
-
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
 	if acl.ACL != "" {
 		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -225,6 +216,9 @@ func verifyACL(acl ACL, access string, permission types.Permission) error {
 
 		return nil
 	} else {
+		if len(acl.Grantees) == 0 {
+			return nil
+		}
 		grantee := Grantee{Access: access, Permission: permission}
 		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
 
@@ -298,10 +292,20 @@ func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) e
 		return nil
 	}
 
-	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+	policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
+	if err != nil {
 		return err
 	}
-	if err := verifyBucketPolicy(ctx, be, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
+
+	// If bucket policy is not set and the ACL is default, only the owner has access
+	if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
+		return err
+	}
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
 		return err
 	}
 

auth/bucket_policy.go

@@ -15,12 +15,10 @@
 package auth
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
 
-	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -41,8 +39,13 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
 
 func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
 	for _, statement := range bp.Statement {
-		if statement.isAllowed(principal, action, resource) {
-			return true
+		if statement.findMatch(principal, action, resource) {
+			switch statement.Effect {
+			case BucketPolicyAccessTypeAllow:
+				return true
+			case BucketPolicyAccessTypeDeny:
+				return false
+			}
 		}
 	}
 
@@ -83,14 +86,9 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 	return nil
 }
 
-func (bpi *BucketPolicyItem) isAllowed(principal string, action Action, resource string) bool {
+func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
 	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
-		switch bpi.Effect {
-		case BucketPolicyAccessTypeAllow:
-			return true
-		case BucketPolicyAccessTypeDeny:
-			return false
-		}
+		return true
 	}
 
 	return false
@@ -117,26 +115,23 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 	return nil
 }
 
-func verifyBucketPolicy(ctx context.Context, be backend.Backend, access, bucket, object string, action Action) error {
-	policyDoc, err := be.GetBucketPolicy(ctx, bucket)
-	if err != nil {
-		return err
-	}
+func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
 	// If bucket policy is not set
-	if len(policyDoc) == 0 {
+	if len(policy) == 0 {
 		return nil
 	}
 
 	var bucketPolicy BucketPolicy
-	if err := json.Unmarshal(policyDoc, &bucketPolicy); err != nil {
+	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
 		return err
 	}
 
 	resource := bucket
 	if object != "" {
-		resource += "" + object
+		resource += "/" + object
 	}
 
+	fmt.Println(access, action, resource)
 	if !bucketPolicy.isAllowed(access, action, resource) {
 		return s3err.GetAPIError(s3err.ErrAccessDenied)
 	}

backend/mkdir.go

@@ -0,0 +1,81 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// MkdirAll borrowed from stdlib to add ability to set ownership
+// as directories are created
+
+package backend
+
+import (
+	"io/fs"
+	"os"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	// TODO: make this configurable
+	defaultDirPerm fs.FileMode = 0755
+)
+
+// MkdirAll is similar to os.MkdirAll but it will return
+// ErrObjectParentIsFile when appropriate
+// MkdirAll creates a directory named path,
+// along with any necessary parents, and returns nil,
+// or else returns an error.
+// The permission bits perm (before umask) are used for all
+// directories that MkdirAll creates.
+// Any newly created directory is set to provided uid/gid ownership.
+// If path is already a directory, MkdirAll does nothing
+// and returns nil.
+// Any directoy created will be set to provided uid/gid ownership
+// if doChown is true.
+func MkdirAll(path string, uid, gid int, doChown bool) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = MkdirAll(path[:j-1], uid, gid, doChown)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, defaultDirPerm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return err
+	}
+	if doChown {
+		err = os.Chown(path, uid, gid)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

backend/posix/posix.go

@@ -46,6 +46,19 @@ type Posix struct {
 
 	rootfd  *os.File
 	rootdir string
+
+	// chownuid/gid enable chowning of files to the account uid/gid
+	// when objects are uploaded
+	chownuid bool
+	chowngid bool
+
+	// read only mode prevents any backend modifications
+	readonly bool
+
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
 }
 
 var _ backend.Backend = &Posix{}
@@ -64,7 +77,13 @@ const (
 	policykey           = "user.policy"
 )
 
-func New(rootdir string) (*Posix, error) {
+type PosixOpts struct {
+	ChownUID bool
+	ChownGID bool
+	ReadOnly bool
+}
+
+func New(rootdir string, opts PosixOpts) (*Posix, error) {
 	err := os.Chdir(rootdir)
 	if err != nil {
 		return nil, fmt.Errorf("chdir %v: %w", rootdir, err)
@@ -81,7 +100,15 @@ func New(rootdir string) (*Posix, error) {
 		return nil, fmt.Errorf("xattr not supported on %v", rootdir)
 	}
 
-	return &Posix{rootfd: f, rootdir: rootdir}, nil
+	return &Posix{
+		rootfd:   f,
+		rootdir:  rootdir,
+		euid:     os.Geteuid(),
+		egid:     os.Getegid(),
+		chownuid: opts.ChownUID,
+		chowngid: opts.ChownGID,
+		readonly: opts.ReadOnly,
+	}, nil
 }
 
 func (p *Posix) Shutdown() {
@@ -168,14 +195,30 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He
 	return &s3.HeadBucketOutput{}, nil
 }
 
-func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl []byte) error {
+var (
+	// TODO: make this configurable
+	defaultDirPerm fs.FileMode = 0755
+)
+
+func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
 
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	bucket := *input.Bucket
 
-	err := os.Mkdir(bucket, 0777)
+	err := os.Mkdir(bucket, defaultDirPerm)
 	if err != nil && os.IsExist(err) {
 		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
 	}
@@ -183,6 +226,13 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl
 		return fmt.Errorf("mkdir bucket: %w", err)
 	}
 
+	if doChown {
+		err := os.Chown(bucket, uid, gid)
+		if err != nil {
+			return fmt.Errorf("chown bucket: %w", err)
+		}
+	}
+
 	if err := xattr.Set(bucket, aclkey, acl); err != nil {
 		return fmt.Errorf("set acl: %w", err)
 	}
@@ -191,6 +241,10 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl
 }
 
 func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -224,6 +278,10 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
 }
 
 func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	if p.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if mpu.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -287,7 +345,35 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
 	}, nil
 }
 
-func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+// getChownIDs returns the uid and gid that should be used for chowning
+// the object to the account uid/gid. It also returns a boolean indicating
+// if chowning is needed.
+func (p *Posix) getChownIDs(acct auth.Account) (int, int, bool) {
+	uid := p.euid
+	gid := p.egid
+	var needsChown bool
+	if p.chownuid && acct.UserID != p.euid {
+		uid = acct.UserID
+		needsChown = true
+	}
+	if p.chowngid && acct.GroupID != p.egid {
+		gid = acct.GroupID
+		needsChown = true
+	}
+
+	return uid, gid, needsChown
+}
+
+func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	if p.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -351,8 +437,12 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 		}
 	}
 
-	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, totalsize)
+	f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object,
+		totalsize, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return nil, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -365,6 +455,9 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 		_, err = io.Copy(f, pf)
 		pf.Close()
 		if err != nil {
+			if errors.Is(err, syscall.EDQUOT) {
+				return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+			}
 			return nil, fmt.Errorf("copy part %v: %v", p.PartNumber, err)
 		}
 	}
@@ -376,9 +469,10 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		uid, gid, doChown := p.getChownIDs(acct)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
-			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			return nil, err
 		}
 	}
 	err = f.link()
@@ -497,52 +591,11 @@ func isValidMeta(val string) bool {
 	return false
 }
 
-// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
-// when appropriate
-func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(path[:j-1], perm, bucket, object)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-	return nil
-}
-
 func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if mpu.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -841,7 +894,16 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
 	}, nil
 }
 
-func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) {
+func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (string, error) {
+	if p.readonly {
+		return "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if input.Bucket == nil {
 		return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -880,9 +942,12 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 
 	partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part))
 
-	f, err := openTmpFile(filepath.Join(bucket, objdir),
-		bucket, partPath, length)
+	f, err := p.openTmpFile(filepath.Join(bucket, objdir),
+		bucket, partPath, length, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("open temp file: %w", err)
 	}
 
@@ -890,6 +955,9 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 	tr := io.TeeReader(r, hash)
 	_, err = io.Copy(f, tr)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("write part data: %w", err)
 	}
 
@@ -907,7 +975,16 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 	return etag, nil
 }
 
-func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	if p.readonly {
+		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if upi.Bucket == nil {
 		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -974,9 +1051,12 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRange)
 	}
 
-	f, err := openTmpFile(filepath.Join(*upi.Bucket, objdir),
-		*upi.Bucket, partPath, length)
+	f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
+		*upi.Bucket, partPath, length, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return s3response.CopyObjectResult{}, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -996,6 +1076,9 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 
 	_, err = io.Copy(f, tr)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return s3response.CopyObjectResult{}, fmt.Errorf("copy part data: %w", err)
 	}
 
@@ -1020,6 +1103,15 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 }
 
 func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
+	if p.readonly {
+		return "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if po.Bucket == nil {
 		return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1053,6 +1145,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 
 	name := filepath.Join(*po.Bucket, *po.Key)
 
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	contentLength := int64(0)
 	if po.ContentLength != nil {
 		contentLength = *po.ContentLength
@@ -1066,8 +1160,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 			return "", s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData)
 		}
 
-		err = mkdirAll(name, os.FileMode(0755), *po.Bucket, *po.Key)
+		err = backend.MkdirAll(name, uid, gid, doChown)
 		if err != nil {
+			if errors.Is(err, syscall.EDQUOT) {
+				return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+			}
 			return "", err
 		}
 
@@ -1087,9 +1184,12 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 		return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 	}
 
-	f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
-		*po.Bucket, *po.Key, contentLength)
+	f, err := p.openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
+		*po.Bucket, *po.Key, contentLength, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -1098,11 +1198,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 	rdr := io.TeeReader(po.Body, hash)
 	_, err = io.Copy(f, rdr)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("write object data: %w", err)
 	}
 	dir := filepath.Dir(name)
 	if dir != "" {
-		err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
 			return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 		}
@@ -1132,6 +1235,10 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 }
 
 func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1196,6 +1303,10 @@ func (p *Posix) removeParents(bucket, object string) error {
 }
 
 func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+	if p.readonly {
+		return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	// delete object already checks bucket
 	delResult, errs := []types.DeletedObject{}, []types.Error{}
 	for _, obj := range input.Delete.Objects {
@@ -1411,6 +1522,10 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
 }
 
 func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	if p.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1680,6 +1795,10 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
 }
 
 func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1718,6 +1837,10 @@ func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]
 }
 
 func (p *Posix) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1728,9 +1851,10 @@ func (p *Posix) PutBucketTagging(_ context.Context, bucket string, tags map[stri
 
 	if tags == nil {
 		err = xattr.Remove(bucket, "user."+tagHdr)
-		if err != nil {
+		if err != nil && !isNoAttr(err) {
 			return fmt.Errorf("remove tags: %w", err)
 		}
+
 		return nil
 	}
 
@@ -1802,6 +1926,10 @@ func (p *Posix) getXattrTags(bucket, object string) (map[string]string, error) {
 }
 
 func (p *Posix) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1842,6 +1970,10 @@ func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object string)
 }
 
 func (p *Posix) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)

backend/posix/with_otmpfile.go

@@ -27,30 +27,42 @@ import (
 	"strconv"
 	"syscall"
 
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 	"golang.org/x/sys/unix"
 )
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	isOTmp  bool
-	size    int64
+	f          *os.File
+	bucket     string
+	objname    string
+	isOTmp     bool
+	size       int64
+	needsChown bool
+	uid        int
+	gid        int
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+var (
+	// TODO: make this configurable
+	defaultFilePerm uint32 = 0644
+)
+
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
 	// Not all filesystems support this, so fallback to CreateTemp for when
 	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err := os.MkdirAll(dir, 0700)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -59,11 +71,27 @@ func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 		if err != nil {
 			return nil, err
 		}
-		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		tmp := &tmpfile{
+			f:          f,
+			bucket:     bucket,
+			objname:    obj,
+			size:       size,
+			needsChown: doChown,
+			uid:        uid,
+			gid:        gid,
+		}
 		// falloc is best effort, its fine if this fails
 		if size > 0 {
 			tmp.falloc()
 		}
+
+		if doChown {
+			err := f.Chown(uid, gid)
+			if err != nil {
+				return nil, fmt.Errorf("set temp file ownership: %w", err)
+			}
+		}
+
 		return tmp, nil
 	}
 
@@ -71,11 +99,29 @@ func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		isOTmp:     true,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+
 	// falloc is best effort, its fine if this fails
 	if size > 0 {
 		tmp.falloc()
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return tmp, nil
 }
 
@@ -100,6 +146,13 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
+	dir := filepath.Dir(objPath)
+
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	if err != nil {
+		return fmt.Errorf("make parent dir: %w", err)
+	}
+
 	if !tmp.isOTmp {
 		// O_TMPFILE not suported, use fallback
 		return tmp.fallbackLink()
@@ -111,14 +164,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dir, err := os.Open(filepath.Dir(objPath))
+	dirf, err := os.Open(dir)
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dir.Close()
+	defer dirf.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile (%q in %q): %w",
 			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
@@ -138,6 +191,9 @@ func (tmp *tmpfile) fallbackLink() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(fs.FileMode(defaultFilePerm))
+
 	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)

backend/posix/without_otmpfile.go

@@ -24,6 +24,9 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 )
 
 type tmpfile struct {
@@ -33,20 +36,36 @@ type tmpfile struct {
 	size    int64
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	// Create a temp file for upload while in progress (see link comments below).
-	err := os.MkdirAll(dir, 0700)
+	var err error
+	err = backend.MkdirAll(dir, uid, gid, doChown)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create temp file: %w", err)
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
 }
 
+var (
+	// TODO: make this configurable
+	defaultFilePerm fs.FileMode = 0644
+)
+
 func (tmp *tmpfile) link() error {
 	tempname := tmp.f.Name()
 	// cleanup in case anything goes wrong, if rename succeeds then
@@ -64,6 +83,9 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(defaultFilePerm)
+
 	err = tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)

backend/scoutfs/scoutfs.go

@@ -25,15 +25,24 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
 )
 
+type ScoutfsOpts struct {
+	ChownUID    bool
+	ChownGID    bool
+	ReadOnly    bool
+	GlacierMode bool
+}
+
 type ScoutFS struct {
 	*posix.Posix
 	rootfd  *os.File
@@ -49,6 +58,19 @@ type ScoutFS struct {
 	// ListObjects: if file offline, set obj storage class to GLACIER
 	// RestoreObject: add batch stage request to file
 	glaciermode bool
+
+	// chownuid/gid enable chowning of files to the account uid/gid
+	// when objects are uploaded
+	chownuid bool
+	chowngid bool
+
+	// read only mode prevents any backend modifications
+	readonly bool
+
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -92,14 +114,6 @@ const (
 	ExtCacheDone
 )
 
-// Option sets various options for scoutfs
-type Option func(s *ScoutFS)
-
-// WithGlacierEmulation sets glacier mode emulation
-func WithGlacierEmulation() Option {
-	return func(s *ScoutFS) { s.glaciermode = true }
-}
-
 func (s *ScoutFS) Shutdown() {
 	s.Posix.Shutdown()
 	s.rootfd.Close()
@@ -110,10 +124,51 @@ func (*ScoutFS) String() string {
 	return "ScoutFS Gateway"
 }
 
+// getChownIDs returns the uid and gid that should be used for chowning
+// the object to the account uid/gid. It also returns a boolean indicating
+// if chowning is needed.
+func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
+	uid := s.euid
+	gid := s.egid
+	var needsChown bool
+	if s.chownuid && acct.UserID != s.euid {
+		uid = acct.UserID
+		needsChown = true
+	}
+	if s.chowngid && acct.GroupID != s.egid {
+		gid = acct.GroupID
+		needsChown = true
+	}
+
+	return uid, gid, needsChown
+}
+
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
-func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	if s.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	if input.Key == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if input.UploadId == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
+	}
+	if input.MultipartUpload == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
@@ -174,8 +229,11 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 
 	// use totalsize=0 because we wont be writing to the file, only moving
 	// extents around.  so we dont want to fallocate this.
-	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
+	f, err := s.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return nil, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -203,9 +261,10 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		uid, gid, doChown := s.getChownIDs(acct)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
-			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			return nil, err
 		}
 	}
 	err = f.link()
@@ -310,51 +369,6 @@ func isValidMeta(val string) bool {
 	return false
 }
 
-// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
-// when appropriate
-func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(path[:j-1], perm, bucket, object)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-	return nil
-}
-
 func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key

backend/scoutfs/scoutfs_compat.go

@@ -17,7 +17,6 @@
 package scoutfs
 
 import (
-	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -29,11 +28,17 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/versity/scoutfs-go"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 )
 
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
-	p, err := posix.New(rootdir)
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
+	p, err := posix.New(rootdir, posix.PosixOpts{
+		ChownUID: opts.ChownUID,
+		ChownGID: opts.ChownGID,
+		ReadOnly: opts.ReadOnly,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -43,60 +48,71 @@ func New(rootdir string, opts ...Option) (*ScoutFS, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
-	s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
-	for _, opt := range opts {
-		opt(s)
-	}
-
-	return s, nil
+	return &ScoutFS{
+		Posix:    p,
+		rootfd:   f,
+		rootdir:  rootdir,
+		chownuid: opts.ChownUID,
+		chowngid: opts.ChownGID,
+		readonly: opts.ReadOnly,
+	}, nil
 }
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	isOTmp  bool
-	size    int64
+	f          *os.File
+	bucket     string
+	objname    string
+	size       int64
+	needsChown bool
+	uid        int
+	gid        int
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+var (
+	// TODO: make this configurable
+	defaultFilePerm uint32 = 0644
+)
+
+func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := s.getChownIDs(acct)
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
-	// Not all filesystems support this, so fallback to CreateTemp for when
-	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
-		// O_TMPFILE not supported, try fallback
-		err := os.MkdirAll(dir, 0700)
-		if err != nil {
-			return nil, fmt.Errorf("make temp dir: %w", err)
-		}
-		f, err := os.CreateTemp(dir,
-			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-		if err != nil {
-			return nil, err
-		}
-		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
-		// falloc is best effort, its fine if this fails
-		if size > 0 {
-			tmp.falloc()
-		}
-		return tmp, nil
+		return nil, err
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+
 	// falloc is best effort, its fine if this fails
 	if size > 0 {
 		tmp.falloc()
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return tmp, nil
 }
 
@@ -121,9 +137,11 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	if !tmp.isOTmp {
-		// O_TMPFILE not suported, use fallback
-		return tmp.fallbackLink()
+	dir := filepath.Dir(objPath)
+
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	if err != nil {
+		return fmt.Errorf("make parent dir: %w", err)
 	}
 
 	procdir, err := os.Open(procfddir)
@@ -132,14 +150,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dir, err := os.Open(filepath.Dir(objPath))
+	dirf, err := os.Open(dir)
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dir.Close()
+	defer dirf.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile: %w", err)
 	}
@@ -152,26 +170,6 @@ func (tmp *tmpfile) link() error {
 	return nil
 }
 
-func (tmp *tmpfile) fallbackLink() error {
-	tempname := tmp.f.Name()
-	// cleanup in case anything goes wrong, if rename succeeds then
-	// this will no longer exist
-	defer os.Remove(tempname)
-
-	err := tmp.f.Close()
-	if err != nil {
-		return fmt.Errorf("close tmpfile: %w", err)
-	}
-
-	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
-}
-
 func (tmp *tmpfile) Write(b []byte) (int, error) {
 	if int64(len(b)) > tmp.size {
 		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)

backend/scoutfs/scoutfs_incompat.go

@@ -20,9 +20,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
+
+	"github.com/versity/versitygw/auth"
 )
 
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
@@ -34,7 +36,13 @@ var (
 	errNotSupported = errors.New("not supported")
 )
 
-func openTmpFile(_, _, _ string, _ int64) (*tmpfile, error) {
+func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile, error) {
+	// make these look used for static check
+	_ = s.chownuid
+	_ = s.chowngid
+	_ = s.euid
+	_ = s.egid
+	_ = s.readonly
 	return nil, errNotSupported
 }
 

cmd/versitygw/gateway_test.go

@@ -56,7 +56,7 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir)
+	be, err := posix.New(tempdir, posix.PosixOpts{})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}

cmd/versitygw/main.go

@@ -19,6 +19,8 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"github.com/gofiber/fiber/v2"
@@ -40,10 +42,13 @@ var (
 	certFile, keyFile                      string
 	kafkaURL, kafkaTopic, kafkaKey         string
 	natsURL, natsTopic                     string
+	eventWebhookURL                        string
+	eventConfigFilePath                    string
 	logWebhookURL                          string
 	accessLog                              string
 	healthPath                             string
 	debug                                  bool
+	pprof                                  string
 	quiet                                  bool
 	iamDir                                 string
 	ldapURL, ldapBindDN, ldapPassword      string
@@ -79,6 +84,7 @@ func main() {
 		azureCommand(),
 		adminCommand(),
 		testCommand(),
+		utilsCommand(),
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -187,6 +193,12 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
+		&cli.StringFlag{
+			Name:        "pprof",
+			Usage:       "enable pprof debug on specified port",
+			EnvVars:     []string{"VGW_PPROF"},
+			Destination: &pprof,
+		},
 		&cli.BoolFlag{
 			Name:        "quiet",
 			Usage:       "silence stdout request logging output",
@@ -241,6 +253,20 @@ func initFlags() []cli.Flag {
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
+		&cli.StringFlag{
+			Name:        "event-webhook-url",
+			Usage:       "webhook url to send bucket notifications",
+			EnvVars:     []string{"VGW_EVENT_WEBHOOK_URL"},
+			Destination: &eventWebhookURL,
+			Aliases:     []string{"ewu"},
+		},
+		&cli.StringFlag{
+			Name:        "event-filter",
+			Usage:       "bucket event notifications filters configuration file path",
+			EnvVars:     []string{"VGW_EVENT_FILTER"},
+			Destination: &eventConfigFilePath,
+			Aliases:     []string{"ef"},
+		},
 		&cli.StringFlag{
 			Name:        "iam-dir",
 			Usage:       "if defined, run internal iam service within this directory",
@@ -373,6 +399,14 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		return fmt.Errorf("root user access and secret key must be provided")
 	}
 
+	if pprof != "" {
+		// listen on specified port for pprof debug
+		// point browser to http://<ip:port>/debug/pprof/
+		go func() {
+			log.Fatal(http.ListenAndServe(pprof, nil))
+		}()
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:           "versitygw",
 		ServerHeader:      "VERSITYGW",
@@ -465,14 +499,16 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
-		KafkaURL:      kafkaURL,
-		KafkaTopic:    kafkaTopic,
-		KafkaTopicKey: kafkaKey,
-		NatsURL:       natsURL,
-		NatsTopic:     natsTopic,
+		KafkaURL:             kafkaURL,
+		KafkaTopic:           kafkaTopic,
+		KafkaTopicKey:        kafkaKey,
+		NatsURL:              natsURL,
+		NatsTopic:            natsTopic,
+		WebhookURL:           eventWebhookURL,
+		FilterConfigFilePath: eventConfigFilePath,
 	})
 	if err != nil {
-		return fmt.Errorf("unable to connect to the message broker: %w", err)
+		return fmt.Errorf("init bucket event notifications: %w", err)
 	}
 
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
@@ -531,5 +567,15 @@ Loop:
 		}
 	}
 
+	if evSender != nil {
+		err := evSender.Close()
+		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
+			fmt.Fprintf(os.Stderr, "close event sender: %v\n", err)
+		}
+	}
+
 	return saveErr
 }

cmd/versitygw/posix.go

@@ -21,6 +21,11 @@ import (
 	"github.com/versity/versitygw/backend/posix"
 )
 
+var (
+	chownuid, chowngid bool
+	readonly           bool
+)
+
 func posixCommand() *cli.Command {
 	return &cli.Command{
 		Name:  "posix",
@@ -36,6 +41,26 @@ bucket: mybucket
 object: a/b/c/myobject
 will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 		Action: runPosix,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
+			&cli.BoolFlag{
+				Name:        "readonly",
+				Usage:       "allow only read operations to backend",
+				EnvVars:     []string{"VGW_READ_ONLY"},
+				Destination: &readonly,
+			},
+		},
 	}
 }
 
@@ -44,7 +69,11 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	be, err := posix.New(ctx.Args().Get(0))
+	be, err := posix.New(ctx.Args().Get(0), posix.PosixOpts{
+		ChownUID: chownuid,
+		ChownGID: chowngid,
+		ReadOnly: readonly,
+	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)
 	}

cmd/versitygw/scoutfs.go

@@ -23,6 +23,10 @@ import (
 
 var (
 	glacier bool
+
+	// defined in posix.go:
+	// chownuid, chowngid bool
+	// readonly           bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -51,6 +55,24 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
+			&cli.BoolFlag{
+				Name:        "readonly",
+				Usage:       "allow only read operations to backend",
+				EnvVars:     []string{"VGW_READ_ONLY"},
+				Destination: &readonly,
+			},
 		},
 	}
 }
@@ -60,12 +82,13 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	var opts []scoutfs.Option
-	if glacier {
-		opts = append(opts, scoutfs.WithGlacierEmulation())
-	}
+	var opts scoutfs.ScoutfsOpts
+	opts.GlacierMode = glacier
+	opts.ChownUID = chownuid
+	opts.ChownGID = chowngid
+	opts.ReadOnly = readonly
 
-	be, err := scoutfs.New(ctx.Args().Get(0), opts...)
+	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}

cmd/versitygw/test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package main
 
 import (
@@ -84,6 +98,11 @@ func initTestCommands() []*cli.Command {
 			Usage:  "Tests iam service",
 			Action: getAction(integration.TestIAM),
 		},
+		{
+			Name:   "access-control",
+			Usage:  "Tests gateway access control with bucket ACLs and Policies",
+			Action: getAction(integration.TestAccessControl),
+		},
 		{
 			Name:  "bench",
 			Usage: "Runs download/upload performance test on the gateway",

cmd/versitygw/utils.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/s3event"
+)
+
+func utilsCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "utils",
+		Usage: "utility helper CLI tool",
+		Subcommands: []*cli.Command{
+			{
+				Name:    "gen-event-filter-config",
+				Aliases: []string{"gefc"},
+				Usage:   "Create a new configuration file for bucket event notifications filter.",
+				Action:  generateEventFiltersConfig,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:    "path",
+						Usage:   "the path where the config file has to be created",
+						Aliases: []string{"p"},
+					},
+				},
+			},
+		},
+	}
+}
+
+func generateEventFiltersConfig(ctx *cli.Context) error {
+	pathFlag := ctx.String("path")
+	path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
+	if err != nil {
+		return err
+	}
+
+	config := s3event.EventFilter{
+		s3event.EventObjectCreated:           true,
+		s3event.EventObjectCreatedPut:        true,
+		s3event.EventObjectCreatedPost:       true,
+		s3event.EventObjectCreatedCopy:       true,
+		s3event.EventCompleteMultipartUpload: true,
+		s3event.EventObjectDeleted:           true,
+		s3event.EventObjectTagging:           true,
+		s3event.EventObjectTaggingPut:        true,
+		s3event.EventObjectTaggingDelete:     true,
+		s3event.EventObjectAclPut:            true,
+		s3event.EventObjectRestore:           true,
+		s3event.EventObjectRestorePost:       true,
+		s3event.EventObjectRestoreCompleted:  true,
+	}
+
+	configBytes, err := json.Marshal(config)
+	if err != nil {
+		return fmt.Errorf("parse event config: %w", err)
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create config file: %w", err)
+	}
+	defer file.Close()
+
+	_, err = file.Write(configBytes)
+	if err != nil {
+		return fmt.Errorf("write config file: %w", err)
+	}
+
+	return nil
+}

extra/example.conf

@@ -121,28 +121,55 @@ ROOT_SECRET_ACCESS_KEY=
 # Event Logs #
 ##############
 
+# The gateway events are similar to AWS S3 events, and are documented in the
+# wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+
+# The VGW_EVENT_FILTER option specifies a config file that contains the
+# event filter rules. The event filter rules are used to determine which
+# events are sent to the configured event services.
+# Use the following to generate a default rules file in /etc/versitygw.d/:
+# versitygw utils gen-event-filter-config -p /etc/versitygw.d
+# The resulting file, /etc/versitygw.d/event_config.json, can be modified and
+# specified in the VGW_EVENT_FILTER option.
+# When VGW_EVENT_FILTER is not specified, all events are sent to the configured
+# event service.
+#VGW_EVENT_FILTER=
+
 # Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
 # VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
-# bucket events will be sent to the kafka service. The gateway events are
-# similar to AWS S3 events, and are documented in the wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
+# configured bucket events will be sent to the kafka service.
 #VGW_EVENT_KAFKA_URL=
 #VGW_EVENT_KAFKA_TOPIC=
 #VGW_EVENT_KAFKA_KEY=
 
 # Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
-# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the
-# the NATS messaging service. The gateway events are  similar to AWS S3 events,
-# and are documented in the wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
+# and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be
+# sent to the the NATS messaging service.
 #VGW_EVENT_NATS_URL=
 #VGW_EVENT_NATS_TOPIC=
 
+# Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is
+# specified, all configured bucket events will be sent to the webhook.
+#VGW_EVENT_WEBHOOK_URL=
+
+#######################
+# Debug / Diagnostics #
+#######################
+
 # The VGW_DEBUG option enables verbose debug log output to stdout. This output
 # includes details for signature verification steps. This is generally only
 # useful for debugging the S3 server, and should not be used in production.
 #VGW_DEBUG=false
 
+# The VGW_PPROF option enables the pprof HTTP server for profiling the S3
+# server. See the following for more information:
+# https://pkg.go.dev/net/http/pprof
+# To enable, set the VGW_PPROF option to the listening address for the pprof
+# server. For example, to listen on localhost port 6060, set the option to
+# "localhost:6060".
+#VGW_PPROF=
+
 ################
 # IAM services #
 ################
@@ -216,20 +243,30 @@ ROOT_SECRET_ACCESS_KEY=
 # below the "bucket directory" are treated as the objects. The object
 # name is split on "/" separator to translate to posix storage.
 # For example:
-# top level: /mnt/fs/gwroot
+# top level (VGW_BACKEND_ARG): /mnt/fs/gwroot
 # bucket: mybucket
 # object: a/b/c/myobject
 # will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
 
-# There are currently no further options other than VGW_BACKEND_ARG for the
-# posix backend.
+# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
+# change the ownership of newly created files and directories to the IAM
+# account UID/GID.
+#VGW_CHOWN_UID=false
+#VGW_CHOWN_GID=false
+
+# The VGW_READ_ONLY option will disable all write operations to the backend
+# filesystem. This is useful for creating a read-only gateway for clients.
+# This will prevent any PUT, POST, DELETE, and multipart upload operations
+# for objects and buckets as well as preventing updating metadata for objects.
+#VGW_READ_ONLY=false
 
 ###########
 # scoutfs #
 ###########
 
 # The scoutfs backend requires a ScoutFS filesystem type for the backend
-# path. The glacier mode functionality requires ScoutAM to be configured
+# path. The object to posix name mappings follow the same rules as posix for
+# scoutfs. The glacier mode functionality requires ScoutAM to be configured
 # for tiering data from the ScoutFS filesystem to a mass stroage system.
 # The mass storage system is often one or more tape libraries. Due to the
 # high latency of tape, the glacier mode functionality is designed to
@@ -248,6 +285,18 @@ ROOT_SECRET_ACCESS_KEY=
 # RestoreObject: add batch stage request to file
 #VGW_SCOUTFS_GLACIER=false
 
+# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
+# change the ownership of newly created files and directories to the IAM
+# account UID/GID.
+#VGW_CHOWN_UID=false
+#VGW_CHOWN_GID=false
+
+# The VGW_READ_ONLY option will disable all write operations to the backend
+# filesystem. This is useful for creating a read-only gateway for clients.
+# This will prevent any PUT, POST, DELETE, and multipart upload operations
+# for objects and buckets as well as preventing updating metadata for objects.
+#VGW_READ_ONLY=false
+
 ######
 # s3 #
 ######

go.mod

@@ -3,68 +3,68 @@ module github.com/versity/versitygw
 go 1.21
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
-	github.com/aws/aws-sdk-go-v2 v1.26.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
-	github.com/aws/smithy-go v1.20.1
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/aws/smithy-go v1.20.2
 	github.com/go-ldap/ldap/v3 v3.4.6
-	github.com/gofiber/fiber/v2 v2.52.3
+	github.com/gofiber/fiber/v2 v2.52.4
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.34.0
+	github.com/nats-io/nats.go v1.34.1
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/urfave/cli/v2 v2.27.1
 	github.com/valyala/fasthttp v1.52.0
-	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.18.0
+	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
+	golang.org/x/sys v0.19.0
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.9
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.9
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 )

go.sum

@@ -1,5 +1,5 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
@@ -10,52 +10,52 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
-github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
-github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg=
-github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 h1:F+PUZee9mlfpEJVZdgyewRumKekS9O3fftj8fEMt0rQ=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13/go.mod h1:Rl7i2dEWGHGsBIJCpUxlRt7VwK/HyXxICxdvIRssQHE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48=
+github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
+github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
+github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
+github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
-github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
-github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -65,10 +65,10 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
 github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
-github.com/gofiber/fiber/v2 v2.52.3 h1:bgAZwPv0aHIfRwIUdkWhg6U8D3MEYnoJjT+HfW/dDTo=
-github.com/gofiber/fiber/v2 v2.52.3/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
-github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
-github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
+github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -79,8 +79,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
-github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -90,15 +90,15 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk=
-github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
+github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
-github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
@@ -106,8 +106,8 @@ github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6kt
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
@@ -126,23 +126,23 @@ github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7g
 github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
-github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
-github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
+github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
+github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -151,8 +151,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -169,8 +169,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

s3api/controllers/base_test.go

@@ -77,7 +77,8 @@ func TestNew(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := New(tt.args.be, tt.args.iam, nil, nil); !reflect.DeepEqual(got, tt.want) {
+			got := New(tt.args.be, tt.args.iam, nil, nil, false)
+			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("New() = %v, want %v", got, tt.want)
 			}
 		})

s3api/middlewares/acl-parser.go

@@ -54,7 +54,6 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 			}
 			return ctx.Next()
 		}
-		//TODO: provide correct action names for the logger, after implementing DetectAction middleware
 		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
 		if err != nil {
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})

s3api/router.go

@@ -27,8 +27,8 @@ type S3ApiRouter struct {
 	WithAdmSrv bool
 }
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
-	s3ApiController := controllers.New(be, iam, logger, evs)
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
+	s3ApiController := controllers.New(be, iam, logger, evs, debug)
 
 	if sa.WithAdmSrv {
 		adminController := controllers.NewAdminController(iam, be)

s3api/router_test.go

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
 		})
 	}
 }

s3api/server.go

@@ -70,7 +70,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l))
 
-	server.router.Init(app, be, iam, l, evs)
+	server.router.Init(app, be, iam, l, evs, server.debug)
 
 	return server, nil
 }

s3err/s3err.go

@@ -116,6 +116,7 @@ const (
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
+	ErrQuotaExceeded
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -414,6 +415,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Directory object contains data payload.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrQuotaExceeded: {
+		Code:           "QuotaExceeded",
+		Description:    "Your request was denied due to quota exceeded.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3event/event.go

@@ -15,13 +15,18 @@
 package s3event
 
 import (
+	"encoding/json"
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 )
 
 type S3EventSender interface {
 	SendEvent(ctx *fiber.Ctx, meta EventMeta)
+	Close() error
 }
 
 type EventMeta struct {
@@ -36,22 +41,6 @@ type EventFields struct {
 	Records []EventSchema
 }
 
-type EventType string
-
-const (
-	EventObjectPut               EventType = "s3:ObjectCreated:Put"
-	EventObjectCopy              EventType = "s3:ObjectCreated:Copy"
-	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
-	EventObjectDelete            EventType = "s3:ObjectRemoved:Delete"
-	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
-	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
-	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
-	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
-	// Not supported
-	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
-	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
-)
-
 type EventSchema struct {
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
@@ -78,9 +67,18 @@ type EventResponseElements struct {
 	HostId    string `json:"x-amz-id-2"`
 }
 
+type ConfigurationId string
+
+// This field will be changed after implementing per bucket notifications
+const (
+	ConfigurationIdKafka   ConfigurationId = "kafka-global"
+	ConfigurationIdNats    ConfigurationId = "nats-global"
+	ConfigurationIdWebhook ConfigurationId = "webhook-global"
+)
+
 type EventS3Data struct {
 	S3SchemaVersion string            `json:"s3SchemaVersion"`
-	ConfigurationId string            `json:"configurationId"`
+	ConfigurationId ConfigurationId   `json:"configurationId"`
 	Bucket          EventS3BucketData `json:"bucket"`
 	Object          EventObjectData   `json:"object"`
 }
@@ -109,22 +107,95 @@ type EventObjectData struct {
 }
 
 type EventConfig struct {
-	KafkaURL      string
-	KafkaTopic    string
-	KafkaTopicKey string
-	NatsURL       string
-	NatsTopic     string
+	KafkaURL             string
+	KafkaTopic           string
+	KafkaTopicKey        string
+	NatsURL              string
+	NatsTopic            string
+	WebhookURL           string
+	FilterConfigFilePath string
 }
 
 func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
-	if cfg.KafkaURL != "" && cfg.NatsURL != "" {
-		return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
+	filter, err := parseEventFilters(cfg.FilterConfigFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("parse event filter config file %w", err)
 	}
-	if cfg.NatsURL != "" {
-		return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
+	var evSender S3EventSender
+	switch {
+	case cfg.WebhookURL != "":
+		evSender, err = InitWebhookEventSender(cfg.WebhookURL, filter)
+		fmt.Printf("initializing S3 Event Notifications with webhook URL %v\n", cfg.WebhookURL)
+	case cfg.KafkaURL != "":
+		evSender, err = InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey, filter)
+		fmt.Printf("initializing S3 Event Notifications with kafka. URL: %v, topic: %v\n", cfg.WebhookURL, cfg.KafkaTopic)
+	case cfg.NatsURL != "":
+		evSender, err = InitNatsEventService(cfg.NatsURL, cfg.NatsTopic, filter)
+		fmt.Printf("initializing S3 Event Notifications with Nats. URL: %v, topic: %v\n", cfg.NatsURL, cfg.NatsTopic)
+	default:
+		return nil, nil
 	}
-	if cfg.KafkaURL != "" {
-		return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
-	}
-	return nil, nil
+
+	return evSender, err
+}
+
+func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) ([]byte, error) {
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+	acc := ctx.Locals("account").(auth.Account)
+
+	event := []EventSchema{
+		{
+			EventVersion: "2.2",
+			EventSource:  "aws:s3",
+			AwsRegion:    ctx.Locals("region").(string),
+			EventTime:    time.Now().Format(time.RFC3339),
+			EventName:    meta.EventName,
+			UserIdentity: EventUserIdentity{
+				PrincipalId: acc.Access,
+			},
+			RequestParameters: EventRequestParams{
+				SourceIPAddress: ctx.IP(),
+			},
+			ResponseElements: EventResponseElements{
+				RequestId: ctx.Get("X-Amz-Request-Id"),
+				HostId:    ctx.Get("X-Amz-Id-2"),
+			},
+			S3: EventS3Data{
+				S3SchemaVersion: "1.0",
+				ConfigurationId: configId,
+				Bucket: EventS3BucketData{
+					Name: bucket,
+					OwnerIdentity: EventUserIdentity{
+						PrincipalId: meta.BucketOwner,
+					},
+					Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+				},
+				Object: EventObjectData{
+					Key:       object,
+					Size:      meta.ObjectSize,
+					ETag:      meta.ObjectETag,
+					VersionId: meta.VersionId,
+					Sequencer: genSequencer(),
+				},
+			},
+			GlacierEventData: EventGlacierData{
+				// Not supported
+				RestoreEventData: EventRestoreData{},
+			},
+		},
+	}
+
+	return json.Marshal(event)
+}
+
+func generateTestEvent() ([]byte, error) {
+	msg := map[string]string{
+		"Service": "S3",
+		"Event":   "s3:TestEvent",
+		"Time":    time.Now().Format(time.RFC3339),
+		"Bucket":  "Test-Bucket",
+	}
+
+	return json.Marshal(msg)
 }

s3event/filter.go

@@ -0,0 +1,122 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type EventType string
+
+const (
+	EventObjectCreated           EventType = "s3:ObjectCreated:*" // ObjectCreated
+	EventObjectCreatedPut        EventType = "s3:ObjectCreated:Put"
+	EventObjectCreatedPost       EventType = "s3:ObjectCreated:Post"
+	EventObjectCreatedCopy       EventType = "s3:ObjectCreated:Copy"
+	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
+	EventObjectDeleted           EventType = "s3:ObjectRemoved:Delete" // ObjectRemoved
+	EventObjectTagging           EventType = "s3:ObjectTagging:*"      // ObjectTagging
+	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
+	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
+	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
+	EventObjectRestore           EventType = "s3:ObjectRestore:*" // ObjectRestore
+	EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
+	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
+)
+
+func (event EventType) IsValid() bool {
+	_, ok := supportedEventFilters[event]
+	return ok
+}
+
+var supportedEventFilters = map[EventType]struct{}{
+	EventObjectCreated:           {},
+	EventObjectCreatedPut:        {},
+	EventObjectCreatedPost:       {},
+	EventObjectCreatedCopy:       {},
+	EventCompleteMultipartUpload: {},
+	EventObjectDeleted:           {},
+	EventObjectTagging:           {},
+	EventObjectTaggingPut:        {},
+	EventObjectTaggingDelete:     {},
+	EventObjectAclPut:            {},
+	EventObjectRestore:           {},
+	EventObjectRestorePost:       {},
+	EventObjectRestoreCompleted:  {},
+}
+
+type EventFilter map[EventType]bool
+
+func parseEventFilters(path string) (EventFilter, error) {
+	// if no filter config file path is specified return nil map
+	if path == "" {
+		return nil, nil
+	}
+
+	configFilePath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Open the JSON file
+	file, err := os.Open(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var filter EventFilter
+	if err := json.NewDecoder(file).Decode(&filter); err != nil {
+		return nil, err
+	}
+
+	if err := filter.Validate(); err != nil {
+		return nil, err
+	}
+
+	return filter, nil
+}
+
+func (ef EventFilter) Validate() error {
+	for event := range ef {
+		if isValid := event.IsValid(); !isValid {
+			return fmt.Errorf("invalid configuration property: %v", event)
+		}
+	}
+
+	return nil
+}
+
+func (ef EventFilter) Filter(event EventType) bool {
+	ev, found := ef[event]
+	if found {
+		return ev
+	}
+
+	// check wildcard match
+	wildCardEv := EventType(string(event[strings.LastIndex(string(event), ":")+1]) + "*")
+	wildcard, found := ef[wildCardEv]
+	if found {
+		return wildcard
+	}
+
+	return false
+}

s3event/kafka.go

@@ -16,10 +16,8 @@ package s3event
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -32,10 +30,11 @@ var sequencer = 0
 type Kafka struct {
 	key    string
 	writer *kafka.Writer
+	filter EventFilter
 	mu     sync.Mutex
 }
 
-func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
+func InitKafkaEventService(url, topic, key string, filter EventFilter) (S3EventSender, error) {
 	if topic == "" {
 		return nil, fmt.Errorf("kafka message topic should be specified")
 	}
@@ -47,26 +46,19 @@ func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
 		BatchTimeout: 5 * time.Millisecond,
 	})
 
-	msg := map[string]string{
-		"Service": "S3",
-		"Event":   "s3:TestEvent",
-		"Time":    time.Now().Format(time.RFC3339),
-		"Bucket":  "Test-Bucket",
-	}
-
-	msgJSON, err := json.Marshal(msg)
+	msg, err := generateTestEvent()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("kafka generate test event: %w", err)
 	}
 
 	message := kafka.Message{
 		Key:   []byte(key),
-		Value: msgJSON,
+		Value: msg,
 	}
 
-	ctx := context.Background()
-
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	err = w.WriteMessages(ctx, message)
+	cancel()
 	if err != nil {
 		return nil, err
 	}
@@ -74,6 +66,7 @@ func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
 	return &Kafka{
 		key:    key,
 		writer: w,
+		filter: filter,
 	}, nil
 }
 
@@ -81,67 +74,31 @@ func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 	ks.mu.Lock()
 	defer ks.mu.Unlock()
 
-	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
-
-	schema := EventSchema{
-		EventVersion: "2.2",
-		EventSource:  "aws:s3",
-		AwsRegion:    ctx.Locals("region").(string),
-		EventTime:    time.Now().Format(time.RFC3339),
-		EventName:    meta.EventName,
-		UserIdentity: EventUserIdentity{
-			PrincipalId: ctx.Locals("access").(string),
-		},
-		RequestParameters: EventRequestParams{
-			SourceIPAddress: ctx.IP(),
-		},
-		ResponseElements: EventResponseElements{
-			RequestId: ctx.Get("X-Amz-Request-Id"),
-			HostId:    ctx.Get("X-Amx-Id-2"),
-		},
-		S3: EventS3Data{
-			S3SchemaVersion: "1.0",
-			// This field will come up after implementing per bucket notifications
-			ConfigurationId: "kafka-global",
-			Bucket: EventS3BucketData{
-				Name: bucket,
-				OwnerIdentity: EventUserIdentity{
-					PrincipalId: ctx.Locals("access").(string),
-				},
-				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
-			},
-			Object: EventObjectData{
-				Key:       object,
-				Size:      meta.ObjectSize,
-				ETag:      meta.ObjectETag,
-				VersionId: meta.VersionId,
-				Sequencer: genSequencer(),
-			},
-		},
-		GlacierEventData: EventGlacierData{
-			// Not supported
-			RestoreEventData: EventRestoreData{},
-		},
-	}
-
-	ks.send([]EventSchema{schema})
-}
-
-func (ks *Kafka) send(evnt []EventSchema) {
-	msg, err := json.Marshal(evnt)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
+	if ks.filter != nil && !ks.filter.Filter(meta.EventName) {
 		return
 	}
 
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdKafka)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create kafka event: %v\n", err.Error())
+		return
+	}
+
+	go ks.send(schema)
+}
+
+func (ks *Kafka) Close() error {
+	return ks.writer.Close()
+}
+
+func (ks *Kafka) send(event []byte) {
 	message := kafka.Message{
 		Key:   []byte(ks.key),
-		Value: msg,
+		Value: event,
 	}
 
 	ctx := context.Background()
-	err = ks.writer.WriteMessages(ctx, message)
+	err := ks.writer.WriteMessages(ctx, message)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
 	}

s3event/nats.go

@@ -15,12 +15,9 @@
 package s3event
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
-	"time"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/nats-io/nats.go"
@@ -30,9 +27,10 @@ type NatsEventSender struct {
 	topic  string
 	client *nats.Conn
 	mu     sync.Mutex
+	filter EventFilter
 }
 
-func InitNatsEventService(url, topic string) (S3EventSender, error) {
+func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {
 	if topic == "" {
 		return nil, fmt.Errorf("nats message topic should be specified")
 	}
@@ -42,9 +40,20 @@ func InitNatsEventService(url, topic string) (S3EventSender, error) {
 		return nil, err
 	}
 
+	msg, err := generateTestEvent()
+	if err != nil {
+		return nil, fmt.Errorf("nats generate test event: %w", err)
+	}
+
+	err = client.Publish(topic, msg)
+	if err != nil {
+		return nil, fmt.Errorf("nats publish test event: %v", err)
+	}
+
 	return &NatsEventSender{
 		topic:  topic,
 		client: client,
+		filter: filter,
 	}, nil
 }
 
@@ -52,60 +61,26 @@ func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 	ns.mu.Lock()
 	defer ns.mu.Unlock()
 
-	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
-
-	schema := EventSchema{
-		EventVersion: "2.2",
-		EventSource:  "aws:s3",
-		AwsRegion:    ctx.Locals("region").(string),
-		EventTime:    time.Now().Format(time.RFC3339),
-		EventName:    meta.EventName,
-		UserIdentity: EventUserIdentity{
-			PrincipalId: ctx.Locals("access").(string),
-		},
-		RequestParameters: EventRequestParams{
-			SourceIPAddress: ctx.IP(),
-		},
-		ResponseElements: EventResponseElements{
-			RequestId: ctx.Get("X-Amz-Request-Id"),
-			HostId:    ctx.Get("X-Amx-Id-2"),
-		},
-		S3: EventS3Data{
-			S3SchemaVersion: "1.0",
-			// This field will come up after implementing per bucket notifications
-			ConfigurationId: "nats-global",
-			Bucket: EventS3BucketData{
-				Name: bucket,
-				OwnerIdentity: EventUserIdentity{
-					PrincipalId: ctx.Locals("access").(string),
-				},
-				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
-			},
-			Object: EventObjectData{
-				Key:       object,
-				Size:      meta.ObjectSize,
-				ETag:      meta.ObjectETag,
-				VersionId: meta.VersionId,
-				Sequencer: genSequencer(),
-			},
-		},
-		GlacierEventData: EventGlacierData{
-			// Not supported
-			RestoreEventData: EventRestoreData{},
-		},
+	if ns.filter != nil && !ns.filter.Filter(meta.EventName) {
+		return
 	}
 
-	ns.send([]EventSchema{schema})
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdNats)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create nats event: %v\n", err.Error())
+		return
+	}
+
+	go ns.send(schema)
 }
 
-func (ns *NatsEventSender) send(evnt []EventSchema) {
-	msg, err := json.Marshal(evnt)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
-	}
+func (ns *NatsEventSender) Close() error {
+	ns.client.Close()
+	return nil
+}
 
-	err = ns.client.Publish(ns.topic, msg)
+func (ns *NatsEventSender) send(event []byte) {
+	err := ns.client.Publish(ns.topic, event)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
 	}

s3event/webhook.go

@@ -0,0 +1,108 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type Webhook struct {
+	url    string
+	client *http.Client
+	filter EventFilter
+	mu     sync.Mutex
+}
+
+func InitWebhookEventSender(url string, filter EventFilter) (S3EventSender, error) {
+	if url == "" {
+		return nil, fmt.Errorf("webhook url should be specified")
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 1,
+	}
+
+	testEv, err := generateTestEvent()
+	if err != nil {
+		return nil, fmt.Errorf("webhook generate test event: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(testEv))
+	if err != nil {
+		return nil, fmt.Errorf("create webhook http request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	_, err = client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			return nil, fmt.Errorf("send webhook test event: %w", err)
+		}
+	}
+
+	return &Webhook{
+		client: &http.Client{
+			Timeout: 3 * time.Second,
+		},
+		url:    url,
+		filter: filter,
+	}, nil
+}
+
+func (w *Webhook) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.filter != nil && !w.filter.Filter(meta.EventName) {
+		return
+	}
+
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create webhook event: %v\n", err.Error())
+		return
+	}
+
+	go w.send(schema)
+}
+
+func (w *Webhook) Close() error {
+	return nil
+}
+
+func (w *Webhook) send(event []byte) {
+	req, err := http.NewRequest(http.MethodPost, w.url, bytes.NewReader(event))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create webhook event request: %v\n", err.Error())
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	_, err = w.client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			fmt.Fprintf(os.Stderr, "failed to send webhook event: %v\n", err.Error())
+		}
+	}
+}

s3log/file.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -88,9 +89,9 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 		}
 	}
 
-	switch ctx.Locals("access").(type) {
-	case string:
-		access = ctx.Locals("access").(string)
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner

s3log/webhook.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -85,9 +86,9 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 		}
 	}
 
-	switch ctx.Locals("access").(type) {
-	case string:
-		access = ctx.Locals("access").(string)
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner

tests/.env.default

@@ -11,4 +11,6 @@ CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
 S3CMD_CONFIG=./tests/s3cfg.local.default
 SECRETS_FILE=./tests/.secrets
-MC_ALIAS=versity
+MC_ALIAS=versity
+LOG_LEVEL=2
+GOCOVERDIR=$PWD/cover

tests/.env.s3.default

@@ -11,4 +11,4 @@ CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
 S3CMD_CONFIG=./tests/s3cfg.local.default
 SECRETS_FILE=./tests/.secrets.s3
-MC_ALIAS=versity
+MC_ALIAS=versity_s3

tests/README.md

@@ -2,6 +2,8 @@
 
 ## Instructions - Running Locally
 
+### Posix Backend
+
 1. Build the `versitygw` binary.
 2. Install the command-line interface(s) you want to test if unavailable on your machine.  
    * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
@@ -28,7 +30,18 @@
 8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
 9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
+### S3 Backend
+
+Instructions are mostly the same; however, testing with the S3 backend requires two S3 accounts.  Ideally, these are two real accounts, but one can also be a dummy account that versity uses internally.
+
+To set up the latter:
+1. Create a new AWS profile with ID and key values set to dummy 20-char allcaps and 40-char alphabetical values respectively.
+1. In the `.secrets` file being used, create the fields `AWS_ACCESS_KEY_ID_TWO` and `AWS_SECRET_ACCESS_KEY_TWO`.  Set these values to the actual AWS ID and key.  
+2. Set the values for `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` the same dummy values set in the AWS profile, and set `AWS_PROFILE` to the profile you just created.
+3. Create a new AWS profile with these dummy values.  In the `.env` file being used, set the `AWS_PROFILE` parameter to the name of this new profile, and the ID and key fields to the dummy values.  
+4. Set `BACKEND` to `s3`.  Also, change the `MC_ALIAS` value if testing **mc** in this configuration.
+
 ## Instructions - Running With Docker
 
 1.  Create a `.secrets` file in the `tests` folder, and add the `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and the `AWS_PROFILE` fields.
-2.  Build and run the `Dockerfile_test_bats` file.
+2.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to an S3-backend-friendly config.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.

tests/integration/group-tests.go

@@ -309,6 +309,7 @@ func TestFullFlow(s *S3Conf) {
 	TestPutBucketPolicy(s)
 	TestGetBucketPolicy(s)
 	TestDeleteBucketPolicy(s)
+	TestAccessControl(s)
 }
 
 func TestPosix(s *S3Conf) {
@@ -325,6 +326,16 @@ func TestIAM(s *S3Conf) {
 	IAM_admin_ChangeBucketOwner(s)
 }
 
+func TestAccessControl(s *S3Conf) {
+	AccessControl_default_ACL_user_access_denied(s)
+	AccessControl_default_ACL_userplus_access_denied(s)
+	AccessControl_default_ACL_admin_successful_access(s)
+	AccessControl_bucket_resource_single_action(s)
+	AccessControl_bucket_resource_all_action(s)
+	AccessControl_single_object_resource_actions(s)
+	AccessControl_multi_statement_policy(s)
+}
+
 type IntTests map[string]func(s *S3Conf) error
 
 func GetIntTests() IntTests {

tests/integration/tests.go

@@ -5797,6 +5797,320 @@ func DeleteBucketPolicy_success(s *S3Conf) error {
 	})
 }
 
+// Access control tests (with bucket ACLs and Policies)
+func AccessControl_default_ACL_user_access_denied(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_user_access_denied"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = usr.access
+		cfg.awsSecret = usr.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_default_ACL_userplus_access_denied(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_userplus_access_denied"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr := user{
+			access: "userplus1",
+			secret: "userplus1secret",
+			role:   "userplus",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = usr.access
+		cfg.awsSecret = usr.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_default_ACL_admin_successful_access(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_admin_successful_access"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		admin := user{
+			access: "admin1",
+			secret: "admin1secret",
+			role:   "admin",
+		}
+		err := createUsers(s, []user{admin})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = admin.access
+		cfg.awsSecret = admin.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_bucket_resource_single_action(s *S3Conf) error {
+	testName := "AccessControl_bucket_resource_single_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		usr2 := user{
+			access: "grt2",
+			secret: "grt2secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr1, usr2})
+		if err != nil {
+			return err
+		}
+
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:PutBucketTagging"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.DeleteBucketTagging(ctx, &s3.DeleteBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		user2Client := getUserS3Client(usr2, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user2Client.DeleteBucketTagging(ctx, &s3.DeleteBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_bucket_resource_all_action(s *S3Conf) error {
+	testName := "AccessControl_bucket_resource_all_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		usr2 := user{
+			access: "grt2",
+			secret: "grt2secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr1, usr2})
+		if err != nil {
+			return err
+		}
+
+		bucketResource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		objectResource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+		err = putObjects(user1Client, []string{"my-obj"}, bucket)
+		if err != nil {
+			return err
+		}
+
+		user2Client := getUserS3Client(usr2, s)
+
+		err = putObjects(user2Client, []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_single_object_resource_actions(s *S3Conf) error {
+	testName := "AccessControl_single_object_resource_actions"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		obj := "my-obj/nested-obj"
+		err := putObjects(s3client, []string{obj}, bucket)
+		if err != nil {
+			return err
+		}
+
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err = createUsers(s, []user{usr1})
+		if err != nil {
+			return err
+		}
+
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%v/%v"`, bucket, obj))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: &bucket,
+			Key:    &obj,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_multi_statement_policy(s *S3Conf) error {
+	testName := "AccessControl_multi_statement_policy"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		policy := fmt.Sprintf(`
+		{
+			"Statement": [
+				{
+					"Effect": "Deny",
+					"Principal": ["grt1"],
+					"Action":  "s3:DeleteBucket",
+					"Resource":  "arn:aws:s3:::%s"
+				},
+				{
+					"Effect": "Allow",
+					"Principal": "grt1",
+					"Action": "s3:*",
+					"Resource": ["arn:aws:s3:::%s", "arn:aws:s3:::%s/*"]
+				}
+			]
+		}	
+		`, bucket, bucket, bucket)
+
+		usr := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &policy,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		userClient := getUserS3Client(usr, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = userClient.ListObjects(ctx, &s3.ListObjectsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = userClient.DeleteBucket(ctx, &s3.DeleteBucketInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 // IAM related tests
 // multi-user iam tests
 func IAM_user_access_denied(s *S3Conf) error {
@@ -5808,13 +6122,8 @@ func IAM_user_access_denied(s *S3Conf) error {
 		secret: "grt1secret",
 		role:   "user",
 	}
-	err := deleteUser(s, usr.access)
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
 
-	err = createUsers(s, []user{usr})
+	err := createUsers(s, []user{usr})
 	if err != nil {
 		failF("%v: %v", testName, err)
 		return fmt.Errorf("%v: %w", testName, err)
@@ -5844,13 +6153,8 @@ func IAM_userplus_access_denied(s *S3Conf) error {
 		secret: "grt1secret",
 		role:   "userplus",
 	}
-	err := deleteUser(s, usr.access)
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
 
-	err = createUsers(s, []user{usr})
+	err := createUsers(s, []user{usr})
 	if err != nil {
 		failF("%v: %v", testName, err)
 		return fmt.Errorf("%v: %w", testName, err)
@@ -5879,12 +6183,8 @@ func IAM_userplus_CreateBucket(s *S3Conf) error {
 			secret: "grt1secret",
 			role:   "userplus",
 		}
-		err := deleteUser(s, usr.access)
-		if err != nil {
-			return err
-		}
 
-		err = createUsers(s, []user{usr})
+		err := createUsers(s, []user{usr})
 		if err != nil {
 			return err
 		}

tests/integration/utils.go

@@ -537,6 +537,10 @@ type user struct {
 
 func createUsers(s *S3Conf, users []user) error {
 	for _, usr := range users {
+		err := deleteUser(s, usr.access)
+		if err != nil {
+			return err
+		}
 		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
 		if err != nil {
 			return err
@@ -633,3 +637,11 @@ func getMalformedPolicyError(msg string) s3err.APIError {
 		HTTPStatusCode: http.StatusBadRequest,
 	}
 }
+
+func getUserS3Client(usr user, cfg *S3Conf) *s3.Client {
+	config := *cfg
+	config.awsID = usr.access
+	config.awsSecret = usr.secret
+
+	return s3.NewFromConfig(config.Config())
+}

tests/logger.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# levels:  1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace
+
+log() {
+  if [[ $# -ne 2 ]]; then
+    echo "log function requires level, message"
+    return 1
+  fi
+  if [[ $1 -gt $LOG_LEVEL ]]; then
+    return 0
+  fi
+  echo "$2"
+}

tests/setup.sh

@@ -54,6 +54,11 @@ check_params() {
     echo "RECREATE_BUCKETS must be 'true' or 'false'"
     return 1
   fi
+  if [[ -z "$LOG_LEVEL" ]]; then
+    export LOG_LEVEL=2
+  else
+    export LOG_LEVEL
+  fi
   return 0
 }
 

tests/test_aws.sh

@@ -10,6 +10,19 @@ source ./tests/test_common.sh
   test_common_create_delete_bucket "aws"
 }
 
+@test "test_create_bucket_invalid_name" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "aws" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"Invalid bucket name "* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
 # test adding and removing an object on versitygw
 @test "test_put_object-with-data" {
   test_common_put_object_with_data "aws"
@@ -300,8 +313,8 @@ source ./tests/test_common.sh
   bucket_file_data="test file\n"
 
   create_test_files "$bucket_file" || local created=$?
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
   [[ $created -eq 0 ]] || fail "Error creating test files"
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
@@ -319,3 +332,88 @@ source ./tests/test_common.sh
 @test "test-presigned-url-utf8-chars" {
   test_common_presigned_url_utf8_chars "aws"
 }
+
+@test "test-list-objects-delimiter" {
+  folder_name="two"
+  object_name="three"
+  create_test_folder "$folder_name" || local created=$?
+  [[ $created -eq 0 ]] || fail "error creating folder"
+  create_test_files "$folder_name"/"$object_name" || created=$?
+  [[ $created -eq 0 ]] || fail "error creating file"
+
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  put_object "aws" "$test_file_folder"/"$folder_name"/"$object_name" "$BUCKET_ONE_NAME"/"$folder_name"/"$object_name" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"
+  prefix=$(echo "${objects[@]}" | jq ".CommonPrefixes[0].Prefix")
+  [[ $prefix == "\""$folder_name/"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"
+  key=$(echo "${objects[@]}" | jq ".Contents[0].Key")
+  [[ $key == "\""$folder_name/$object_name"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $folder_name
+}
+
+# ensure that lists of files greater than a size of 1000 (pagination) are returned properly
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "aws"
+}
+
+#@test "test_filename_length" {
+#  file_name=$(printf "%0.sa" $(seq 1 1025))
+#  echo "$file_name"
+
+#  create_test_files "$file_name" || created=$?
+#  [[ $created -eq 0 ]] || fail "error creating file"
+
+#  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+#  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+#  put_object "aws" "$test_file_folder"/"$file_name" "$BUCKET_ONE_NAME"/"$file_name" || local put_object=$?
+#  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+#}
+
+@test "test_head_bucket" {
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "aws" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_head_bucket_doesnt_exist" {
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_copy_object_aws" {
+
+  bucket_file="bucket_file"
+
+  create_test_files "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  setup_bucket "aws" "$BUCKET_TWO_NAME" || local setup_result_two=$?
+  [[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error"
+  put_object "aws" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  error=$(aws --no-verify-ssl s3api copy-object --copy-source "$BUCKET_ONE_NAME"/"$bucket_file" --key "$bucket_file" --bucket "$BUCKET_TWO_NAME" 2>&1) || local copy_result=$?
+  [[ $copy_result -eq 0 ]] || fail "Error copying file: $error"
+  copy_file "s3://$BUCKET_TWO_NAME"/"$bucket_file" "$test_file_folder/${bucket_file}_copy" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  compare_files "$test_file_folder/$bucket_file" "$test_file_folder/${bucket_file}_copy" || local compare_result=$?
+  [[ $compare_result -eq 0 ]] || file "files don't match"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "aws" "$BUCKET_TWO_NAME"
+  delete_test_files "$bucket_file"
+}

tests/test_common.sh

@@ -148,7 +148,6 @@ test_common_list_objects() {
 }
 
 test_common_set_get_bucket_tags() {
-
   if [[ $# -ne 1 ]]; then
     fail "set/get bucket tags test requires command type"
   fi
@@ -192,7 +191,6 @@ test_common_set_get_bucket_tags() {
 }
 
 test_common_set_get_object_tags() {
-
   if [[ $# -ne 1 ]]; then
     echo "get/set object tags missing command type"
     return 1
@@ -293,3 +291,23 @@ test_common_presigned_url_utf8_chars() {
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
   delete_test_files "$bucket_file" "$bucket_file_copy"
 }
+
+test_common_list_objects_file_count() {
+  if [[ $# -ne 1 ]]; then
+    echo "list objects greater than 1000 missing command type"
+    return 1
+  fi
+  create_test_file_count 1001 || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "error creating test files"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  put_object_multiple "$1" "$test_file_folder/file_*" "$BUCKET_ONE_NAME" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "Failed to copy files to bucket"
+  list_objects "$1" "$BUCKET_ONE_NAME"
+  if [[ $LOG_LEVEL -ge 5 ]]; then
+    log 5 "Array: ${object_array[*]}"
+  fi
+  local file_count="${#object_array[@]}"
+  [[ $file_count == 1001 ]] || fail "file count should be 1001, is $file_count"
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+}

tests/test_mc.sh

@@ -41,3 +41,37 @@ export RUN_MC=true
 @test "test_presigned_url_utf8_chars_mc" {
   test_common_presigned_url_utf8_chars "mc"
 }
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "mc"
+}
+
+@test "test_create_bucket_invalid_name_mc" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "mc" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"Bucket name cannot be empty"* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_mc" {
+  setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "mc" "$BUCKET_ONE_NAME"
+  [[ $bucket_info == *"$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info"
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_doesnt_exist_mc" {
+  setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "mc" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"does not exist"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}

tests/test_s3cmd.sh

@@ -35,4 +35,38 @@ export RUN_S3CMD=true
 
 #@test "test_presigned_url_utf8_chars_s3cmd" {
 #  test_common_presigned_url_utf8_chars "s3cmd"
-#}
+#}
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3cmd"
+}
+
+@test "test_create_bucket_invalid_name_s3cmd" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "s3cmd" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"just the bucket name"* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_s3cmd" {
+  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "s3cmd" "$BUCKET_ONE_NAME"
+  [[ $bucket_info == *"s3://$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info"
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_doesnt_exist_s3cmd" {
+  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "s3cmd" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}

tests/util.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bats
 
 source ./tests/util_mc.sh
+source ./tests/logger.sh
 
 # create an AWS bucket
 # param:  bucket name
@@ -30,6 +31,29 @@ create_bucket() {
   return 0
 }
 
+create_bucket_invalid_name() {
+  if [ $# -ne 1 ]; then
+    echo "create bucket w/invalid name missing command type"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == "aws" ]]; then
+    bucket_create_error=$(aws --no-verify-ssl s3 mb "s3://" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    bucket_create_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb "s3://" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    bucket_create_error=$(mc --insecure mb "$MC_ALIAS" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -eq 0 ]; then
+    echo "error:  bucket should have not been created but was"
+    return 1
+  fi
+  export bucket_create_error
+}
+
 # delete an AWS bucket
 # param:  bucket name
 # return 0 for success, 1 for failure
@@ -281,6 +305,35 @@ put_object() {
   return 0
 }
 
+put_object_multiple() {
+  if [ $# -ne 3 ]; then
+    echo "put object command requires command type, source, destination"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == 'aws' ]]; then
+    # shellcheck disable=SC2086
+    error=$(aws --no-verify-ssl s3 cp "$(dirname "$2")" s3://"$3" --recursive --exclude="*" --include="$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    # shellcheck disable=SC2086
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put $2 "s3://$3/" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    # shellcheck disable=SC2086
+    error=$(mc --insecure cp $2 "$MC_ALIAS"/"$3" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error copying object to bucket: $error"
+    return 1
+  else
+    log 5 "$error"
+  fi
+  return 0
+}
+
 # add object to versitygw if it doesn't exist
 # params:  source file, destination copy location
 # return 0 for success or already exists, 1 for failure
@@ -394,8 +447,10 @@ list_objects() {
 
   object_array=()
   while IFS= read -r line; do
-    object_name=$(echo "$line" | awk '{print $NF}')
-    object_array+=("$object_name")
+    if [[ $line != *InsecureRequestWarning* ]]; then
+      object_name=$(echo "$line" | awk '{print $NF}')
+      object_array+=("$object_name")
+    fi
   done <<< "$output"
 
   export object_array
@@ -600,11 +655,15 @@ get_object_tags() {
 # param:  bucket
 # export objects on success, return 1 for failure
 list_objects_s3api_v1() {
-  if [ $# -ne 1 ]; then
-    echo "list objects command missing bucket"
+  if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+    echo "list objects command requires bucket, (optional) delimiter"
     return 1
   fi
-  objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+  if [ "$2" == "" ]; then
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+  else
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
+  fi
   if [[ $result -ne 0 ]]; then
     echo "error listing objects: $objects"
     return 1
@@ -926,3 +985,27 @@ create_presigned_url() {
   fi
   export presigned_url
 }
+
+head_bucket() {
+  if [ $# -ne 2 ]; then
+    echo "head bucket command missing command type, bucket name"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == "aws" ]]; then
+    bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    bucket_info=$(s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error getting bucket info: $bucket_info"
+    return 1
+  fi
+  export bucket_info
+}

tests/util_file.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bats
 
+source ./tests/logger.sh
+
 # create a test file and export folder.  do so in temp folder
 # params:  filename
 # export test file folder on success, return 1 for error
@@ -8,7 +10,7 @@ create_test_files() {
     echo "create test files command missing filename"
     return 1
   fi
-  test_file_folder=.
+  test_file_folder=$PWD
   if [[ -z "$GITHUB_ACTIONS" ]]; then
     create_test_file_folder
   fi
@@ -21,6 +23,23 @@ create_test_files() {
   export test_file_folder
 }
 
+create_test_folder() {
+  if [ $# -lt 1 ]; then
+    echo "create test folder command missing folder name"
+    return 1
+  fi
+  test_file_folder=$PWD
+  if [[ -z "$GITHUB_ACTIONS" ]]; then
+    create_test_file_folder
+  fi
+  for name in "$@"; do
+    mkdir -p "$test_file_folder"/"$name" || local mkdir_result=$?
+    if [[ $mkdir_result -ne 0 ]]; then
+      echo "error creating file $name"
+    fi
+  done
+}
+
 # delete a test file
 # params:  filename
 # return:  0 for success, 1 for error
@@ -34,7 +53,7 @@ delete_test_files() {
     return 1
   fi
   for name in "$@"; do
-    rm "$test_file_folder"/"$name" || rm_result=$?
+    rm -rf "${test_file_folder:?}"/"${name:?}" || rm_result=$?
     if [[ $rm_result -ne 0 ]]; then
       echo "error deleting file $name"
     fi
@@ -80,7 +99,11 @@ compare_files() {
 }
 
 create_test_file_folder() {
-  test_file_folder=${TMPDIR}versity-gwtest
+  if [[ -n $TMPDIR ]]; then
+    test_file_folder=${TMPDIR}versity-gwtest
+  else
+    test_file_folder=$PWD/versity-gwtest
+  fi
   mkdir -p "$test_file_folder" || local mkdir_result=$?
   if [[ $mkdir_result -ne 0 ]]; then
     echo "error creating test file folder"
@@ -97,16 +120,40 @@ create_large_file() {
     return 1
   fi
 
-  test_file_folder=.
+  test_file_folder=$PWD
   if [[ -z "$GITHUB_ACTIONS" ]]; then
     create_test_file_folder
   fi
 
   filesize=$((160*1024*1024))
-  error=$(dd if=/dev/urandom of=$test_file_folder/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$?
+  error=$(dd if=/dev/urandom of="$test_file_folder"/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$?
   if [[ $dd_result -ne 0 ]]; then
     echo "error creating file: $error"
     return 1
   fi
   return 0
 }
+
+create_test_file_count() {
+  if [[ $# -ne 1 ]]; then
+    echo "create test file count function missing bucket name, count"
+    return 1
+  fi
+  test_file_folder=$PWD
+  if [[ -z "$GITHUB_ACTIONS" ]]; then
+    create_test_file_folder
+  fi
+  local touch_result
+  for ((i=1;i<=$1;i++)) {
+    error=$(touch "$test_file_folder/file_$i") || touch_result=$?
+    if [[ $touch_result -ne 0 ]]; then
+      echo "error creating file_$i:  $error"
+      return 1
+    fi
+  }
+  if [[ $LOG_LEVEL -ge 5 ]]; then
+    ls_result=$(ls "$test_file_folder"/file_*)
+    log 5 "$ls_result"
+  fi
+  return 0
+}

tests/util_mc.sh

@@ -21,4 +21,4 @@ delete_bucket_recursive_mc() {
     return 1
   fi
   return 0
-}
+}

tests/versity.sh

@@ -45,6 +45,9 @@ check_exe_params() {
     echo "RUN_VERSITYGW must be 'true' or 'false'"
     return 1
   fi
+  if [[ -r $GOCOVERDIR ]]; then
+    export GOCOVERDIR=$GOCOVERDIR
+  fi
   if [[ $RUN_VERSITYGW == "true" ]]; then
     local check_result
     check_exe_params_versity || check_result=$?