feat: implement read only option for posix/scoutfs

Fixes #484. When read only option selected, backend will return
AccessDenied for any operations that would update data or metadata
in the backend. This include creating and deleting objects and
buckets as well as updating object metadata.

This will still allow admin to change bucket owner since read only
is mainly intended for s3 api only.

.github/dependabot.yml

@@ -8,3 +8,7 @@ updates:
       dev-dependencies:
         patterns:
           - "*"
+    allow:
+      # Allow both direct and indirect updates for all packages
+      - dependency-type: "all"
+

.github/workflows/docker.yaml

@@ -12,7 +12,7 @@ jobs:
       packages: write
       contents: read
     steps:
-      - name: Check out the repo
+      - name: Checkout
         uses: actions/checkout@v4
 
       - name: Log in to Docker Hub
@@ -43,3 +43,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            VERSION=${{ github.event.release.tag_name }}
+            TIME=${{ github.event.release.published_at }}
+            BUILD=${{ github.sha }}

.github/workflows/functional.yml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go

.github/workflows/go.yml

@@ -8,10 +8,10 @@ jobs:
     steps:
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go

.github/workflows/goreleaser.yml

@@ -15,14 +15,21 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - run: git fetch --force --tags
-      - uses: actions/setup-go@v4
+
+      - name: Fetch tags
+        run: git fetch --force --tags
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
         with:
           go-version: stable
-      - uses: goreleaser/goreleaser-action@v4
+
+      - name: Run Releaser
+        uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser
           version: latest

.github/workflows/static.yml

@@ -7,16 +7,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 'stable'
       id: go
  
     - name: "staticcheck"
-      uses: dominikh/staticcheck-action@v1.3.0
-      with: 
-        install-go: false
+      uses: dominikh/staticcheck-action@v1
+      with:
+        version: "latest"

.github/workflows/system.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install ShellCheck
         run: sudo apt-get install shellcheck
@@ -15,7 +15,7 @@ jobs:
         run: shellcheck -S warning ./tests/*.sh
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 'stable'
         id: go
@@ -38,7 +38,7 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Build and run
+      - name: Build and run, posix backend
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -51,9 +51,23 @@ jobs:
           export WORKSPACE=$GITHUB_WORKSPACE
           openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir /tmp/cover
-          VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+          mkdir cover
+          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
+
+      #- name: Build and run, s3 backend
+      #  run: |
+      #    make testbin
+      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+      #    export AWS_REGION=us-east-1
+      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
+      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
+      #    aws configure set aws_region $AWS_REGION --profile versity_s3
+      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
+      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+      #    export WORKSPACE=$GITHUB_WORKSPACE
+      #    VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
 
       - name: Coverage report
         run: |
-          go tool covdata percent -i=/tmp/cover
+          go tool covdata percent -i=cover

Dockerfile

@@ -1,5 +1,15 @@
 FROM golang:latest
 
+# Set build arguments with default values
+ARG VERSION="none"
+ARG BUILD="none"
+ARG TIME="none"
+
+# Set environment variables
+ENV VERSION=${VERSION}
+ENV BUILD=${BUILD}
+ENV TIME=${TIME}
+
 WORKDIR /app
 
 COPY go.mod ./
@@ -9,7 +19,7 @@ COPY ./ ./
 
 WORKDIR /app/cmd/versitygw
 ENV CGO_ENABLED=0
-RUN go build -o versitygw
+RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
 
 FROM alpine:latest
 

Dockerfile_test_bats

@@ -1,6 +1,9 @@
 FROM --platform=linux/arm64 ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG SECRETS_FILE=tests/.secrets
+ARG CONFIG_FILE=tests/.env.docker
+
 ENV TZ=Etc/UTC
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -12,6 +15,7 @@ RUN apt-get update && \
     tzdata \
     s3cmd \
     jq \
+    bc \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*
@@ -57,11 +61,11 @@ USER tester
 COPY --chown=tester:tester . /home/tester
 
 WORKDIR /home/tester
-RUN cp tests/.env.docker.default tests/.env.docker
+#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
 RUN cp tests/s3cfg.local.default tests/s3cfg.local
 RUN make
 
-RUN . tests/.secrets && \
+RUN . $SECRETS_FILE && \
     export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
@@ -74,6 +78,6 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
         -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
 
 ENV WORKSPACE=.
-ENV VERSITYGW_TEST_ENV=tests/.env.docker
+ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
 
-CMD ["tests/run_all.sh"]
+CMD ["tests/run_all.sh"]

README.md

@@ -8,17 +8,33 @@
 
  [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-**News:**<br>
+### Binary release builds
+Download [latest release](https://github.com/versity/versitygw/releases)
+ | Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
+ |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
+ |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
+ 
+### News
 * New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
 
+### Mailing List
+Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
 
+### Documentation
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
+### Need help?
+Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
+<br>
+Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
+
+### Use Cases
 * Share filesystem directory via S3 protocol
 * Proxy S3 requests to S3 storage
 * Simple to deploy S3 server with a single command
 * Protocol compatibility in `posix` allows common access to files via posix or S3 
 
+### Overview
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.

auth/acl.go

@@ -15,12 +15,14 @@
 package auth
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -203,15 +205,7 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
-	if isRoot {
-		return nil
-	}
-
-	if acl.Owner == access {
-		return nil
-	}
-
+func verifyACL(acl ACL, access string, permission types.Permission) error {
 	if acl.ACL != "" {
 		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -222,6 +216,9 @@ func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool)
 
 		return nil
 	} else {
+		if len(acl.Grantees) == 0 {
+			return nil
+		}
 		grantee := Grantee{Access: access, Permission: permission}
 		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
 
@@ -273,3 +270,88 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 	// Return access denied in all other cases
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
+
+type AccessOptions struct {
+	Acl           ACL
+	AclPermission types.Permission
+	IsRoot        bool
+	Acc           Account
+	Bucket        string
+	Object        string
+	Action        Action
+}
+
+func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+	if opts.Acc.Access == opts.Acl.Owner {
+		return nil
+	}
+
+	policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
+	if err != nil {
+		return err
+	}
+
+	// If bucket policy is not set and the ACL is default, only the owner has access
+	if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
+		return err
+	}
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	// Verify destination bucket access
+	if err := VerifyAccess(ctx, be, opts); err != nil {
+		return err
+	}
+	// Verify source bucket access
+	srcBucket, srcObject, found := strings.Cut(copySource, "/")
+	if !found {
+		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	// Get source bucket ACL
+	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+	if err != nil {
+		return err
+	}
+
+	var srcBucketAcl ACL
+	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+		return err
+	}
+
+	if err := VerifyAccess(ctx, be, AccessOptions{
+		Acl:           srcBucketAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        opts.IsRoot,
+		Acc:           opts.Acc,
+		Bucket:        srcBucket,
+		Object:        srcObject,
+		Action:        GetObjectAction,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}

auth/bucket_policy.go

@@ -0,0 +1,140 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+type BucketPolicy struct {
+	Statement []BucketPolicyItem `json:"Statement"`
+}
+
+func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
+	for _, statement := range bp.Statement {
+		err := statement.Validate(bucket, iam)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	for _, statement := range bp.Statement {
+		if statement.findMatch(principal, action, resource) {
+			switch statement.Effect {
+			case BucketPolicyAccessTypeAllow:
+				return true
+			case BucketPolicyAccessTypeDeny:
+				return false
+			}
+		}
+	}
+
+	return false
+}
+
+type BucketPolicyItem struct {
+	Effect     BucketPolicyAccessType `json:"Effect"`
+	Principals Principals             `json:"Principal"`
+	Actions    Actions                `json:"Action"`
+	Resources  Resources              `json:"Resource"`
+}
+
+func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
+	if err := bpi.Effect.Validate(); err != nil {
+		return err
+	}
+	if err := bpi.Principals.Validate(iam); err != nil {
+		return err
+	}
+	if err := bpi.Resources.Validate(bucket); err != nil {
+		return err
+	}
+
+	containsObjectAction := bpi.Resources.ContainsObjectPattern()
+	containsBucketAction := bpi.Resources.ContainsBucketPattern()
+
+	for action := range bpi.Actions {
+		isObjectAction := action.IsObjectAction()
+		if isObjectAction && !containsObjectAction {
+			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		}
+		if !isObjectAction && !containsBucketAction {
+			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		}
+	}
+
+	return nil
+}
+
+func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
+	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
+		return true
+	}
+
+	return false
+}
+
+func getMalformedPolicyError(err error) error {
+	return s3err.APIError{
+		Code:           "MalformedPolicy",
+		Description:    err.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
+	var policy BucketPolicy
+	if err := json.Unmarshal(policyBin, &policy); err != nil {
+		return getMalformedPolicyError(err)
+	}
+
+	if err := policy.Validate(bucket, iam); err != nil {
+		return getMalformedPolicyError(err)
+	}
+
+	return nil
+}
+
+func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
+	// If bucket policy is not set
+	if len(policy) == 0 {
+		return nil
+	}
+
+	var bucketPolicy BucketPolicy
+	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
+		return err
+	}
+
+	resource := bucket
+	if object != "" {
+		resource += "/" + object
+	}
+
+	fmt.Println(access, action, resource)
+	if !bucketPolicy.isAllowed(access, action, resource) {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}

auth/bucket_policy_actions.go

@@ -0,0 +1,218 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Action string
+
+const (
+	GetBucketAclAction               Action = "s3:GetBucketAcl"
+	CreateBucketAction               Action = "s3:CreateBucket"
+	PutBucketAclAction               Action = "s3:PutBucketAcl"
+	DeleteBucketAction               Action = "s3:DeleteBucket"
+	PutBucketVersioningAction        Action = "s3:PutBucketVersioning"
+	GetBucketVersioningAction        Action = "s3:GetBucketVersioning"
+	PutBucketPolicyAction            Action = "s3:PutBucketPolicy"
+	GetBucketPolicyAction            Action = "s3:GetBucketPolicy"
+	DeleteBucketPolicyAction         Action = "s3:DeleteBucketPolicy"
+	AbortMultipartUploadAction       Action = "s3:AbortMultipartUpload"
+	ListMultipartUploadPartsAction   Action = "s3:ListMultipartUploadParts"
+	ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
+	PutObjectAction                  Action = "s3:PutObject"
+	GetObjectAction                  Action = "s3:GetObject"
+	DeleteObjectAction               Action = "s3:DeleteObject"
+	GetObjectAclAction               Action = "s3:GetObjectAcl"
+	GetObjectAttributesAction        Action = "s3:GetObjectAttributes"
+	PutObjectAclAction               Action = "s3:PutObjectAcl"
+	RestoreObjectAction              Action = "s3:RestoreObject"
+	GetBucketTaggingAction           Action = "s3:GetBucketTagging"
+	PutBucketTaggingAction           Action = "s3:PutBucketTagging"
+	GetObjectTaggingAction           Action = "s3:GetObjectTagging"
+	PutObjectTaggingAction           Action = "s3:PutObjectTagging"
+	DeleteObjectTaggingAction        Action = "s3:DeleteObjectTagging"
+	ListBucketVersionsAction         Action = "s3:ListBucketVersions"
+	ListBucketAction                 Action = "s3:ListBucket"
+	AllActions                       Action = "s3:*"
+)
+
+var supportedActionList = map[Action]struct{}{
+	GetBucketAclAction:               {},
+	CreateBucketAction:               {},
+	PutBucketAclAction:               {},
+	DeleteBucketAction:               {},
+	PutBucketVersioningAction:        {},
+	GetBucketVersioningAction:        {},
+	PutBucketPolicyAction:            {},
+	GetBucketPolicyAction:            {},
+	DeleteBucketPolicyAction:         {},
+	AbortMultipartUploadAction:       {},
+	ListMultipartUploadPartsAction:   {},
+	ListBucketMultipartUploadsAction: {},
+	PutObjectAction:                  {},
+	GetObjectAction:                  {},
+	DeleteObjectAction:               {},
+	GetObjectAclAction:               {},
+	GetObjectAttributesAction:        {},
+	PutObjectAclAction:               {},
+	RestoreObjectAction:              {},
+	GetBucketTaggingAction:           {},
+	PutBucketTaggingAction:           {},
+	GetObjectTaggingAction:           {},
+	PutObjectTaggingAction:           {},
+	DeleteObjectTaggingAction:        {},
+	ListBucketVersionsAction:         {},
+	ListBucketAction:                 {},
+	AllActions:                       {},
+}
+
+var supportedObjectActionList = map[Action]struct{}{
+	AbortMultipartUploadAction:     {},
+	ListMultipartUploadPartsAction: {},
+	PutObjectAction:                {},
+	GetObjectAction:                {},
+	DeleteObjectAction:             {},
+	GetObjectAclAction:             {},
+	GetObjectAttributesAction:      {},
+	PutObjectAclAction:             {},
+	RestoreObjectAction:            {},
+	GetObjectTaggingAction:         {},
+	PutObjectTaggingAction:         {},
+	DeleteObjectTaggingAction:      {},
+	AllActions:                     {},
+}
+
+// Validates Action: it should either wildcard match with supported actions list or be in it
+func (a Action) IsValid() error {
+	if !strings.HasPrefix(string(a), "s3:") {
+		return fmt.Errorf("invalid action: %v", a)
+	}
+
+	if a == AllActions {
+		return nil
+	}
+
+	if a[len(a)-1] == '*' {
+		pattern := strings.TrimSuffix(string(a), "*")
+		for act := range supportedActionList {
+			if strings.HasPrefix(string(act), pattern) {
+				return nil
+			}
+		}
+
+		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+	}
+
+	_, found := supportedActionList[a]
+	if !found {
+		return fmt.Errorf("unsupported action: %v", a)
+	}
+	return nil
+}
+
+// Checks if the action is object action
+func (a Action) IsObjectAction() bool {
+	if a[len(a)-1] == '*' {
+		pattern := strings.TrimSuffix(string(a), "*")
+		for act := range supportedObjectActionList {
+			if strings.HasPrefix(string(act), pattern) {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	_, found := supportedObjectActionList[a]
+	return found
+}
+
+func (a Action) WildCardMatch(act Action) bool {
+	if strings.HasSuffix(string(a), "*") {
+		pattern := strings.TrimSuffix(string(a), "*")
+		return strings.HasPrefix(string(act), pattern)
+	}
+	return false
+}
+
+type Actions map[Action]struct{}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (a *Actions) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("actions can't be empty")
+		}
+		*a = make(Actions)
+		for _, s := range ss {
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("actions can't be empty")
+			}
+			*a = make(Actions)
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Validates and adds a new Action to Actions map
+func (a Actions) Add(str string) error {
+	action := Action(str)
+	err := action.IsValid()
+	if err != nil {
+		return err
+	}
+
+	a[action] = struct{}{}
+	return nil
+}
+
+func (a Actions) FindMatch(action Action) bool {
+	_, ok := a[AllActions]
+	if ok {
+		return true
+	}
+	// First O(1) check for non wildcard actions
+	_, found := a[action]
+	if found {
+		return true
+	}
+
+	for act := range a {
+		if strings.HasSuffix(string(act), "*") && act.WildCardMatch(action) {
+			return true
+		}
+	}
+
+	return false
+}

auth/bucket_policy_effect.go

@@ -0,0 +1,34 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import "fmt"
+
+type BucketPolicyAccessType string
+
+const (
+	BucketPolicyAccessTypeDeny  BucketPolicyAccessType = "Deny"
+	BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
+)
+
+// Checks policy statement Effect to be valid ("Deny", "Allow")
+func (bpat BucketPolicyAccessType) Validate() error {
+	switch bpat {
+	case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
+		return nil
+	}
+
+	return fmt.Errorf("invalid effect: %v", bpat)
+}

auth/bucket_policy_principals.go

@@ -0,0 +1,97 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type Principals map[string]struct{}
+
+func (p Principals) Add(key string) {
+	p[key] = struct{}{}
+}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (p *Principals) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("principals can't be empty")
+		}
+		*p = make(Principals)
+		for _, s := range ss {
+			p.Add(s)
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("principals can't be empty")
+			}
+			*p = make(Principals)
+			p.Add(s)
+		}
+	}
+
+	return err
+}
+
+// Converts Principals map to a slice, by omitting "*"
+func (p Principals) ToSlice() []string {
+	principals := []string{}
+	for p := range p {
+		if p == "*" {
+			continue
+		}
+		principals = append(principals, p)
+	}
+
+	return principals
+}
+
+// Validates Principals by checking user account access keys existence
+func (p Principals) Validate(iam IAMService) error {
+	_, containsWildCard := p["*"]
+	if containsWildCard {
+		if len(p) == 1 {
+			return nil
+		}
+		return fmt.Errorf("principals should either contain * or user access keys")
+	}
+
+	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return fmt.Errorf("user accounts don't exist: %v", accs)
+	}
+
+	return nil
+}
+
+func (p Principals) Contains(userAccess string) bool {
+	// "*" means it matches for any user account
+	_, ok := p["*"]
+	if ok {
+		return true
+	}
+
+	_, found := p[userAccess]
+	return found
+}

auth/bucket_policy_resources.go

@@ -0,0 +1,142 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Resources map[string]struct{}
+
+const ResourceArnPrefix = "arn:aws:s3:::"
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (r *Resources) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("resources can't be empty")
+		}
+		*r = make(Resources)
+		for _, s := range ss {
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("resources can't be empty")
+			}
+			*r = make(Resources)
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Adds and validates a new resource to Resources map
+func (r Resources) Add(rc string) error {
+	ok, pattern := isValidResource(rc)
+	if !ok {
+		return fmt.Errorf("invalid resource: %v", rc)
+	}
+
+	_, found := r[pattern]
+	if found {
+		return fmt.Errorf("duplicate resource: %v", rc)
+	}
+
+	r[pattern] = struct{}{}
+
+	return nil
+}
+
+// Checks if the resources contain object pattern
+func (r Resources) ContainsObjectPattern() bool {
+	for resource := range r {
+		if resource == "*" || strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Checks if the resources contain bucket pattern
+func (r Resources) ContainsBucketPattern() bool {
+	for resource := range r {
+		if resource == "*" || !strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
+func (r Resources) Validate(bucket string) error {
+	for resource := range r {
+		if !strings.HasPrefix(resource, bucket) {
+			return fmt.Errorf("incorrect bucket name in %v", resource)
+		}
+	}
+
+	return nil
+}
+
+func (r Resources) FindMatch(resource string) bool {
+	for res := range r {
+		if strings.HasSuffix(res, "*") {
+			pattern := strings.TrimSuffix(res, "*")
+			if strings.HasPrefix(resource, pattern) {
+				return true
+			}
+		} else {
+			if res == resource {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// Checks the resource to have arn prefix and not starting with /
+func isValidResource(rc string) (isValid bool, pattern string) {
+	if !strings.HasPrefix(rc, ResourceArnPrefix) {
+		return false, ""
+	}
+
+	res := strings.TrimPrefix(rc, ResourceArnPrefix)
+	if res == "" {
+		return false, ""
+	}
+	// The resource can't start with / (bucket name comes first)
+	if strings.HasPrefix(res, "/") {
+		return false, ""
+	}
+
+	return true, res
+}

backend/azure/azure.go

@@ -466,16 +466,6 @@ func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3
 		return nil, azureErrToS3Err(err)
 	}
 
-	dstContainerAcl, err := getAclFromMetadata(res.Metadata, aclKeyCapital)
-	if err != nil {
-		return nil, err
-	}
-
-	err = auth.VerifyACL(*dstContainerAcl, *input.ExpectedBucketOwner, types.PermissionWrite, false)
-	if err != nil {
-		return nil, err
-	}
-
 	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource && isMetaSame(res.Metadata, input.Metadata) {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
 	}

backend/backend.go

@@ -42,6 +42,8 @@ type Backend interface {
 	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
+	DeleteBucketPolicy(_ context.Context, bucket string) error
+
 	// multipart operations
 	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
@@ -125,6 +127,9 @@ func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, poli
 func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
 func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)

backend/mkdir.go

@@ -0,0 +1,81 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// MkdirAll borrowed from stdlib to add ability to set ownership
+// as directories are created
+
+package backend
+
+import (
+	"io/fs"
+	"os"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+var (
+	// TODO: make this configurable
+	defaultDirPerm fs.FileMode = 0755
+)
+
+// MkdirAll is similar to os.MkdirAll but it will return
+// ErrObjectParentIsFile when appropriate
+// MkdirAll creates a directory named path,
+// along with any necessary parents, and returns nil,
+// or else returns an error.
+// The permission bits perm (before umask) are used for all
+// directories that MkdirAll creates.
+// Any newly created directory is set to provided uid/gid ownership.
+// If path is already a directory, MkdirAll does nothing
+// and returns nil.
+// Any directoy created will be set to provided uid/gid ownership
+// if doChown is true.
+func MkdirAll(path string, uid, gid int, doChown bool) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = MkdirAll(path[:j-1], uid, gid, doChown)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, defaultDirPerm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return err
+	}
+	if doChown {
+		err = os.Chown(path, uid, gid)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

backend/posix/posix.go

@@ -46,6 +46,19 @@ type Posix struct {
 
 	rootfd  *os.File
 	rootdir string
+
+	// chownuid/gid enable chowning of files to the account uid/gid
+	// when objects are uploaded
+	chownuid bool
+	chowngid bool
+
+	// read only mode prevents any backend modifications
+	readonly bool
+
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
 }
 
 var _ backend.Backend = &Posix{}
@@ -61,9 +74,16 @@ const (
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
 	aclkey              = "user.acl"
 	etagkey             = "user.etag"
+	policykey           = "user.policy"
 )
 
-func New(rootdir string) (*Posix, error) {
+type PosixOpts struct {
+	ChownUID bool
+	ChownGID bool
+	ReadOnly bool
+}
+
+func New(rootdir string, opts PosixOpts) (*Posix, error) {
 	err := os.Chdir(rootdir)
 	if err != nil {
 		return nil, fmt.Errorf("chdir %v: %w", rootdir, err)
@@ -80,7 +100,15 @@ func New(rootdir string) (*Posix, error) {
 		return nil, fmt.Errorf("xattr not supported on %v", rootdir)
 	}
 
-	return &Posix{rootfd: f, rootdir: rootdir}, nil
+	return &Posix{
+		rootfd:   f,
+		rootdir:  rootdir,
+		euid:     os.Geteuid(),
+		egid:     os.Getegid(),
+		chownuid: opts.ChownUID,
+		chowngid: opts.ChownGID,
+		readonly: opts.ReadOnly,
+	}, nil
 }
 
 func (p *Posix) Shutdown() {
@@ -167,14 +195,30 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He
 	return &s3.HeadBucketOutput{}, nil
 }
 
-func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl []byte) error {
+var (
+	// TODO: make this configurable
+	defaultDirPerm fs.FileMode = 0755
+)
+
+func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
 
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	bucket := *input.Bucket
 
-	err := os.Mkdir(bucket, 0777)
+	err := os.Mkdir(bucket, defaultDirPerm)
 	if err != nil && os.IsExist(err) {
 		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
 	}
@@ -182,6 +226,13 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl
 		return fmt.Errorf("mkdir bucket: %w", err)
 	}
 
+	if doChown {
+		err := os.Chown(bucket, uid, gid)
+		if err != nil {
+			return fmt.Errorf("chown bucket: %w", err)
+		}
+	}
+
 	if err := xattr.Set(bucket, aclkey, acl); err != nil {
 		return fmt.Errorf("set acl: %w", err)
 	}
@@ -190,6 +241,10 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl
 }
 
 func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -223,6 +278,10 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
 }
 
 func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	if p.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if mpu.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -286,7 +345,35 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
 	}, nil
 }
 
-func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+// getChownIDs returns the uid and gid that should be used for chowning
+// the object to the account uid/gid. It also returns a boolean indicating
+// if chowning is needed.
+func (p *Posix) getChownIDs(acct auth.Account) (int, int, bool) {
+	uid := p.euid
+	gid := p.egid
+	var needsChown bool
+	if p.chownuid && acct.UserID != p.euid {
+		uid = acct.UserID
+		needsChown = true
+	}
+	if p.chowngid && acct.GroupID != p.egid {
+		gid = acct.GroupID
+		needsChown = true
+	}
+
+	return uid, gid, needsChown
+}
+
+func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	if p.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -350,8 +437,12 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 		}
 	}
 
-	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, totalsize)
+	f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object,
+		totalsize, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return nil, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -364,6 +455,9 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 		_, err = io.Copy(f, pf)
 		pf.Close()
 		if err != nil {
+			if errors.Is(err, syscall.EDQUOT) {
+				return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+			}
 			return nil, fmt.Errorf("copy part %v: %v", p.PartNumber, err)
 		}
 	}
@@ -375,9 +469,10 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		uid, gid, doChown := p.getChownIDs(acct)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
-			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			return nil, err
 		}
 	}
 	err = f.link()
@@ -496,52 +591,11 @@ func isValidMeta(val string) bool {
 	return false
 }
 
-// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
-// when appropriate
-func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(path[:j-1], perm, bucket, object)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-	return nil
-}
-
 func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if mpu.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -840,7 +894,16 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
 	}, nil
 }
 
-func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) {
+func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (string, error) {
+	if p.readonly {
+		return "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if input.Bucket == nil {
 		return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -879,9 +942,12 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 
 	partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part))
 
-	f, err := openTmpFile(filepath.Join(bucket, objdir),
-		bucket, partPath, length)
+	f, err := p.openTmpFile(filepath.Join(bucket, objdir),
+		bucket, partPath, length, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("open temp file: %w", err)
 	}
 
@@ -889,6 +955,9 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 	tr := io.TeeReader(r, hash)
 	_, err = io.Copy(f, tr)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("write part data: %w", err)
 	}
 
@@ -906,7 +975,16 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 	return etag, nil
 }
 
-func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	if p.readonly {
+		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if upi.Bucket == nil {
 		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -973,9 +1051,12 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRange)
 	}
 
-	f, err := openTmpFile(filepath.Join(*upi.Bucket, objdir),
-		*upi.Bucket, partPath, length)
+	f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
+		*upi.Bucket, partPath, length, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return s3response.CopyObjectResult{}, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -995,6 +1076,9 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 
 	_, err = io.Copy(f, tr)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return s3response.CopyObjectResult{}, fmt.Errorf("copy part data: %w", err)
 	}
 
@@ -1019,6 +1103,15 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 }
 
 func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
+	if p.readonly {
+		return "", s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
 	if po.Bucket == nil {
 		return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1052,6 +1145,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 
 	name := filepath.Join(*po.Bucket, *po.Key)
 
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	contentLength := int64(0)
 	if po.ContentLength != nil {
 		contentLength = *po.ContentLength
@@ -1065,8 +1160,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 			return "", s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData)
 		}
 
-		err = mkdirAll(name, os.FileMode(0755), *po.Bucket, *po.Key)
+		err = backend.MkdirAll(name, uid, gid, doChown)
 		if err != nil {
+			if errors.Is(err, syscall.EDQUOT) {
+				return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+			}
 			return "", err
 		}
 
@@ -1086,9 +1184,12 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 		return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 	}
 
-	f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
-		*po.Bucket, *po.Key, contentLength)
+	f, err := p.openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
+		*po.Bucket, *po.Key, contentLength, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
@@ -1097,11 +1198,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 	rdr := io.TeeReader(po.Body, hash)
 	_, err = io.Copy(f, rdr)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return "", s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return "", fmt.Errorf("write object data: %w", err)
 	}
 	dir := filepath.Dir(name)
 	if dir != "" {
-		err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
 			return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 		}
@@ -1131,6 +1235,10 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 }
 
 func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1195,6 +1303,10 @@ func (p *Posix) removeParents(bucket, object string) error {
 }
 
 func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+	if p.readonly {
+		return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	// delete object already checks bucket
 	delResult, errs := []types.DeletedObject{}, []types.Error{}
 	for _, obj := range input.Delete.Objects {
@@ -1410,6 +1522,10 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
 }
 
 func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	if p.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -1428,7 +1544,6 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 	}
 	dstBucket := *input.Bucket
 	dstObject := *input.Key
-	owner := *input.ExpectedBucketOwner
 
 	_, err := os.Stat(srcBucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1446,22 +1561,6 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
-	dstBucketACLBytes, err := xattr.Get(dstBucket, aclkey)
-	if err != nil {
-		return nil, fmt.Errorf("get dst bucket acl tag: %w", err)
-	}
-
-	var dstBucketACL auth.ACL
-	err = json.Unmarshal(dstBucketACLBytes, &dstBucketACL)
-	if err != nil {
-		return nil, fmt.Errorf("parse dst bucket acl: %w", err)
-	}
-
-	err = auth.VerifyACL(dstBucketACL, owner, types.PermissionWrite, false)
-	if err != nil {
-		return nil, err
-	}
-
 	objPath := filepath.Join(srcBucket, srcObject)
 	f, err := os.Open(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1696,6 +1795,10 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
 }
 
 func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1734,6 +1837,10 @@ func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]
 }
 
 func (p *Posix) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1744,9 +1851,10 @@ func (p *Posix) PutBucketTagging(_ context.Context, bucket string, tags map[stri
 
 	if tags == nil {
 		err = xattr.Remove(bucket, "user."+tagHdr)
-		if err != nil {
+		if err != nil && !isNoAttr(err) {
 			return fmt.Errorf("remove tags: %w", err)
 		}
+
 		return nil
 	}
 
@@ -1818,6 +1926,10 @@ func (p *Posix) getXattrTags(bucket, object string) (map[string]string, error) {
 }
 
 func (p *Posix) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1857,6 +1969,62 @@ func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object string)
 	return p.PutObjectTagging(ctx, bucket, object, nil)
 }
 
+func (p *Posix) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
+	if p.readonly {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if policy == nil {
+		if err := xattr.Remove(bucket, policykey); err != nil {
+			if isNoAttr(err) {
+				return nil
+			}
+
+			return fmt.Errorf("remove policy: %w", err)
+		}
+
+		return nil
+	}
+
+	if err := xattr.Set(bucket, policykey, policy); err != nil {
+		return fmt.Errorf("set policy: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Posix) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	policy, err := xattr.Get(bucket, policykey)
+	if isNoAttr(err) {
+		return []byte{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get bucket policy: %w", err)
+	}
+
+	return policy, nil
+}
+
+func (p *Posix) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	return p.PutBucketPolicy(ctx, bucket, nil)
+}
+
 func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {

backend/posix/with_otmpfile.go

@@ -27,30 +27,42 @@ import (
 	"strconv"
 	"syscall"
 
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 	"golang.org/x/sys/unix"
 )
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	isOTmp  bool
-	size    int64
+	f          *os.File
+	bucket     string
+	objname    string
+	isOTmp     bool
+	size       int64
+	needsChown bool
+	uid        int
+	gid        int
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+var (
+	// TODO: make this configurable
+	defaultFilePerm uint32 = 0644
+)
+
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
 	// Not all filesystems support this, so fallback to CreateTemp for when
 	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err := os.MkdirAll(dir, 0700)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -59,11 +71,27 @@ func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 		if err != nil {
 			return nil, err
 		}
-		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		tmp := &tmpfile{
+			f:          f,
+			bucket:     bucket,
+			objname:    obj,
+			size:       size,
+			needsChown: doChown,
+			uid:        uid,
+			gid:        gid,
+		}
 		// falloc is best effort, its fine if this fails
 		if size > 0 {
 			tmp.falloc()
 		}
+
+		if doChown {
+			err := f.Chown(uid, gid)
+			if err != nil {
+				return nil, fmt.Errorf("set temp file ownership: %w", err)
+			}
+		}
+
 		return tmp, nil
 	}
 
@@ -71,11 +99,29 @@ func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		isOTmp:     true,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+
 	// falloc is best effort, its fine if this fails
 	if size > 0 {
 		tmp.falloc()
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return tmp, nil
 }
 
@@ -100,6 +146,13 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
+	dir := filepath.Dir(objPath)
+
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	if err != nil {
+		return fmt.Errorf("make parent dir: %w", err)
+	}
+
 	if !tmp.isOTmp {
 		// O_TMPFILE not suported, use fallback
 		return tmp.fallbackLink()
@@ -111,14 +164,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dir, err := os.Open(filepath.Dir(objPath))
+	dirf, err := os.Open(dir)
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dir.Close()
+	defer dirf.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile (%q in %q): %w",
 			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
@@ -138,6 +191,9 @@ func (tmp *tmpfile) fallbackLink() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(fs.FileMode(defaultFilePerm))
+
 	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)

backend/posix/without_otmpfile.go

@@ -24,6 +24,9 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 )
 
 type tmpfile struct {
@@ -33,20 +36,36 @@ type tmpfile struct {
 	size    int64
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := p.getChownIDs(acct)
+
 	// Create a temp file for upload while in progress (see link comments below).
-	err := os.MkdirAll(dir, 0700)
+	var err error
+	err = backend.MkdirAll(dir, uid, gid, doChown)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create temp file: %w", err)
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
 }
 
+var (
+	// TODO: make this configurable
+	defaultFilePerm fs.FileMode = 0644
+)
+
 func (tmp *tmpfile) link() error {
 	tempname := tmp.f.Name()
 	// cleanup in case anything goes wrong, if rename succeeds then
@@ -64,6 +83,9 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
+	// reset default file mode because CreateTemp uses 0600
+	tmp.f.Chmod(defaultFilePerm)
+
 	err = tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)

backend/scoutfs/scoutfs.go

@@ -25,15 +25,24 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
 )
 
+type ScoutfsOpts struct {
+	ChownUID    bool
+	ChownGID    bool
+	ReadOnly    bool
+	GlacierMode bool
+}
+
 type ScoutFS struct {
 	*posix.Posix
 	rootfd  *os.File
@@ -49,6 +58,19 @@ type ScoutFS struct {
 	// ListObjects: if file offline, set obj storage class to GLACIER
 	// RestoreObject: add batch stage request to file
 	glaciermode bool
+
+	// chownuid/gid enable chowning of files to the account uid/gid
+	// when objects are uploaded
+	chownuid bool
+	chowngid bool
+
+	// read only mode prevents any backend modifications
+	readonly bool
+
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -92,14 +114,6 @@ const (
 	ExtCacheDone
 )
 
-// Option sets various options for scoutfs
-type Option func(s *ScoutFS)
-
-// WithGlacierEmulation sets glacier mode emulation
-func WithGlacierEmulation() Option {
-	return func(s *ScoutFS) { s.glaciermode = true }
-}
-
 func (s *ScoutFS) Shutdown() {
 	s.Posix.Shutdown()
 	s.rootfd.Close()
@@ -110,10 +124,51 @@ func (*ScoutFS) String() string {
 	return "ScoutFS Gateway"
 }
 
+// getChownIDs returns the uid and gid that should be used for chowning
+// the object to the account uid/gid. It also returns a boolean indicating
+// if chowning is needed.
+func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
+	uid := s.euid
+	gid := s.egid
+	var needsChown bool
+	if s.chownuid && acct.UserID != s.euid {
+		uid = acct.UserID
+		needsChown = true
+	}
+	if s.chowngid && acct.GroupID != s.egid {
+		gid = acct.GroupID
+		needsChown = true
+	}
+
+	return uid, gid, needsChown
+}
+
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
-func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	if s.readonly {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		acct = auth.Account{}
+	}
+
+	if input.Bucket == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+	}
+	if input.Key == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+	}
+	if input.UploadId == nil {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
+	}
+	if input.MultipartUpload == nil {
+		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+	}
+
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
@@ -139,7 +194,10 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	partsize := int64(0)
 	var totalsize int64
 	for i, p := range parts {
-		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
+		if p.PartNumber == nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
 		fi, err := os.Lstat(partPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
@@ -171,16 +229,19 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 
 	// use totalsize=0 because we wont be writing to the file, only moving
 	// extents around.  so we dont want to fallocate this.
-	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
+	f, err := s.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0, acct)
 	if err != nil {
+		if errors.Is(err, syscall.EDQUOT) {
+			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
+		}
 		return nil, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
 
 	for _, p := range parts {
-		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
+		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
+			return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
@@ -189,7 +250,7 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
+			return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
 		}
 	}
 
@@ -200,9 +261,10 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		uid, gid, doChown := s.getChownIDs(acct)
+		err = backend.MkdirAll(dir, uid, gid, doChown)
 		if err != nil {
-			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			return nil, err
 		}
 	}
 	err = f.link()
@@ -307,51 +369,6 @@ func isValidMeta(val string) bool {
 	return false
 }
 
-// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
-// when appropriate
-func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = mkdirAll(path[:j-1], perm, bucket, object)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-	return nil
-}
-
 func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
@@ -454,6 +471,13 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, err
 	}
 
+	objSize := fi.Size()
+	if fi.IsDir() {
+		// directory objects are always 0 len
+		objSize = 0
+		length = 0
+	}
+
 	if length == -1 {
 		length = fi.Size() - startOffset + 1
 	}
@@ -462,6 +486,11 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
+	var contentRange string
+	if acceptRange != "" {
+		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
+	}
+
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
@@ -519,6 +548,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		Metadata:        userMetaData,
 		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
+		ContentRange:    &contentRange,
 	}, nil
 }
 

backend/scoutfs/scoutfs_compat.go

@@ -17,7 +17,6 @@
 package scoutfs
 
 import (
-	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -29,11 +28,17 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/versity/scoutfs-go"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/backend/posix"
 )
 
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
-	p, err := posix.New(rootdir)
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
+	p, err := posix.New(rootdir, posix.PosixOpts{
+		ChownUID: opts.ChownUID,
+		ChownGID: opts.ChownGID,
+		ReadOnly: opts.ReadOnly,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -43,60 +48,71 @@ func New(rootdir string, opts ...Option) (*ScoutFS, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
-	s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
-	for _, opt := range opts {
-		opt(s)
-	}
-
-	return s, nil
+	return &ScoutFS{
+		Posix:    p,
+		rootfd:   f,
+		rootdir:  rootdir,
+		chownuid: opts.ChownUID,
+		chowngid: opts.ChownGID,
+		readonly: opts.ReadOnly,
+	}, nil
 }
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	isOTmp  bool
-	size    int64
+	f          *os.File
+	bucket     string
+	objname    string
+	size       int64
+	needsChown bool
+	uid        int
+	gid        int
 }
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+var (
+	// TODO: make this configurable
+	defaultFilePerm uint32 = 0644
+)
+
+func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
+	uid, gid, doChown := s.getChownIDs(acct)
+
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
-	// Not all filesystems support this, so fallback to CreateTemp for when
-	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
 	if err != nil {
-		// O_TMPFILE not supported, try fallback
-		err := os.MkdirAll(dir, 0700)
-		if err != nil {
-			return nil, fmt.Errorf("make temp dir: %w", err)
-		}
-		f, err := os.CreateTemp(dir,
-			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-		if err != nil {
-			return nil, err
-		}
-		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
-		// falloc is best effort, its fine if this fails
-		if size > 0 {
-			tmp.falloc()
-		}
-		return tmp, nil
+		return nil, err
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	tmp := &tmpfile{
+		f:          f,
+		bucket:     bucket,
+		objname:    obj,
+		size:       size,
+		needsChown: doChown,
+		uid:        uid,
+		gid:        gid,
+	}
+
 	// falloc is best effort, its fine if this fails
 	if size > 0 {
 		tmp.falloc()
 	}
+
+	if doChown {
+		err := f.Chown(uid, gid)
+		if err != nil {
+			return nil, fmt.Errorf("set temp file ownership: %w", err)
+		}
+	}
+
 	return tmp, nil
 }
 
@@ -121,9 +137,11 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	if !tmp.isOTmp {
-		// O_TMPFILE not suported, use fallback
-		return tmp.fallbackLink()
+	dir := filepath.Dir(objPath)
+
+	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
+	if err != nil {
+		return fmt.Errorf("make parent dir: %w", err)
 	}
 
 	procdir, err := os.Open(procfddir)
@@ -132,14 +150,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dir, err := os.Open(filepath.Dir(objPath))
+	dirf, err := os.Open(dir)
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dir.Close()
+	defer dirf.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile: %w", err)
 	}
@@ -152,26 +170,6 @@ func (tmp *tmpfile) link() error {
 	return nil
 }
 
-func (tmp *tmpfile) fallbackLink() error {
-	tempname := tmp.f.Name()
-	// cleanup in case anything goes wrong, if rename succeeds then
-	// this will no longer exist
-	defer os.Remove(tempname)
-
-	err := tmp.f.Close()
-	if err != nil {
-		return fmt.Errorf("close tmpfile: %w", err)
-	}
-
-	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
-}
-
 func (tmp *tmpfile) Write(b []byte) (int, error) {
 	if int64(len(b)) > tmp.size {
 		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)

backend/scoutfs/scoutfs_incompat.go

@@ -20,9 +20,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
+
+	"github.com/versity/versitygw/auth"
 )
 
-func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
@@ -34,7 +36,13 @@ var (
 	errNotSupported = errors.New("not supported")
 )
 
-func openTmpFile(_, _, _ string, _ int64) (*tmpfile, error) {
+func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile, error) {
+	// make these look used for static check
+	_ = s.chownuid
+	_ = s.chowngid
+	_ = s.euid
+	_ = s.egid
+	_ = s.readonly
 	return nil, errNotSupported
 }
 

cmd/versitygw/gateway_test.go

@@ -56,7 +56,7 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir)
+	be, err := posix.New(tempdir, posix.PosixOpts{})
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}

cmd/versitygw/main.go

@@ -19,6 +19,8 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"github.com/gofiber/fiber/v2"
@@ -40,10 +42,13 @@ var (
 	certFile, keyFile                      string
 	kafkaURL, kafkaTopic, kafkaKey         string
 	natsURL, natsTopic                     string
+	eventWebhookURL                        string
+	eventConfigFilePath                    string
 	logWebhookURL                          string
 	accessLog                              string
 	healthPath                             string
 	debug                                  bool
+	pprof                                  string
 	quiet                                  bool
 	iamDir                                 string
 	ldapURL, ldapBindDN, ldapPassword      string
@@ -79,6 +84,7 @@ func main() {
 		azureCommand(),
 		adminCommand(),
 		testCommand(),
+		utilsCommand(),
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -187,6 +193,12 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
+		&cli.StringFlag{
+			Name:        "pprof",
+			Usage:       "enable pprof debug on specified port",
+			EnvVars:     []string{"VGW_PPROF"},
+			Destination: &pprof,
+		},
 		&cli.BoolFlag{
 			Name:        "quiet",
 			Usage:       "silence stdout request logging output",
@@ -241,6 +253,20 @@ func initFlags() []cli.Flag {
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
+		&cli.StringFlag{
+			Name:        "event-webhook-url",
+			Usage:       "webhook url to send bucket notifications",
+			EnvVars:     []string{"VGW_EVENT_WEBHOOK_URL"},
+			Destination: &eventWebhookURL,
+			Aliases:     []string{"ewu"},
+		},
+		&cli.StringFlag{
+			Name:        "event-filter",
+			Usage:       "bucket event notifications filters configuration file path",
+			EnvVars:     []string{"VGW_EVENT_FILTER"},
+			Destination: &eventConfigFilePath,
+			Aliases:     []string{"ef"},
+		},
 		&cli.StringFlag{
 			Name:        "iam-dir",
 			Usage:       "if defined, run internal iam service within this directory",
@@ -373,6 +399,14 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		return fmt.Errorf("root user access and secret key must be provided")
 	}
 
+	if pprof != "" {
+		// listen on specified port for pprof debug
+		// point browser to http://<ip:port>/debug/pprof/
+		go func() {
+			log.Fatal(http.ListenAndServe(pprof, nil))
+		}()
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:           "versitygw",
 		ServerHeader:      "VERSITYGW",
@@ -465,14 +499,16 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
-		KafkaURL:      kafkaURL,
-		KafkaTopic:    kafkaTopic,
-		KafkaTopicKey: kafkaKey,
-		NatsURL:       natsURL,
-		NatsTopic:     natsTopic,
+		KafkaURL:             kafkaURL,
+		KafkaTopic:           kafkaTopic,
+		KafkaTopicKey:        kafkaKey,
+		NatsURL:              natsURL,
+		NatsTopic:            natsTopic,
+		WebhookURL:           eventWebhookURL,
+		FilterConfigFilePath: eventConfigFilePath,
 	})
 	if err != nil {
-		return fmt.Errorf("unable to connect to the message broker: %w", err)
+		return fmt.Errorf("init bucket event notifications: %w", err)
 	}
 
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
@@ -531,5 +567,15 @@ Loop:
 		}
 	}
 
+	if evSender != nil {
+		err := evSender.Close()
+		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
+			fmt.Fprintf(os.Stderr, "close event sender: %v\n", err)
+		}
+	}
+
 	return saveErr
 }

cmd/versitygw/posix.go

@@ -21,6 +21,11 @@ import (
 	"github.com/versity/versitygw/backend/posix"
 )
 
+var (
+	chownuid, chowngid bool
+	readonly           bool
+)
+
 func posixCommand() *cli.Command {
 	return &cli.Command{
 		Name:  "posix",
@@ -36,6 +41,26 @@ bucket: mybucket
 object: a/b/c/myobject
 will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 		Action: runPosix,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
+			&cli.BoolFlag{
+				Name:        "readonly",
+				Usage:       "allow only read operations to backend",
+				EnvVars:     []string{"VGW_READ_ONLY"},
+				Destination: &readonly,
+			},
+		},
 	}
 }
 
@@ -44,7 +69,11 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	be, err := posix.New(ctx.Args().Get(0))
+	be, err := posix.New(ctx.Args().Get(0), posix.PosixOpts{
+		ChownUID: chownuid,
+		ChownGID: chowngid,
+		ReadOnly: readonly,
+	})
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)
 	}

cmd/versitygw/scoutfs.go

@@ -23,6 +23,10 @@ import (
 
 var (
 	glacier bool
+
+	// defined in posix.go:
+	// chownuid, chowngid bool
+	// readonly           bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -51,6 +55,24 @@ move interfaces as well as support for tiered filesystems.`,
 				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
+			&cli.BoolFlag{
+				Name:        "chuid",
+				Usage:       "chown newly created files and directories to client account UID",
+				EnvVars:     []string{"VGW_CHOWN_UID"},
+				Destination: &chownuid,
+			},
+			&cli.BoolFlag{
+				Name:        "chgid",
+				Usage:       "chown newly created files and directories to client account GID",
+				EnvVars:     []string{"VGW_CHOWN_GID"},
+				Destination: &chowngid,
+			},
+			&cli.BoolFlag{
+				Name:        "readonly",
+				Usage:       "allow only read operations to backend",
+				EnvVars:     []string{"VGW_READ_ONLY"},
+				Destination: &readonly,
+			},
 		},
 	}
 }
@@ -60,12 +82,13 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	var opts []scoutfs.Option
-	if glacier {
-		opts = append(opts, scoutfs.WithGlacierEmulation())
-	}
+	var opts scoutfs.ScoutfsOpts
+	opts.GlacierMode = glacier
+	opts.ChownUID = chownuid
+	opts.ChownGID = chowngid
+	opts.ReadOnly = readonly
 
-	be, err := scoutfs.New(ctx.Args().Get(0), opts...)
+	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}

cmd/versitygw/test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package main
 
 import (
@@ -84,6 +98,11 @@ func initTestCommands() []*cli.Command {
 			Usage:  "Tests iam service",
 			Action: getAction(integration.TestIAM),
 		},
+		{
+			Name:   "access-control",
+			Usage:  "Tests gateway access control with bucket ACLs and Policies",
+			Action: getAction(integration.TestAccessControl),
+		},
 		{
 			Name:  "bench",
 			Usage: "Runs download/upload performance test on the gateway",

cmd/versitygw/utils.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/s3event"
+)
+
+func utilsCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "utils",
+		Usage: "utility helper CLI tool",
+		Subcommands: []*cli.Command{
+			{
+				Name:    "gen-event-filter-config",
+				Aliases: []string{"gefc"},
+				Usage:   "Create a new configuration file for bucket event notifications filter.",
+				Action:  generateEventFiltersConfig,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:    "path",
+						Usage:   "the path where the config file has to be created",
+						Aliases: []string{"p"},
+					},
+				},
+			},
+		},
+	}
+}
+
+func generateEventFiltersConfig(ctx *cli.Context) error {
+	pathFlag := ctx.String("path")
+	path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
+	if err != nil {
+		return err
+	}
+
+	config := s3event.EventFilter{
+		s3event.EventObjectCreated:           true,
+		s3event.EventObjectCreatedPut:        true,
+		s3event.EventObjectCreatedPost:       true,
+		s3event.EventObjectCreatedCopy:       true,
+		s3event.EventCompleteMultipartUpload: true,
+		s3event.EventObjectDeleted:           true,
+		s3event.EventObjectTagging:           true,
+		s3event.EventObjectTaggingPut:        true,
+		s3event.EventObjectTaggingDelete:     true,
+		s3event.EventObjectAclPut:            true,
+		s3event.EventObjectRestore:           true,
+		s3event.EventObjectRestorePost:       true,
+		s3event.EventObjectRestoreCompleted:  true,
+	}
+
+	configBytes, err := json.Marshal(config)
+	if err != nil {
+		return fmt.Errorf("parse event config: %w", err)
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create config file: %w", err)
+	}
+	defer file.Close()
+
+	_, err = file.Write(configBytes)
+	if err != nil {
+		return fmt.Errorf("write config file: %w", err)
+	}
+
+	return nil
+}

extra/example.conf

@@ -121,28 +121,55 @@ ROOT_SECRET_ACCESS_KEY=
 # Event Logs #
 ##############
 
+# The gateway events are similar to AWS S3 events, and are documented in the
+# wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+
+# The VGW_EVENT_FILTER option specifies a config file that contains the
+# event filter rules. The event filter rules are used to determine which
+# events are sent to the configured event services.
+# Use the following to generate a default rules file in /etc/versitygw.d/:
+# versitygw utils gen-event-filter-config -p /etc/versitygw.d
+# The resulting file, /etc/versitygw.d/event_config.json, can be modified and
+# specified in the VGW_EVENT_FILTER option.
+# When VGW_EVENT_FILTER is not specified, all events are sent to the configured
+# event service.
+#VGW_EVENT_FILTER=
+
 # Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
 # VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
-# bucket events will be sent to the kafka service. The gateway events are
-# similar to AWS S3 events, and are documented in the wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
+# configured bucket events will be sent to the kafka service.
 #VGW_EVENT_KAFKA_URL=
 #VGW_EVENT_KAFKA_TOPIC=
 #VGW_EVENT_KAFKA_KEY=
 
 # Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
-# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the
-# the NATS messaging service. The gateway events are  similar to AWS S3 events,
-# and are documented in the wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
+# and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be
+# sent to the the NATS messaging service.
 #VGW_EVENT_NATS_URL=
 #VGW_EVENT_NATS_TOPIC=
 
+# Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is
+# specified, all configured bucket events will be sent to the webhook.
+#VGW_EVENT_WEBHOOK_URL=
+
+#######################
+# Debug / Diagnostics #
+#######################
+
 # The VGW_DEBUG option enables verbose debug log output to stdout. This output
 # includes details for signature verification steps. This is generally only
 # useful for debugging the S3 server, and should not be used in production.
 #VGW_DEBUG=false
 
+# The VGW_PPROF option enables the pprof HTTP server for profiling the S3
+# server. See the following for more information:
+# https://pkg.go.dev/net/http/pprof
+# To enable, set the VGW_PPROF option to the listening address for the pprof
+# server. For example, to listen on localhost port 6060, set the option to
+# "localhost:6060".
+#VGW_PPROF=
+
 ################
 # IAM services #
 ################
@@ -216,20 +243,30 @@ ROOT_SECRET_ACCESS_KEY=
 # below the "bucket directory" are treated as the objects. The object
 # name is split on "/" separator to translate to posix storage.
 # For example:
-# top level: /mnt/fs/gwroot
+# top level (VGW_BACKEND_ARG): /mnt/fs/gwroot
 # bucket: mybucket
 # object: a/b/c/myobject
 # will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
 
-# There are currently no further options other than VGW_BACKEND_ARG for the
-# posix backend.
+# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
+# change the ownership of newly created files and directories to the IAM
+# account UID/GID.
+#VGW_CHOWN_UID=false
+#VGW_CHOWN_GID=false
+
+# The VGW_READ_ONLY option will disable all write operations to the backend
+# filesystem. This is useful for creating a read-only gateway for clients.
+# This will prevent any PUT, POST, DELETE, and multipart upload operations
+# for objects and buckets as well as preventing updating metadata for objects.
+#VGW_READ_ONLY=false
 
 ###########
 # scoutfs #
 ###########
 
 # The scoutfs backend requires a ScoutFS filesystem type for the backend
-# path. The glacier mode functionality requires ScoutAM to be configured
+# path. The object to posix name mappings follow the same rules as posix for
+# scoutfs. The glacier mode functionality requires ScoutAM to be configured
 # for tiering data from the ScoutFS filesystem to a mass stroage system.
 # The mass storage system is often one or more tape libraries. Due to the
 # high latency of tape, the glacier mode functionality is designed to
@@ -248,6 +285,18 @@ ROOT_SECRET_ACCESS_KEY=
 # RestoreObject: add batch stage request to file
 #VGW_SCOUTFS_GLACIER=false
 
+# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
+# change the ownership of newly created files and directories to the IAM
+# account UID/GID.
+#VGW_CHOWN_UID=false
+#VGW_CHOWN_GID=false
+
+# The VGW_READ_ONLY option will disable all write operations to the backend
+# filesystem. This is useful for creating a read-only gateway for clients.
+# This will prevent any PUT, POST, DELETE, and multipart upload operations
+# for objects and buckets as well as preventing updating metadata for objects.
+#VGW_READ_ONLY=false
+
 ######
 # s3 #
 ######

go.mod

@@ -3,68 +3,68 @@ module github.com/versity/versitygw
 go 1.21
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
-	github.com/aws/aws-sdk-go-v2 v1.26.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
-	github.com/aws/smithy-go v1.20.1
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/aws/smithy-go v1.20.2
 	github.com/go-ldap/ldap/v3 v3.4.6
-	github.com/gofiber/fiber/v2 v2.52.2
+	github.com/gofiber/fiber/v2 v2.52.4
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.33.1
+	github.com/nats-io/nats.go v1.34.1
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/urfave/cli/v2 v2.27.1
 	github.com/valyala/fasthttp v1.52.0
-	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.18.0
+	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
+	golang.org/x/sys v0.19.0
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.8
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.8
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 )

go.sum

@@ -1,5 +1,5 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
@@ -10,52 +10,52 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
-github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
-github.com/aws/aws-sdk-go-v2/config v1.27.8 h1:0r8epOsiJ7YJz65MGcb8i91ehFp4kvvFe2qkq5oYeRI=
-github.com/aws/aws-sdk-go-v2/config v1.27.8/go.mod h1:XsmYKxYNuIhLsFddpNds+j9H5XKzjWDdg/SZngiwFio=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.8 h1:WUdNLXbyNbU07V/WFrSOBXqZTDgmmMNMgUFzpYOKJhw=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.8/go.mod h1:iPZzLpaBIfhyvVS/XGD3JvR1GP3YdHTqpySKDlqkfs8=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 h1:S+L2QSKhUuShih3aq9P/mkzDBiOO5tTyVg+vXREfsfg=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12 h1:rfAytUY7OgbOMDkzxdiigZkbTe9SDER2dIpO/Fzi9+0=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12/go.mod h1:BaY3WWSgUwV/zq0K3HePyXhRYZxGnDATYERkR0f1RTs=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48=
+github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
+github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
+github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
+github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
-github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
-github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -65,10 +65,10 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
 github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
-github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo=
-github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
-github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
-github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
+github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -79,8 +79,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
-github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -90,15 +90,15 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
-github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
+github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
-github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
@@ -106,8 +106,8 @@ github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6kt
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
@@ -126,23 +126,23 @@ github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7g
 github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
-github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
-github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
+github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
+github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -151,8 +151,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -169,8 +169,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

s3api/controllers/backend_moq_test.go

@@ -44,6 +44,9 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketPolicy method")
+//			},
 //			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketTagging method")
 //			},
@@ -53,7 +56,7 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error {
 //				panic("mock out the DeleteObjectTagging method")
 //			},
-//			DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+//			DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 //				panic("mock out the DeleteObjects method")
 //			},
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
@@ -174,6 +177,9 @@ type BackendMock struct {
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
+	// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
+	DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketTaggingFunc mocks the DeleteBucketTagging method.
 	DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -331,6 +337,13 @@ type BackendMock struct {
 			// DeleteBucketInput is the deleteBucketInput argument value.
 			DeleteBucketInput *s3.DeleteBucketInput
 		}
+		// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
+		DeleteBucketPolicy []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketTagging holds details about calls to the DeleteBucketTagging method.
 		DeleteBucketTagging []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -585,6 +598,7 @@ type BackendMock struct {
 	lockCreateBucket            sync.RWMutex
 	lockCreateMultipartUpload   sync.RWMutex
 	lockDeleteBucket            sync.RWMutex
+	lockDeleteBucketPolicy      sync.RWMutex
 	lockDeleteBucketTagging     sync.RWMutex
 	lockDeleteObject            sync.RWMutex
 	lockDeleteObjectTagging     sync.RWMutex
@@ -881,6 +895,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketPolicy calls DeleteBucketPolicyFunc.
+func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketPolicyFunc == nil {
+		panic("BackendMock.DeleteBucketPolicyFunc: method is nil but Backend.DeleteBucketPolicy was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketPolicy.Lock()
+	mock.calls.DeleteBucketPolicy = append(mock.calls.DeleteBucketPolicy, callInfo)
+	mock.lockDeleteBucketPolicy.Unlock()
+	return mock.DeleteBucketPolicyFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketPolicyCalls gets all the calls that were made to DeleteBucketPolicy.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketPolicyCalls())
+func (mock *BackendMock) DeleteBucketPolicyCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketPolicy.RLock()
+	calls = mock.calls.DeleteBucketPolicy
+	mock.lockDeleteBucketPolicy.RUnlock()
+	return calls
+}
+
 // DeleteBucketTagging calls DeleteBucketTaggingFunc.
 func (mock *BackendMock) DeleteBucketTagging(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketTaggingFunc == nil {

s3api/controllers/base_test.go

@@ -77,7 +77,8 @@ func TestNew(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := New(tt.args.be, tt.args.iam, nil, nil); !reflect.DeepEqual(got, tt.want) {
+			got := New(tt.args.be, tt.args.iam, nil, nil, false)
+			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("New() = %v, want %v", got, tt.want)
 			}
 		})
@@ -570,6 +571,19 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</VersioningConfiguration>
 	`
 
+	policyBody := `
+	{
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:GetObject",
+				"Resource": "arn:aws:s3:::my-bucket/*"
+			}
+		]
+	}
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -667,12 +681,21 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 200,
 		},
 		{
-			name: "Put-bucket-policy-success",
+			name: "Put-bucket-policy-invalid-body",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", nil),
 			},
 			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", strings.NewReader(policyBody)),
+			},
+			wantErr:    false,
 			statusCode: 200,
 		},
 		{

s3api/middlewares/acl-parser.go

@@ -54,7 +54,6 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 			}
 			return ctx.Next()
 		}
-		//TODO: provide correct action names for the logger, after implementing DetectAction middleware
 		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
 		if err != nil {
 			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})

s3api/router.go

@@ -27,8 +27,8 @@ type S3ApiRouter struct {
 	WithAdmSrv bool
 }
 
-func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
-	s3ApiController := controllers.New(be, iam, logger, evs)
+func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
+	s3ApiController := controllers.New(be, iam, logger, evs, debug)
 
 	if sa.WithAdmSrv {
 		adminController := controllers.NewAdminController(iam, be)

s3api/router_test.go

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
+			tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
 		})
 	}
 }

s3api/server.go

@@ -70,7 +70,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 	app.Use(middlewares.VerifyMD5Body(l))
 	app.Use(middlewares.AclParser(be, l))
 
-	server.router.Init(app, be, iam, l, evs)
+	server.router.Init(app, be, iam, l, evs, server.debug)
 
 	return server, nil
 }

s3err/s3err.go

@@ -116,6 +116,7 @@ const (
 	ErrExistingObjectIsDirectory
 	ErrObjectParentIsFile
 	ErrDirectoryObjectContainsData
+	ErrQuotaExceeded
 )
 
 var errorCodeResponse = map[ErrorCode]APIError{
@@ -414,6 +415,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Directory object contains data payload.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrQuotaExceeded: {
+		Code:           "QuotaExceeded",
+		Description:    "Your request was denied due to quota exceeded.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.

s3event/event.go

@@ -15,13 +15,18 @@
 package s3event
 
 import (
+	"encoding/json"
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 )
 
 type S3EventSender interface {
 	SendEvent(ctx *fiber.Ctx, meta EventMeta)
+	Close() error
 }
 
 type EventMeta struct {
@@ -36,22 +41,6 @@ type EventFields struct {
 	Records []EventSchema
 }
 
-type EventType string
-
-const (
-	EventObjectPut               EventType = "s3:ObjectCreated:Put"
-	EventObjectCopy              EventType = "s3:ObjectCreated:Copy"
-	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
-	EventObjectDelete            EventType = "s3:ObjectRemoved:Delete"
-	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
-	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
-	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
-	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
-	// Not supported
-	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
-	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
-)
-
 type EventSchema struct {
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
@@ -78,9 +67,18 @@ type EventResponseElements struct {
 	HostId    string `json:"x-amz-id-2"`
 }
 
+type ConfigurationId string
+
+// This field will be changed after implementing per bucket notifications
+const (
+	ConfigurationIdKafka   ConfigurationId = "kafka-global"
+	ConfigurationIdNats    ConfigurationId = "nats-global"
+	ConfigurationIdWebhook ConfigurationId = "webhook-global"
+)
+
 type EventS3Data struct {
 	S3SchemaVersion string            `json:"s3SchemaVersion"`
-	ConfigurationId string            `json:"configurationId"`
+	ConfigurationId ConfigurationId   `json:"configurationId"`
 	Bucket          EventS3BucketData `json:"bucket"`
 	Object          EventObjectData   `json:"object"`
 }
@@ -109,22 +107,95 @@ type EventObjectData struct {
 }
 
 type EventConfig struct {
-	KafkaURL      string
-	KafkaTopic    string
-	KafkaTopicKey string
-	NatsURL       string
-	NatsTopic     string
+	KafkaURL             string
+	KafkaTopic           string
+	KafkaTopicKey        string
+	NatsURL              string
+	NatsTopic            string
+	WebhookURL           string
+	FilterConfigFilePath string
 }
 
 func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
-	if cfg.KafkaURL != "" && cfg.NatsURL != "" {
-		return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
+	filter, err := parseEventFilters(cfg.FilterConfigFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("parse event filter config file %w", err)
 	}
-	if cfg.NatsURL != "" {
-		return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
+	var evSender S3EventSender
+	switch {
+	case cfg.WebhookURL != "":
+		evSender, err = InitWebhookEventSender(cfg.WebhookURL, filter)
+		fmt.Printf("initializing S3 Event Notifications with webhook URL %v\n", cfg.WebhookURL)
+	case cfg.KafkaURL != "":
+		evSender, err = InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey, filter)
+		fmt.Printf("initializing S3 Event Notifications with kafka. URL: %v, topic: %v\n", cfg.WebhookURL, cfg.KafkaTopic)
+	case cfg.NatsURL != "":
+		evSender, err = InitNatsEventService(cfg.NatsURL, cfg.NatsTopic, filter)
+		fmt.Printf("initializing S3 Event Notifications with Nats. URL: %v, topic: %v\n", cfg.NatsURL, cfg.NatsTopic)
+	default:
+		return nil, nil
 	}
-	if cfg.KafkaURL != "" {
-		return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
-	}
-	return nil, nil
+
+	return evSender, err
+}
+
+func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) ([]byte, error) {
+	path := strings.Split(ctx.Path(), "/")
+	bucket, object := path[1], strings.Join(path[2:], "/")
+	acc := ctx.Locals("account").(auth.Account)
+
+	event := []EventSchema{
+		{
+			EventVersion: "2.2",
+			EventSource:  "aws:s3",
+			AwsRegion:    ctx.Locals("region").(string),
+			EventTime:    time.Now().Format(time.RFC3339),
+			EventName:    meta.EventName,
+			UserIdentity: EventUserIdentity{
+				PrincipalId: acc.Access,
+			},
+			RequestParameters: EventRequestParams{
+				SourceIPAddress: ctx.IP(),
+			},
+			ResponseElements: EventResponseElements{
+				RequestId: ctx.Get("X-Amz-Request-Id"),
+				HostId:    ctx.Get("X-Amz-Id-2"),
+			},
+			S3: EventS3Data{
+				S3SchemaVersion: "1.0",
+				ConfigurationId: configId,
+				Bucket: EventS3BucketData{
+					Name: bucket,
+					OwnerIdentity: EventUserIdentity{
+						PrincipalId: meta.BucketOwner,
+					},
+					Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
+				},
+				Object: EventObjectData{
+					Key:       object,
+					Size:      meta.ObjectSize,
+					ETag:      meta.ObjectETag,
+					VersionId: meta.VersionId,
+					Sequencer: genSequencer(),
+				},
+			},
+			GlacierEventData: EventGlacierData{
+				// Not supported
+				RestoreEventData: EventRestoreData{},
+			},
+		},
+	}
+
+	return json.Marshal(event)
+}
+
+func generateTestEvent() ([]byte, error) {
+	msg := map[string]string{
+		"Service": "S3",
+		"Event":   "s3:TestEvent",
+		"Time":    time.Now().Format(time.RFC3339),
+		"Bucket":  "Test-Bucket",
+	}
+
+	return json.Marshal(msg)
 }

s3event/filter.go

@@ -0,0 +1,122 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type EventType string
+
+const (
+	EventObjectCreated           EventType = "s3:ObjectCreated:*" // ObjectCreated
+	EventObjectCreatedPut        EventType = "s3:ObjectCreated:Put"
+	EventObjectCreatedPost       EventType = "s3:ObjectCreated:Post"
+	EventObjectCreatedCopy       EventType = "s3:ObjectCreated:Copy"
+	EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
+	EventObjectDeleted           EventType = "s3:ObjectRemoved:Delete" // ObjectRemoved
+	EventObjectTagging           EventType = "s3:ObjectTagging:*"      // ObjectTagging
+	EventObjectTaggingPut        EventType = "s3:ObjectTagging:Put"
+	EventObjectTaggingDelete     EventType = "s3:ObjectTagging:Delete"
+	EventObjectAclPut            EventType = "s3:ObjectAcl:Put"
+	EventObjectRestore           EventType = "s3:ObjectRestore:*" // ObjectRestore
+	EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	EventObjectRestoreCompleted  EventType = "s3:ObjectRestore:Completed"
+	// EventObjectRestorePost       EventType = "s3:ObjectRestore:Post"
+	// EventObjectRestoreDelete     EventType = "s3:ObjectRestore:Delete"
+)
+
+func (event EventType) IsValid() bool {
+	_, ok := supportedEventFilters[event]
+	return ok
+}
+
+var supportedEventFilters = map[EventType]struct{}{
+	EventObjectCreated:           {},
+	EventObjectCreatedPut:        {},
+	EventObjectCreatedPost:       {},
+	EventObjectCreatedCopy:       {},
+	EventCompleteMultipartUpload: {},
+	EventObjectDeleted:           {},
+	EventObjectTagging:           {},
+	EventObjectTaggingPut:        {},
+	EventObjectTaggingDelete:     {},
+	EventObjectAclPut:            {},
+	EventObjectRestore:           {},
+	EventObjectRestorePost:       {},
+	EventObjectRestoreCompleted:  {},
+}
+
+type EventFilter map[EventType]bool
+
+func parseEventFilters(path string) (EventFilter, error) {
+	// if no filter config file path is specified return nil map
+	if path == "" {
+		return nil, nil
+	}
+
+	configFilePath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Open the JSON file
+	file, err := os.Open(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var filter EventFilter
+	if err := json.NewDecoder(file).Decode(&filter); err != nil {
+		return nil, err
+	}
+
+	if err := filter.Validate(); err != nil {
+		return nil, err
+	}
+
+	return filter, nil
+}
+
+func (ef EventFilter) Validate() error {
+	for event := range ef {
+		if isValid := event.IsValid(); !isValid {
+			return fmt.Errorf("invalid configuration property: %v", event)
+		}
+	}
+
+	return nil
+}
+
+func (ef EventFilter) Filter(event EventType) bool {
+	ev, found := ef[event]
+	if found {
+		return ev
+	}
+
+	// check wildcard match
+	wildCardEv := EventType(string(event[strings.LastIndex(string(event), ":")+1]) + "*")
+	wildcard, found := ef[wildCardEv]
+	if found {
+		return wildcard
+	}
+
+	return false
+}

s3event/kafka.go

@@ -16,10 +16,8 @@ package s3event
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -32,10 +30,11 @@ var sequencer = 0
 type Kafka struct {
 	key    string
 	writer *kafka.Writer
+	filter EventFilter
 	mu     sync.Mutex
 }
 
-func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
+func InitKafkaEventService(url, topic, key string, filter EventFilter) (S3EventSender, error) {
 	if topic == "" {
 		return nil, fmt.Errorf("kafka message topic should be specified")
 	}
@@ -47,26 +46,19 @@ func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
 		BatchTimeout: 5 * time.Millisecond,
 	})
 
-	msg := map[string]string{
-		"Service": "S3",
-		"Event":   "s3:TestEvent",
-		"Time":    time.Now().Format(time.RFC3339),
-		"Bucket":  "Test-Bucket",
-	}
-
-	msgJSON, err := json.Marshal(msg)
+	msg, err := generateTestEvent()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("kafka generate test event: %w", err)
 	}
 
 	message := kafka.Message{
 		Key:   []byte(key),
-		Value: msgJSON,
+		Value: msg,
 	}
 
-	ctx := context.Background()
-
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	err = w.WriteMessages(ctx, message)
+	cancel()
 	if err != nil {
 		return nil, err
 	}
@@ -74,6 +66,7 @@ func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
 	return &Kafka{
 		key:    key,
 		writer: w,
+		filter: filter,
 	}, nil
 }
 
@@ -81,67 +74,31 @@ func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 	ks.mu.Lock()
 	defer ks.mu.Unlock()
 
-	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
-
-	schema := EventSchema{
-		EventVersion: "2.2",
-		EventSource:  "aws:s3",
-		AwsRegion:    ctx.Locals("region").(string),
-		EventTime:    time.Now().Format(time.RFC3339),
-		EventName:    meta.EventName,
-		UserIdentity: EventUserIdentity{
-			PrincipalId: ctx.Locals("access").(string),
-		},
-		RequestParameters: EventRequestParams{
-			SourceIPAddress: ctx.IP(),
-		},
-		ResponseElements: EventResponseElements{
-			RequestId: ctx.Get("X-Amz-Request-Id"),
-			HostId:    ctx.Get("X-Amx-Id-2"),
-		},
-		S3: EventS3Data{
-			S3SchemaVersion: "1.0",
-			// This field will come up after implementing per bucket notifications
-			ConfigurationId: "kafka-global",
-			Bucket: EventS3BucketData{
-				Name: bucket,
-				OwnerIdentity: EventUserIdentity{
-					PrincipalId: ctx.Locals("access").(string),
-				},
-				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
-			},
-			Object: EventObjectData{
-				Key:       object,
-				Size:      meta.ObjectSize,
-				ETag:      meta.ObjectETag,
-				VersionId: meta.VersionId,
-				Sequencer: genSequencer(),
-			},
-		},
-		GlacierEventData: EventGlacierData{
-			// Not supported
-			RestoreEventData: EventRestoreData{},
-		},
-	}
-
-	ks.send([]EventSchema{schema})
-}
-
-func (ks *Kafka) send(evnt []EventSchema) {
-	msg, err := json.Marshal(evnt)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
+	if ks.filter != nil && !ks.filter.Filter(meta.EventName) {
 		return
 	}
 
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdKafka)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create kafka event: %v\n", err.Error())
+		return
+	}
+
+	go ks.send(schema)
+}
+
+func (ks *Kafka) Close() error {
+	return ks.writer.Close()
+}
+
+func (ks *Kafka) send(event []byte) {
 	message := kafka.Message{
 		Key:   []byte(ks.key),
-		Value: msg,
+		Value: event,
 	}
 
 	ctx := context.Background()
-	err = ks.writer.WriteMessages(ctx, message)
+	err := ks.writer.WriteMessages(ctx, message)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
 	}

s3event/nats.go

@@ -15,12 +15,9 @@
 package s3event
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
-	"time"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/nats-io/nats.go"
@@ -30,9 +27,10 @@ type NatsEventSender struct {
 	topic  string
 	client *nats.Conn
 	mu     sync.Mutex
+	filter EventFilter
 }
 
-func InitNatsEventService(url, topic string) (S3EventSender, error) {
+func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {
 	if topic == "" {
 		return nil, fmt.Errorf("nats message topic should be specified")
 	}
@@ -42,9 +40,20 @@ func InitNatsEventService(url, topic string) (S3EventSender, error) {
 		return nil, err
 	}
 
+	msg, err := generateTestEvent()
+	if err != nil {
+		return nil, fmt.Errorf("nats generate test event: %w", err)
+	}
+
+	err = client.Publish(topic, msg)
+	if err != nil {
+		return nil, fmt.Errorf("nats publish test event: %v", err)
+	}
+
 	return &NatsEventSender{
 		topic:  topic,
 		client: client,
+		filter: filter,
 	}, nil
 }
 
@@ -52,60 +61,26 @@ func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
 	ns.mu.Lock()
 	defer ns.mu.Unlock()
 
-	path := strings.Split(ctx.Path(), "/")
-	bucket, object := path[1], strings.Join(path[2:], "/")
-
-	schema := EventSchema{
-		EventVersion: "2.2",
-		EventSource:  "aws:s3",
-		AwsRegion:    ctx.Locals("region").(string),
-		EventTime:    time.Now().Format(time.RFC3339),
-		EventName:    meta.EventName,
-		UserIdentity: EventUserIdentity{
-			PrincipalId: ctx.Locals("access").(string),
-		},
-		RequestParameters: EventRequestParams{
-			SourceIPAddress: ctx.IP(),
-		},
-		ResponseElements: EventResponseElements{
-			RequestId: ctx.Get("X-Amz-Request-Id"),
-			HostId:    ctx.Get("X-Amx-Id-2"),
-		},
-		S3: EventS3Data{
-			S3SchemaVersion: "1.0",
-			// This field will come up after implementing per bucket notifications
-			ConfigurationId: "nats-global",
-			Bucket: EventS3BucketData{
-				Name: bucket,
-				OwnerIdentity: EventUserIdentity{
-					PrincipalId: ctx.Locals("access").(string),
-				},
-				Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
-			},
-			Object: EventObjectData{
-				Key:       object,
-				Size:      meta.ObjectSize,
-				ETag:      meta.ObjectETag,
-				VersionId: meta.VersionId,
-				Sequencer: genSequencer(),
-			},
-		},
-		GlacierEventData: EventGlacierData{
-			// Not supported
-			RestoreEventData: EventRestoreData{},
-		},
+	if ns.filter != nil && !ns.filter.Filter(meta.EventName) {
+		return
 	}
 
-	ns.send([]EventSchema{schema})
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdNats)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create nats event: %v\n", err.Error())
+		return
+	}
+
+	go ns.send(schema)
 }
 
-func (ns *NatsEventSender) send(evnt []EventSchema) {
-	msg, err := json.Marshal(evnt)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
-	}
+func (ns *NatsEventSender) Close() error {
+	ns.client.Close()
+	return nil
+}
 
-	err = ns.client.Publish(ns.topic, msg)
+func (ns *NatsEventSender) send(event []byte) {
+	err := ns.client.Publish(ns.topic, event)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
 	}

s3event/webhook.go

@@ -0,0 +1,108 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package s3event
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type Webhook struct {
+	url    string
+	client *http.Client
+	filter EventFilter
+	mu     sync.Mutex
+}
+
+func InitWebhookEventSender(url string, filter EventFilter) (S3EventSender, error) {
+	if url == "" {
+		return nil, fmt.Errorf("webhook url should be specified")
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 1,
+	}
+
+	testEv, err := generateTestEvent()
+	if err != nil {
+		return nil, fmt.Errorf("webhook generate test event: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(testEv))
+	if err != nil {
+		return nil, fmt.Errorf("create webhook http request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	_, err = client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			return nil, fmt.Errorf("send webhook test event: %w", err)
+		}
+	}
+
+	return &Webhook{
+		client: &http.Client{
+			Timeout: 3 * time.Second,
+		},
+		url:    url,
+		filter: filter,
+	}, nil
+}
+
+func (w *Webhook) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.filter != nil && !w.filter.Filter(meta.EventName) {
+		return
+	}
+
+	schema, err := createEventSchema(ctx, meta, ConfigurationIdWebhook)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create webhook event: %v\n", err.Error())
+		return
+	}
+
+	go w.send(schema)
+}
+
+func (w *Webhook) Close() error {
+	return nil
+}
+
+func (w *Webhook) send(event []byte) {
+	req, err := http.NewRequest(http.MethodPost, w.url, bytes.NewReader(event))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create webhook event request: %v\n", err.Error())
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	_, err = w.client.Do(req)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && !err.Timeout() {
+			fmt.Fprintf(os.Stderr, "failed to send webhook event: %v\n", err.Error())
+		}
+	}
+}

s3log/file.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -88,9 +89,9 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
 		}
 	}
 
-	switch ctx.Locals("access").(type) {
-	case string:
-		access = ctx.Locals("access").(string)
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner

s3log/webhook.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -85,9 +86,9 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
 		}
 	}
 
-	switch ctx.Locals("access").(type) {
-	case string:
-		access = ctx.Locals("access").(string)
+	switch ctx.Locals("account").(type) {
+	case auth.Account:
+		access = ctx.Locals("account").(auth.Account).Access
 	}
 
 	lf.BucketOwner = meta.BucketOwner

tests/.env.default

@@ -11,4 +11,6 @@ CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
 S3CMD_CONFIG=./tests/s3cfg.local.default
 SECRETS_FILE=./tests/.secrets
-MC_ALIAS=versity
+MC_ALIAS=versity
+LOG_LEVEL=2
+GOCOVERDIR=$PWD/cover

tests/.env.s3.default

@@ -0,0 +1,14 @@
+AWS_PROFILE=versity_s3
+AWS_ENDPOINT_URL=https://127.0.0.1:7070
+VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
+BACKEND=s3
+LOCAL_FOLDER=/tmp/gw
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+#RECREATE_BUCKETS=true
+CERT=$PWD/cert.pem
+KEY=$PWD/versitygw.pem
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets.s3
+MC_ALIAS=versity_s3

tests/README.md

@@ -2,6 +2,8 @@
 
 ## Instructions - Running Locally
 
+### Posix Backend
+
 1. Build the `versitygw` binary.
 2. Install the command-line interface(s) you want to test if unavailable on your machine.  
    * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
@@ -28,7 +30,18 @@
 8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
 9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
+### S3 Backend
+
+Instructions are mostly the same; however, testing with the S3 backend requires two S3 accounts.  Ideally, these are two real accounts, but one can also be a dummy account that versity uses internally.
+
+To set up the latter:
+1. Create a new AWS profile with ID and key values set to dummy 20-char allcaps and 40-char alphabetical values respectively.
+1. In the `.secrets` file being used, create the fields `AWS_ACCESS_KEY_ID_TWO` and `AWS_SECRET_ACCESS_KEY_TWO`.  Set these values to the actual AWS ID and key.  
+2. Set the values for `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` the same dummy values set in the AWS profile, and set `AWS_PROFILE` to the profile you just created.
+3. Create a new AWS profile with these dummy values.  In the `.env` file being used, set the `AWS_PROFILE` parameter to the name of this new profile, and the ID and key fields to the dummy values.  
+4. Set `BACKEND` to `s3`.  Also, change the `MC_ALIAS` value if testing **mc** in this configuration.
+
 ## Instructions - Running With Docker
 
 1.  Create a `.secrets` file in the `tests` folder, and add the `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and the `AWS_PROFILE` fields.
-2.  Build and run the `Dockerfile_test_bats` file.
+2.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to an S3-backend-friendly config.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.

tests/integration/group-tests.go

@@ -242,6 +242,41 @@ func TestGetBucketAcl(s *S3Conf) {
 	GetBucketAcl_success(s)
 }
 
+func TestPutBucketPolicy(s *S3Conf) {
+	PutBucketPolicy_non_existing_bucket(s)
+	PutBucketPolicy_invalid_effect(s)
+	PutBucketPolicy_empty_actions_string(s)
+	PutBucketPolicy_empty_actions_array(s)
+	PutBucketPolicy_invalid_action(s)
+	PutBucketPolicy_unsupported_action(s)
+	PutBucketPolicy_incorrect_action_wildcard_usage(s)
+	PutBucketPolicy_empty_principals_string(s)
+	PutBucketPolicy_empty_principals_array(s)
+	PutBucketPolicy_principals_incorrect_wildcard_usage(s)
+	PutBucketPolicy_non_existing_principals(s)
+	PutBucketPolicy_empty_resources_string(s)
+	PutBucketPolicy_empty_resources_array(s)
+	PutBucketPolicy_invalid_resource_prefix(s)
+	PutBucketPolicy_invalid_resource_with_starting_slash(s)
+	PutBucketPolicy_duplicate_resource(s)
+	PutBucketPolicy_incorrect_bucket_name(s)
+	PutBucketPolicy_object_action_on_bucket_resource(s)
+	PutBucketPolicy_bucket_action_on_object_resource(s)
+	PutBucketPolicy_success(s)
+}
+
+func TestGetBucketPolicy(s *S3Conf) {
+	GetBucketPolicy_non_existing_bucket(s)
+	GetBucketPolicy_default_empty_policy(s)
+	GetBucketPolicy_success(s)
+}
+
+func TestDeleteBucketPolicy(s *S3Conf) {
+	DeleteBucketPolicy_non_existing_bucket(s)
+	DeleteBucketPolicy_remove_before_setting(s)
+	DeleteBucketPolicy_success(s)
+}
+
 func TestFullFlow(s *S3Conf) {
 	TestAuthentication(s)
 	TestPresignedAuthentication(s)
@@ -271,6 +306,10 @@ func TestFullFlow(s *S3Conf) {
 	TestCompleteMultipartUpload(s)
 	TestPutBucketAcl(s)
 	TestGetBucketAcl(s)
+	TestPutBucketPolicy(s)
+	TestGetBucketPolicy(s)
+	TestDeleteBucketPolicy(s)
+	TestAccessControl(s)
 }
 
 func TestPosix(s *S3Conf) {
@@ -287,6 +326,16 @@ func TestIAM(s *S3Conf) {
 	IAM_admin_ChangeBucketOwner(s)
 }
 
+func TestAccessControl(s *S3Conf) {
+	AccessControl_default_ACL_user_access_denied(s)
+	AccessControl_default_ACL_userplus_access_denied(s)
+	AccessControl_default_ACL_admin_successful_access(s)
+	AccessControl_bucket_resource_single_action(s)
+	AccessControl_bucket_resource_all_action(s)
+	AccessControl_single_object_resource_actions(s)
+	AccessControl_multi_statement_policy(s)
+}
+
 type IntTests map[string]func(s *S3Conf) error
 
 func GetIntTests() IntTests {
@@ -445,6 +494,32 @@ func GetIntTests() IntTests {
 		"GetBucketAcl_non_existing_bucket":                      GetBucketAcl_non_existing_bucket,
 		"GetBucketAcl_access_denied":                            GetBucketAcl_access_denied,
 		"GetBucketAcl_success":                                  GetBucketAcl_success,
+		"PutBucketPolicy_non_existing_bucket":                   PutBucketPolicy_non_existing_bucket,
+		"PutBucketPolicy_invalid_effect":                        PutBucketPolicy_invalid_effect,
+		"PutBucketPolicy_empty_actions_string":                  PutBucketPolicy_empty_actions_string,
+		"PutBucketPolicy_empty_actions_array":                   PutBucketPolicy_empty_actions_array,
+		"PutBucketPolicy_invalid_action":                        PutBucketPolicy_invalid_action,
+		"PutBucketPolicy_unsupported_action":                    PutBucketPolicy_unsupported_action,
+		"PutBucketPolicy_incorrect_action_wildcard_usage":       PutBucketPolicy_incorrect_action_wildcard_usage,
+		"PutBucketPolicy_empty_principals_string":               PutBucketPolicy_empty_principals_string,
+		"PutBucketPolicy_empty_principals_array":                PutBucketPolicy_empty_principals_array,
+		"PutBucketPolicy_principals_incorrect_wildcard_usage":   PutBucketPolicy_principals_incorrect_wildcard_usage,
+		"PutBucketPolicy_non_existing_principals":               PutBucketPolicy_non_existing_principals,
+		"PutBucketPolicy_empty_resources_string":                PutBucketPolicy_empty_resources_string,
+		"PutBucketPolicy_empty_resources_array":                 PutBucketPolicy_empty_resources_array,
+		"PutBucketPolicy_invalid_resource_prefix":               PutBucketPolicy_invalid_resource_prefix,
+		"PutBucketPolicy_invalid_resource_with_starting_slash":  PutBucketPolicy_invalid_resource_with_starting_slash,
+		"PutBucketPolicy_duplicate_resource":                    PutBucketPolicy_duplicate_resource,
+		"PutBucketPolicy_incorrect_bucket_name":                 PutBucketPolicy_incorrect_bucket_name,
+		"PutBucketPolicy_object_action_on_bucket_resource":      PutBucketPolicy_object_action_on_bucket_resource,
+		"PutBucketPolicy_bucket_action_on_object_resource":      PutBucketPolicy_bucket_action_on_object_resource,
+		"PutBucketPolicy_success":                               PutBucketPolicy_success,
+		"GetBucketPolicy_non_existing_bucket":                   GetBucketPolicy_non_existing_bucket,
+		"GetBucketPolicy_default_empty_policy":                  GetBucketPolicy_default_empty_policy,
+		"GetBucketPolicy_success":                               GetBucketPolicy_success,
+		"DeleteBucketPolicy_non_existing_bucket":                DeleteBucketPolicy_non_existing_bucket,
+		"DeleteBucketPolicy_remove_before_setting":              DeleteBucketPolicy_remove_before_setting,
+		"DeleteBucketPolicy_success":                            DeleteBucketPolicy_success,
 		"PutObject_overwrite_dir_obj":                           PutObject_overwrite_dir_obj,
 		"PutObject_overwrite_file_obj":                          PutObject_overwrite_file_obj,
 		"PutObject_dir_obj_with_data":                           PutObject_dir_obj_with_data,

tests/integration/tests.go

@@ -3265,28 +3265,35 @@ func CopyObject_not_owned_source_bucket(s *S3Conf) error {
 		}
 
 		usr := user{
-			access: "admin1",
-			secret: "admin1secret",
-			role:   "admin",
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
 		}
 
 		cfg := *s
 		cfg.awsID = usr.access
 		cfg.awsSecret = usr.secret
 
+		userS3Client := s3.NewFromConfig(cfg.Config())
+
 		err = createUsers(s, []user{usr})
 		if err != nil {
 			return err
 		}
 
 		dstBucket := getBucketName()
-		err = setup(&cfg, dstBucket)
+		err = setup(s, dstBucket)
+		if err != nil {
+			return err
+		}
+
+		err = changeBucketsOwner(s, []string{bucket}, usr.access)
 		if err != nil {
 			return err
 		}
 
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-		_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
+		_, err = userS3Client.CopyObject(ctx, &s3.CopyObjectInput{
 			Bucket:     &dstBucket,
 			Key:        getPtr("obj-1"),
 			CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
@@ -5241,6 +5248,869 @@ func GetBucketAcl_success(s *S3Conf) error {
 	})
 }
 
+func PutBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "PutBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_effect(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_effect"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("invalid_effect", `"*"`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid effect: invalid_effect")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_actions_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_actions_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `""`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_actions_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_actions_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `[]`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_action(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"ListObjects"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid action: ListObjects")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_unsupported_action(s *S3Conf) error {
+	testName := "PutBucketPolicy_unsupported_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:PutLifecycleConfiguration"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported action: s3:PutLifecycleConfiguration")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_incorrect_action_wildcard_usage(s *S3Conf) error {
+	testName := "PutBucketPolicy_incorrect_action_wildcard_usage"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:hello*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid wildcard usage: s3:hello prefix is not in the supported actions list")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_principals_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_principals_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `""`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_principals_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_principals_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `[]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_principals_incorrect_wildcard_usage(s *S3Conf) error {
+	testName := "PutBucketPolicy_principals_incorrect_wildcard_usage"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*", "grt1"]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals should either contain * or user access keys")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_non_existing_principals(s *S3Conf) error {
+	testName := "PutBucketPolicy_non_existing_principals"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		apiErr1 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"}))
+		apiErr2 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_2", "a_rarely_existing_user_account_1"}))
+
+		err1 := checkApiErr(err, apiErr1)
+		err2 := checkApiErr(err, apiErr2)
+
+		if err1 != nil && err2 != nil {
+			return err1
+		}
+
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_resources_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_resources_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, `""`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_resources_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_resources_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, `[]`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_resource_prefix(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_resource_prefix"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:iam:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_resource_with_starting_slash(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_resource_with_starting_slash"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::/%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_duplicate_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_duplicate_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, fmt.Sprintf("[%v, %v]", resource, resource))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("duplicate resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_incorrect_bucket_name(s *S3Conf) error {
+	testName := "PutBucketPolicy_incorrect_bucket_name"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::prefix-%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("incorrect bucket name in prefix-%v", bucket))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_object_action_on_bucket_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_object_action_on_bucket_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:PutObjectTagging"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported object action 's3:PutObjectTagging' on the specified resources")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_bucket_action_on_object_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_object_action_on_bucket_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:DeleteBucket"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported bucket action 's3:DeleteBucket' on the specified resources")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_success(s *S3Conf) error {
+	testName := "PutBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		err := createUsers(s, []user{
+			{"grt1", "grt1secret", "user"},
+			{"grt2", "grt2secret", "user"},
+		})
+		if err != nil {
+			return err
+		}
+
+		bucketResource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		objectResource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+
+		for _, doc := range []string{
+			genPolicyDoc("Allow", `["grt1", "grt2"]`, `["s3:DeleteBucket", "s3:GetBucketAcl"]`, bucketResource),
+			genPolicyDoc("Deny", `"*"`, `"s3:DeleteBucket"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)),
+			genPolicyDoc("Allow", `"grt1"`, `["s3:PutBucketVersioning", "s3:ListMultipartUploadParts", "s3:ListBucket"]`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+			genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+			genPolicyDoc("Allow", `"*"`, `"s3:Get*"`, objectResource),
+			genPolicyDoc("Deny", `"*"`, `"s3:Create*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+		} {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+				Bucket: &bucket,
+				Policy: &doc,
+			})
+			cancel()
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func GetBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "GetBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func GetBucketPolicy_default_empty_policy(s *S3Conf) error {
+	testName := "GetBucketPolicy_default_empty_policy"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy != nil {
+			return fmt.Errorf("expected policy to be nil, instead got %s", *out.Policy)
+		}
+
+		return nil
+	})
+}
+
+func GetBucketPolicy_success(s *S3Conf) error {
+	testName := "GetBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `["s3:DeleteBucket", "s3:GetBucketTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy == nil {
+			return fmt.Errorf("expected non nil policy result")
+		}
+
+		if *out.Policy != doc {
+			return fmt.Errorf("expected the bucket policy to be %v, instead got %v", doc, *out.Policy)
+		}
+
+		return nil
+	})
+}
+
+func DeleteBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func DeleteBucketPolicy_remove_before_setting(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_remove_before_setting"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+
+		return err
+	})
+}
+
+func DeleteBucketPolicy_success(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `["s3:DeleteBucket", "s3:GetBucketTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy != nil {
+			return fmt.Errorf("expected policy to be nil, instead got %s", *out.Policy)
+		}
+
+		return err
+	})
+}
+
+// Access control tests (with bucket ACLs and Policies)
+func AccessControl_default_ACL_user_access_denied(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_user_access_denied"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = usr.access
+		cfg.awsSecret = usr.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_default_ACL_userplus_access_denied(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_userplus_access_denied"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr := user{
+			access: "userplus1",
+			secret: "userplus1secret",
+			role:   "userplus",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = usr.access
+		cfg.awsSecret = usr.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_default_ACL_admin_successful_access(s *S3Conf) error {
+	testName := "AccessControl_default_ACL_admin_successful_access"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		admin := user{
+			access: "admin1",
+			secret: "admin1secret",
+			role:   "admin",
+		}
+		err := createUsers(s, []user{admin})
+		if err != nil {
+			return err
+		}
+
+		cfg := *s
+		cfg.awsID = admin.access
+		cfg.awsSecret = admin.secret
+
+		err = putObjects(s3.NewFromConfig(cfg.Config()), []string{"my-obj"}, bucket)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_bucket_resource_single_action(s *S3Conf) error {
+	testName := "AccessControl_bucket_resource_single_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		usr2 := user{
+			access: "grt2",
+			secret: "grt2secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr1, usr2})
+		if err != nil {
+			return err
+		}
+
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:PutBucketTagging"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.DeleteBucketTagging(ctx, &s3.DeleteBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		user2Client := getUserS3Client(usr2, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user2Client.DeleteBucketTagging(ctx, &s3.DeleteBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_bucket_resource_all_action(s *S3Conf) error {
+	testName := "AccessControl_bucket_resource_all_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		usr2 := user{
+			access: "grt2",
+			secret: "grt2secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr1, usr2})
+		if err != nil {
+			return err
+		}
+
+		bucketResource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		objectResource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+		err = putObjects(user1Client, []string{"my-obj"}, bucket)
+		if err != nil {
+			return err
+		}
+
+		user2Client := getUserS3Client(usr2, s)
+
+		err = putObjects(user2Client, []string{"my-obj"}, bucket)
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_single_object_resource_actions(s *S3Conf) error {
+	testName := "AccessControl_single_object_resource_actions"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		obj := "my-obj/nested-obj"
+		err := putObjects(s3client, []string{obj}, bucket)
+		if err != nil {
+			return err
+		}
+
+		usr1 := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err = createUsers(s, []user{usr1})
+		if err != nil {
+			return err
+		}
+
+		doc := genPolicyDoc("Allow", `["grt1"]`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%v/%v"`, bucket, obj))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		user1Client := getUserS3Client(usr1, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: &bucket,
+			Key:    &obj,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = user1Client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func AccessControl_multi_statement_policy(s *S3Conf) error {
+	testName := "AccessControl_multi_statement_policy"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		policy := fmt.Sprintf(`
+		{
+			"Statement": [
+				{
+					"Effect": "Deny",
+					"Principal": ["grt1"],
+					"Action":  "s3:DeleteBucket",
+					"Resource":  "arn:aws:s3:::%s"
+				},
+				{
+					"Effect": "Allow",
+					"Principal": "grt1",
+					"Action": "s3:*",
+					"Resource": ["arn:aws:s3:::%s", "arn:aws:s3:::%s/*"]
+				}
+			]
+		}	
+		`, bucket, bucket, bucket)
+
+		usr := user{
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
+		}
+		err := createUsers(s, []user{usr})
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &policy,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		userClient := getUserS3Client(usr, s)
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = userClient.ListObjects(ctx, &s3.ListObjectsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = userClient.DeleteBucket(ctx, &s3.DeleteBucketInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 // IAM related tests
 // multi-user iam tests
 func IAM_user_access_denied(s *S3Conf) error {
@@ -5252,13 +6122,8 @@ func IAM_user_access_denied(s *S3Conf) error {
 		secret: "grt1secret",
 		role:   "user",
 	}
-	err := deleteUser(s, usr.access)
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
 
-	err = createUsers(s, []user{usr})
+	err := createUsers(s, []user{usr})
 	if err != nil {
 		failF("%v: %v", testName, err)
 		return fmt.Errorf("%v: %w", testName, err)
@@ -5288,13 +6153,8 @@ func IAM_userplus_access_denied(s *S3Conf) error {
 		secret: "grt1secret",
 		role:   "userplus",
 	}
-	err := deleteUser(s, usr.access)
-	if err != nil {
-		failF("%v: %v", testName, err)
-		return fmt.Errorf("%v: %w", testName, err)
-	}
 
-	err = createUsers(s, []user{usr})
+	err := createUsers(s, []user{usr})
 	if err != nil {
 		failF("%v: %v", testName, err)
 		return fmt.Errorf("%v: %w", testName, err)
@@ -5323,12 +6183,8 @@ func IAM_userplus_CreateBucket(s *S3Conf) error {
 			secret: "grt1secret",
 			role:   "userplus",
 		}
-		err := deleteUser(s, usr.access)
-		if err != nil {
-			return err
-		}
 
-		err = createUsers(s, []user{usr})
+		err := createUsers(s, []user{usr})
 		if err != nil {
 			return err
 		}

tests/integration/utils.go

@@ -219,12 +219,17 @@ func checkApiErr(err error, apiErr s3err.APIError) error {
 	}
 	var ae smithy.APIError
 	if errors.As(err, &ae) {
-		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
-			return nil
+		if ae.ErrorCode() != apiErr.Code {
+			return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, ae.ErrorCode())
 		}
 
-		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+		if ae.ErrorMessage() != apiErr.Description {
+			return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, ae.ErrorMessage())
+		}
+
+		return nil
 	}
+
 	return fmt.Errorf("expected aws api error, instead got: %w", err)
 }
 
@@ -532,6 +537,10 @@ type user struct {
 
 func createUsers(s *S3Conf, users []user) error {
 	for _, usr := range users {
+		err := deleteUser(s, usr.access)
+		if err != nil {
+			return err
+		}
 		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
 		if err != nil {
 			return err
@@ -603,3 +612,36 @@ func changeAuthCred(uri, newVal string, index int) (string, error) {
 
 	return urlParsed.String(), nil
 }
+
+func genPolicyDoc(effect, principal, action, resource string) string {
+	jsonTemplate := `
+	{
+		"Statement": [
+			{
+				"Effect":  "%s",
+				"Principal": %s,
+				"Action":  %s,
+				"Resource":  %s
+			}
+		]
+	}
+	`
+
+	return fmt.Sprintf(jsonTemplate, effect, principal, action, resource)
+}
+
+func getMalformedPolicyError(msg string) s3err.APIError {
+	return s3err.APIError{
+		Code:           "MalformedPolicy",
+		Description:    msg,
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func getUserS3Client(usr user, cfg *S3Conf) *s3.Client {
+	config := *cfg
+	config.awsID = usr.access
+	config.awsSecret = usr.secret
+
+	return s3.NewFromConfig(config.Config())
+}

tests/logger.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# levels:  1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace
+
+log() {
+  if [[ $# -ne 2 ]]; then
+    echo "log function requires level, message"
+    return 1
+  fi
+  if [[ $1 -gt $LOG_LEVEL ]]; then
+    return 0
+  fi
+  echo "$2"
+}

tests/s3cfg.local.default

@@ -3,6 +3,7 @@ host_base = 127.0.0.1:7070
 host_bucket = 127.0.0.1:7070
 bucket_location = us-east-1
 use_https = True
+signurl_use_https = True
 
 # Enable S3 v4 signature APIs
 signature_v2 = False

tests/setup.sh

@@ -54,6 +54,11 @@ check_params() {
     echo "RECREATE_BUCKETS must be 'true' or 'false'"
     return 1
   fi
+  if [[ -z "$LOG_LEVEL" ]]; then
+    export LOG_LEVEL=2
+  else
+    export LOG_LEVEL
+  fi
   return 0
 }
 

tests/test_aws.sh

@@ -10,6 +10,19 @@ source ./tests/test_common.sh
   test_common_create_delete_bucket "aws"
 }
 
+@test "test_create_bucket_invalid_name" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "aws" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"Invalid bucket name "* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
 # test adding and removing an object on versitygw
 @test "test_put_object-with-data" {
   test_common_put_object_with_data "aws"
@@ -300,8 +313,8 @@ source ./tests/test_common.sh
   bucket_file_data="test file\n"
 
   create_test_files "$bucket_file" || local created=$?
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
   [[ $created -eq 0 ]] || fail "Error creating test files"
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
@@ -315,3 +328,92 @@ source ./tests/test_common.sh
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
   delete_test_files $bucket_file
 }
+
+@test "test-presigned-url-utf8-chars" {
+  test_common_presigned_url_utf8_chars "aws"
+}
+
+@test "test-list-objects-delimiter" {
+  folder_name="two"
+  object_name="three"
+  create_test_folder "$folder_name" || local created=$?
+  [[ $created -eq 0 ]] || fail "error creating folder"
+  create_test_files "$folder_name"/"$object_name" || created=$?
+  [[ $created -eq 0 ]] || fail "error creating file"
+
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  put_object "aws" "$test_file_folder"/"$folder_name"/"$object_name" "$BUCKET_ONE_NAME"/"$folder_name"/"$object_name" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"
+  prefix=$(echo "${objects[@]}" | jq ".CommonPrefixes[0].Prefix")
+  [[ $prefix == "\""$folder_name/"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"
+  key=$(echo "${objects[@]}" | jq ".Contents[0].Key")
+  [[ $key == "\""$folder_name/$object_name"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $folder_name
+}
+
+# ensure that lists of files greater than a size of 1000 (pagination) are returned properly
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "aws"
+}
+
+#@test "test_filename_length" {
+#  file_name=$(printf "%0.sa" $(seq 1 1025))
+#  echo "$file_name"
+
+#  create_test_files "$file_name" || created=$?
+#  [[ $created -eq 0 ]] || fail "error creating file"
+
+#  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+#  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+#  put_object "aws" "$test_file_folder"/"$file_name" "$BUCKET_ONE_NAME"/"$file_name" || local put_object=$?
+#  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+#}
+
+@test "test_head_bucket" {
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "aws" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_head_bucket_doesnt_exist" {
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_copy_object_aws" {
+
+  bucket_file="bucket_file"
+
+  create_test_files "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  setup_bucket "aws" "$BUCKET_TWO_NAME" || local setup_result_two=$?
+  [[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error"
+  put_object "aws" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  error=$(aws --no-verify-ssl s3api copy-object --copy-source "$BUCKET_ONE_NAME"/"$bucket_file" --key "$bucket_file" --bucket "$BUCKET_TWO_NAME" 2>&1) || local copy_result=$?
+  [[ $copy_result -eq 0 ]] || fail "Error copying file: $error"
+  copy_file "s3://$BUCKET_TWO_NAME"/"$bucket_file" "$test_file_folder/${bucket_file}_copy" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  compare_files "$test_file_folder/$bucket_file" "$test_file_folder/${bucket_file}_copy" || local compare_result=$?
+  [[ $compare_result -eq 0 ]] || file "files don't match"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "aws" "$BUCKET_TWO_NAME"
+  delete_test_files "$bucket_file"
+}

tests/test_common.sh

@@ -148,7 +148,6 @@ test_common_list_objects() {
 }
 
 test_common_set_get_bucket_tags() {
-
   if [[ $# -ne 1 ]]; then
     fail "set/get bucket tags test requires command type"
   fi
@@ -192,7 +191,6 @@ test_common_set_get_bucket_tags() {
 }
 
 test_common_set_get_object_tags() {
-
   if [[ $# -ne 1 ]]; then
     echo "get/set object tags missing command type"
     return 1
@@ -256,3 +254,60 @@ test_common_multipart_upload() {
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
   delete_test_files $bucket_file
 }
+
+test_common_presigned_url_utf8_chars() {
+
+  if [[ $# -ne 1 ]]; then
+    echo "presigned url command missing command type"
+    return 1
+  fi
+
+  local bucket_file="my-$%^&*;"
+  local bucket_file_copy="bucket-file-copy"
+  bucket_file_data="test file\n"
+
+  create_test_files "$bucket_file" || local created=$?
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/"$bucket_file"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || put_result=$?
+  [[ $put_result -eq 0 ]] || fail "Failed to add object $bucket_file"
+
+  create_presigned_url "$1" "$BUCKET_ONE_NAME" "$bucket_file" || presigned_result=$?
+  [[ $presigned_result -eq 0 ]] || fail "presigned url creation failure"
+
+  error=$(curl -k -v "$presigned_url" -o "$test_file_folder"/"$bucket_file_copy") || curl_result=$?
+  if [[ $curl_result -ne 0 ]]; then
+    fail "error downloading file with curl: $error"
+  fi
+  compare_files "$test_file_folder"/"$bucket_file" "$test_file_folder"/"$bucket_file_copy" || compare_result=$?
+  if [[ $compare_result -ne 0 ]]; then
+    echo "file one: $(cat "$test_file_folder"/"$bucket_file")"
+    echo "file two: $(cat "$test_file_folder"/"$bucket_file_copy")"
+    fail "files don't match"
+  fi
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files "$bucket_file" "$bucket_file_copy"
+}
+
+test_common_list_objects_file_count() {
+  if [[ $# -ne 1 ]]; then
+    echo "list objects greater than 1000 missing command type"
+    return 1
+  fi
+  create_test_file_count 1001 || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "error creating test files"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  put_object_multiple "$1" "$test_file_folder/file_*" "$BUCKET_ONE_NAME" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "Failed to copy files to bucket"
+  list_objects "$1" "$BUCKET_ONE_NAME"
+  if [[ $LOG_LEVEL -ge 5 ]]; then
+    log 5 "Array: ${object_array[*]}"
+  fi
+  local file_count="${#object_array[@]}"
+  [[ $file_count == 1001 ]] || fail "file count should be 1001, is $file_count"
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+}

tests/test_mc.sh

@@ -37,3 +37,41 @@ export RUN_MC=true
 @test "test_multipart_upload_mc" {
   test_common_multipart_upload "mc"
 }
+
+@test "test_presigned_url_utf8_chars_mc" {
+  test_common_presigned_url_utf8_chars "mc"
+}
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "mc"
+}
+
+@test "test_create_bucket_invalid_name_mc" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "mc" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"Bucket name cannot be empty"* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_mc" {
+  setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "mc" "$BUCKET_ONE_NAME"
+  [[ $bucket_info == *"$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info"
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_doesnt_exist_mc" {
+  setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "mc" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"does not exist"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME"
+}

tests/test_s3cmd.sh

@@ -31,4 +31,42 @@ export RUN_S3CMD=true
 
 @test "test_multipart_upload_s3cmd" {
   test_common_multipart_upload "s3cmd"
-}
+}
+
+#@test "test_presigned_url_utf8_chars_s3cmd" {
+#  test_common_presigned_url_utf8_chars "s3cmd"
+#}
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3cmd"
+}
+
+@test "test_create_bucket_invalid_name_s3cmd" {
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  create_bucket_invalid_name "s3cmd" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Invalid name test failed"
+
+  [[ "$bucket_create_error" == *"just the bucket name"* ]] || fail "unexpected error:  $bucket_create_error"
+
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_s3cmd" {
+  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "s3cmd" "$BUCKET_ONE_NAME"
+  [[ $bucket_info == *"s3://$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info"
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}
+
+@test "test_get_bucket_info_doesnt_exist_s3cmd" {
+  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  head_bucket "s3cmd" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+}

tests/util.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bats
 
 source ./tests/util_mc.sh
+source ./tests/logger.sh
 
 # create an AWS bucket
 # param:  bucket name
@@ -30,6 +31,29 @@ create_bucket() {
   return 0
 }
 
+create_bucket_invalid_name() {
+  if [ $# -ne 1 ]; then
+    echo "create bucket w/invalid name missing command type"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == "aws" ]]; then
+    bucket_create_error=$(aws --no-verify-ssl s3 mb "s3://" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    bucket_create_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb "s3://" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    bucket_create_error=$(mc --insecure mb "$MC_ALIAS" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -eq 0 ]; then
+    echo "error:  bucket should have not been created but was"
+    return 1
+  fi
+  export bucket_create_error
+}
+
 # delete an AWS bucket
 # param:  bucket name
 # return 0 for success, 1 for failure
@@ -281,6 +305,35 @@ put_object() {
   return 0
 }
 
+put_object_multiple() {
+  if [ $# -ne 3 ]; then
+    echo "put object command requires command type, source, destination"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == 'aws' ]]; then
+    # shellcheck disable=SC2086
+    error=$(aws --no-verify-ssl s3 cp "$(dirname "$2")" s3://"$3" --recursive --exclude="*" --include="$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    # shellcheck disable=SC2086
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put $2 "s3://$3/" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    # shellcheck disable=SC2086
+    error=$(mc --insecure cp $2 "$MC_ALIAS"/"$3" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error copying object to bucket: $error"
+    return 1
+  else
+    log 5 "$error"
+  fi
+  return 0
+}
+
 # add object to versitygw if it doesn't exist
 # params:  source file, destination copy location
 # return 0 for success or already exists, 1 for failure
@@ -394,8 +447,10 @@ list_objects() {
 
   object_array=()
   while IFS= read -r line; do
-    object_name=$(echo "$line" | awk '{print $NF}')
-    object_array+=("$object_name")
+    if [[ $line != *InsecureRequestWarning* ]]; then
+      object_name=$(echo "$line" | awk '{print $NF}')
+      object_array+=("$object_name")
+    fi
   done <<< "$output"
 
   export object_array
@@ -600,11 +655,15 @@ get_object_tags() {
 # param:  bucket
 # export objects on success, return 1 for failure
 list_objects_s3api_v1() {
-  if [ $# -ne 1 ]; then
-    echo "list objects command missing bucket"
+  if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+    echo "list objects command requires bucket, (optional) delimiter"
     return 1
   fi
-  objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+  if [ "$2" == "" ]; then
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
+  else
+    objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
+  fi
   if [[ $result -ne 0 ]]; then
     echo "error listing objects: $objects"
     return 1
@@ -901,3 +960,52 @@ upload_part_copy() {
   etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag')
   export etag
 }
+
+create_presigned_url() {
+  if [[ $# -ne 3 ]]; then
+    echo "create presigned url function requires command type, bucket, and filename"
+    return 1
+  fi
+
+  local presign_result=0
+  if [[ $1 == 'aws' ]]; then
+    presigned_url=$(aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    presigned_url=$(s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    presigned_url_data=$(mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
+    presigned_url="${presigned_url_data#*Share: }"
+  else
+    echo "unrecognized command type $1"
+    return 1
+  fi
+  if [[ $presign_result -ne 0 ]]; then
+    echo "error generating presigned url: $presigned_url"
+    return 1
+  fi
+  export presigned_url
+}
+
+head_bucket() {
+  if [ $# -ne 2 ]; then
+    echo "head bucket command missing command type, bucket name"
+    return 1
+  fi
+  local exit_code=0
+  local error
+  if [[ $1 == "aws" ]]; then
+    bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$?
+  elif [[ $1 == "s3cmd" ]]; then
+    bucket_info=$(s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    echo "error getting bucket info: $bucket_info"
+    return 1
+  fi
+  export bucket_info
+}

tests/util_file.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bats
 
+source ./tests/logger.sh
+
 # create a test file and export folder.  do so in temp folder
 # params:  filename
 # export test file folder on success, return 1 for error
@@ -8,7 +10,7 @@ create_test_files() {
     echo "create test files command missing filename"
     return 1
   fi
-  test_file_folder=.
+  test_file_folder=$PWD
   if [[ -z "$GITHUB_ACTIONS" ]]; then
     create_test_file_folder
   fi
@@ -21,6 +23,23 @@ create_test_files() {
   export test_file_folder
 }
 
+create_test_folder() {
+  if [ $# -lt 1 ]; then
+    echo "create test folder command missing folder name"
+    return 1
+  fi
+  test_file_folder=$PWD
+  if [[ -z "$GITHUB_ACTIONS" ]]; then
+    create_test_file_folder
+  fi
+  for name in "$@"; do
+    mkdir -p "$test_file_folder"/"$name" || local mkdir_result=$?
+    if [[ $mkdir_result -ne 0 ]]; then
+      echo "error creating file $name"
+    fi
+  done
+}
+
 # delete a test file
 # params:  filename
 # return:  0 for success, 1 for error
@@ -34,7 +53,7 @@ delete_test_files() {
     return 1
   fi
   for name in "$@"; do
-    rm "$test_file_folder"/"$name" || rm_result=$?
+    rm -rf "${test_file_folder:?}"/"${name:?}" || rm_result=$?
     if [[ $rm_result -ne 0 ]]; then
       echo "error deleting file $name"
     fi
@@ -80,7 +99,11 @@ compare_files() {
 }
 
 create_test_file_folder() {
-  test_file_folder=${TMPDIR}versity-gwtest
+  if [[ -n $TMPDIR ]]; then
+    test_file_folder=${TMPDIR}versity-gwtest
+  else
+    test_file_folder=$PWD/versity-gwtest
+  fi
   mkdir -p "$test_file_folder" || local mkdir_result=$?
   if [[ $mkdir_result -ne 0 ]]; then
     echo "error creating test file folder"
@@ -97,16 +120,40 @@ create_large_file() {
     return 1
   fi
 
-  test_file_folder=.
+  test_file_folder=$PWD
   if [[ -z "$GITHUB_ACTIONS" ]]; then
     create_test_file_folder
   fi
 
   filesize=$((160*1024*1024))
-  error=$(dd if=/dev/urandom of=$test_file_folder/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$?
+  error=$(dd if=/dev/urandom of="$test_file_folder"/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$?
   if [[ $dd_result -ne 0 ]]; then
     echo "error creating file: $error"
     return 1
   fi
   return 0
 }
+
+create_test_file_count() {
+  if [[ $# -ne 1 ]]; then
+    echo "create test file count function missing bucket name, count"
+    return 1
+  fi
+  test_file_folder=$PWD
+  if [[ -z "$GITHUB_ACTIONS" ]]; then
+    create_test_file_folder
+  fi
+  local touch_result
+  for ((i=1;i<=$1;i++)) {
+    error=$(touch "$test_file_folder/file_$i") || touch_result=$?
+    if [[ $touch_result -ne 0 ]]; then
+      echo "error creating file_$i:  $error"
+      return 1
+    fi
+  }
+  if [[ $LOG_LEVEL -ge 5 ]]; then
+    ls_result=$(ls "$test_file_folder"/file_*)
+    log 5 "$ls_result"
+  fi
+  return 0
+}

tests/util_mc.sh

@@ -21,4 +21,4 @@ delete_bucket_recursive_mc() {
     return 1
   fi
   return 0
-}
+}

tests/versity.sh

@@ -1,5 +1,30 @@
 #!/bin/bash
 
+source ./tests/util_file.sh
+
+check_exe_params_versity() {
+  if [ -z "$LOCAL_FOLDER" ]; then
+    echo "No local storage folder set"
+    return 1
+  elif [ -z "$VERSITY_EXE" ]; then
+    echo "No versity executable location set"
+    return 1
+  elif [ -z "$BACKEND" ]; then
+    echo "No backend parameter set (options: 'posix')"
+    return 1
+  fi
+  if [ "$BACKEND" == 's3' ]; then
+    if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then
+      echo "missing second AWS access key ID for s3 backend"
+      return 1
+    fi
+    if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then
+      echo "missing second AWS secret access key for s3 backend"
+      return 1
+    fi
+  fi
+}
+
 check_exe_params() {
   if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     echo "No AWS access key set"
@@ -20,15 +45,13 @@ check_exe_params() {
     echo "RUN_VERSITYGW must be 'true' or 'false'"
     return 1
   fi
+  if [[ -r $GOCOVERDIR ]]; then
+    export GOCOVERDIR=$GOCOVERDIR
+  fi
   if [[ $RUN_VERSITYGW == "true" ]]; then
-    if [ -z "$LOCAL_FOLDER" ]; then
-      echo "No local storage folder set"
-      return 1
-    elif [ -z "$VERSITY_EXE" ]; then
-      echo "No versity executable location set"
-      return 1
-    elif [ -z "$BACKEND" ]; then
-      echo "No backend parameter set (options: 'posix')"
+    local check_result
+    check_exe_params_versity || check_result=$?
+    if [[ $check_result -ne 0 ]]; then
       return 1
     fi
   fi
@@ -59,32 +82,142 @@ start_versity() {
   fi
 
   if [ "$RUN_VERSITYGW" == "true" ]; then
-    base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY VGW_REGION=$AWS_REGION $VERSITY_EXE"
-    if [ -n "$CERT" ] && [ -n "$KEY" ]; then
-      base_command+=" --cert $CERT --key $KEY"
+    run_versity_app || run_result=$?
+    if [[ $run_result -ne 0 ]]; then
+      echo "error starting versity apps"
+      return 1
     fi
-    base_command+=" $BACKEND $LOCAL_FOLDER &"
-    eval "$base_command"
-    versitygw_pid=$!
-
-    export versitygw_pid VERSITY_EXE BACKEND LOCAL_FOLDER
   fi
 
-  export versitygw_pid AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
+  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
+}
+
+start_versity_process() {
+  if [[ $# -ne 1 ]]; then
+    echo "start versity process function requires number"
+    return 1
+  fi
+  create_test_file_folder || create_result=$?
+  if [[ $create_result -ne 0 ]]; then
+    echo "error creating test log folder"
+    return 1
+  fi
+  base_command+=(">" "$test_file_folder/versity_log_$1.txt" "2>&1")
+  ("${base_command[@]}") &
+  if [[ $? -ne 0 ]]; then
+    echo "error running versitygw command: $(cat "$test_file_folder/versity_log_$1.txt")"
+    return 1
+  fi
+  eval versitygw_pid_"$1"=$!
+  local pid
+  eval pid=\$versitygw_pid_"$1"
+  sleep 1
+
+  local proc_check
+  check_result=$(kill -0 $pid 2>&1) || proc_check=$?
+  if [[ $proc_check -ne 0 ]]; then
+    echo "versitygw failed to start: $check_result"
+    echo "log data: $(cat "$test_file_folder/versity_log_$1.txt")"
+    return 1
+  fi
+  export versitygw_pid_"$1"
+}
+
+run_versity_app_posix() {
+  if [[ $# -ne 3 ]]; then
+    echo "run versity app w/posix command requires access ID, secret key, process number"
+    return 1
+  fi
+  base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION")
+  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
+    base_command+=(--cert "$CERT" --key "$KEY")
+  fi
+  base_command+=(posix "$LOCAL_FOLDER")
+  export base_command
+
+  local versity_result
+  start_versity_process "$3" || versity_result=$?
+  if [[ $versity_result -ne 0 ]]; then
+    echo "error starting versity process"
+    return 1
+  fi
+  return 0
+}
+
+run_versity_app_s3() {
+  if [[ $# -ne 1 ]]; then
+    echo "run versity app w/s3 command requires process number"
+    return 1
+  fi
+  base_command=("$VERSITY_EXE" --port=":7071" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY")
+  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
+    base_command+=(--cert "$CERT" --key "$KEY")
+  fi
+  base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com)
+  export base_command
+
+  local versity_result
+  start_versity_process "$1" || versity_result=$?
+  if [[ $versity_result -ne 0 ]]; then
+    echo "error starting versity process"
+    return 1
+  fi
+  return 0
+}
+
+run_versity_app() {
+  if [[ $BACKEND == 'posix' ]]; then
+    run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
+    if [[ $result_one -ne 0 ]]; then
+      echo "error starting versity app"
+      return 1
+    fi
+  elif [[ $BACKEND == 's3' ]]; then
+    run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
+    if [[ $result_one -ne 0 ]]; then
+      echo "error starting versity app"
+      return 1
+    fi
+    run_versity_app_s3 "2" || result_two=$?
+    if [[ $result_two -ne 0 ]]; then
+      echo "error starting second versity app"
+      return 1
+    fi
+  else
+    echo "unrecognized backend type $BACKEND"
+    return 1
+  fi
+}
+
+stop_single_process() {
+  if [[ $# -ne 1 ]]; then
+    echo "stop single process function requires process ID"
+    return 1
+  fi
+  if ps -p "$1" > /dev/null; then
+    kill "$1"
+    wait "$1" || true
+  else
+    echo "Process with PID $1 does not exist."
+  fi
 }
 
 stop_versity() {
   if [ "$RUN_VERSITYGW" == "false" ]; then
     return
   fi
-  if [ -n "$versitygw_pid" ]; then
-    if ps -p "$versitygw_pid" > /dev/null; then
-      kill "$versitygw_pid"
-      wait "$versitygw_pid" || true
-    else
-      echo "Process with PID $versitygw_pid does not exist."
+  local result_one
+  local result_two
+  # shellcheck disable=SC2154
+  stop_single_process "$versitygw_pid_1" || result_one=$?
+  if [[ $result_one -ne 0 ]]; then
+    echo "error stopping versity process"
+  fi
+  if [[ $BACKEND == 's3' ]]; then
+    # shellcheck disable=SC2154
+    stop_single_process "$versitygw_pid_2" || result_two=$?
+    if [[ $result_two -ne 0 ]]; then
+      echo "error stopping versity process two"
     fi
-  else
-    echo "versitygw_pid is not set or empty."
   fi
 }