Merge pull request #473 from versity/test_cmdline_presign

Test cmdline presign

.github/workflows/system.yml

@@ -38,10 +38,9 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Build and run
+      - name: Build and run, posix backend
         run: |
           make testbin
-          echo $PATH
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
@@ -52,4 +51,23 @@ jobs:
           export WORKSPACE=$GITHUB_WORKSPACE
           openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
+          mkdir /tmp/cover
+          VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+
+      #- name: Build and run, s3 backend
+      #  run: |
+      #    make testbin
+      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+      #    export AWS_REGION=us-east-1
+      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
+      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
+      #    aws configure set aws_region $AWS_REGION --profile versity_s3
+      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
+      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+      #    export WORKSPACE=$GITHUB_WORKSPACE
+      #    VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+
+      - name: Coverage report
+        run: |
+          go tool covdata percent -i=/tmp/cover

.gitignore

@@ -41,7 +41,7 @@ VERSION
 dist/
 
 # secrets file for local github-actions testing
-tests/.secrets
+tests/.secrets*
 
 # IAM users files often created in testing
 users.json

.goreleaser.yaml

@@ -32,11 +32,28 @@ archives:
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
+
+    # Set this to true if you want all files in the archive to be in a single directory.
+    # If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
+    # you'll get a folder 'goreleaser_Linux_arm64'.
+    # If set to false, all files are extracted separately.
+    # You can also set it to a custom folder name (templating is supported).
+    wrap_in_directory: true
+
     # use zip for windows archives
     format_overrides:
     - goos: windows
       format: zip
 
+    # Additional files/globs you want to add to the archive.
+    #
+    # Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
+    # Templates: allowed
+    files:
+      - README.md
+      - LICENSE
+      - NOTICE
+
 checksum:
   name_template: 'checksums.txt'
 
@@ -83,6 +100,27 @@ nfpms:
 
     rpm:
       group: "System Environment/Daemons"
+      # RPM specific scripts.
+      scripts:
+        # The pretrans script runs before all RPM package transactions / stages.
+        #pretrans: ./extra/pretrans.sh
+        # The posttrans script runs after all RPM package transactions / stages.
+        posttrans: ./extra/posttrans.sh
+
+    contents:
+      - src: extra/versitygw@.service
+        dst: /lib/systemd/system/versitygw@.service
+
+      - src: extra/example.conf
+        dst: /etc/versitygw.d/example.conf
+        type: config
+  
+      - dst: /etc/versitygw.d
+        type: dir
+        file_info:
+          mode: 0700
+
+
 
 # yaml-language-server: $schema=https://goreleaser.com/static/schema.json
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile_test_bats

@@ -1,6 +1,9 @@
 FROM --platform=linux/arm64 ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG SECRETS_FILE=tests/.secrets
+ARG CONFIG_FILE=tests/.env.docker
+
 ENV TZ=Etc/UTC
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -12,6 +15,7 @@ RUN apt-get update && \
     tzdata \
     s3cmd \
     jq \
+    bc \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*
@@ -57,11 +61,12 @@ USER tester
 COPY --chown=tester:tester . /home/tester
 
 WORKDIR /home/tester
-RUN cp tests/.env.docker.default tests/.env.docker
+RUN cp ${CONFIG_FILE}.default $CONFIG_FILE
+#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
 RUN cp tests/s3cfg.local.default tests/s3cfg.local
 RUN make
 
-RUN . tests/.secrets && \
+RUN . $SECRETS_FILE && \
     export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
@@ -74,6 +79,6 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
         -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
 
 ENV WORKSPACE=.
-ENV VERSITYGW_TEST_ENV=tests/.env.docker
+ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
 
-CMD ["tests/run_all.sh"]
+CMD ["tests/run_all.sh"]

Makefile

@@ -59,20 +59,13 @@ cleanall: clean
 	rm -f $(BIN)
 	rm -f versitygw-*.tar
 	rm -f versitygw-*.tar.gz
-	rm -f versitygw.spec
-
-%.spec: %.spec.in
-	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
-	mv $@+ $@
 
 TARFILE = $(BIN)-$(VERSION).tar
 
-dist: $(BIN).spec
+dist:
 	echo $(VERSION) >VERSION
 	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
-	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
 	rm -f VERSION
-	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
 
 # Creates and runs S3 gateway instance in a docker container

README.md

@@ -8,17 +8,33 @@
 
  [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-**News:**<br>
+### Binary release builds
+Download [latest release](https://github.com/versity/versitygw/releases)
+ | Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
+ |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
+ |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
+ 
+### News
 * New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
 
+### Mailing List
+Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
 
+### Documentation
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
+### Need help?
+Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
+<br>
+Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
+
+### Use Cases
 * Share filesystem directory via S3 protocol
 * Proxy S3 requests to S3 storage
 * Simple to deploy S3 server with a single command
 * Protocol compatibility in `posix` allows common access to files via posix or S3 
 
+### Overview
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.

auth/acl.go

@@ -15,12 +15,14 @@
 package auth
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -203,13 +205,14 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
-	if isRoot {
-		return nil
-	}
+func verifyACL(acl ACL, access string, permission types.Permission) error {
+	// Default disabled ACL case
+	if acl.ACL == "" && len(acl.Grantees) == 0 {
+		if acl.Owner == access {
+			return nil
+		}
 
-	if acl.Owner == access {
-		return nil
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
 	}
 
 	if acl.ACL != "" {
@@ -273,3 +276,78 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 	// Return access denied in all other cases
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
+
+type AccessOptions struct {
+	Acl           ACL
+	AclPermission types.Permission
+	IsRoot        bool
+	Acc           Account
+	Bucket        string
+	Object        string
+	Action        Action
+}
+
+func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+	if opts.Acc.Access == opts.Acl.Owner {
+		return nil
+	}
+
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+		return err
+	}
+	if err := verifyBucketPolicy(ctx, be, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	// Verify destination bucket access
+	if err := VerifyAccess(ctx, be, opts); err != nil {
+		return err
+	}
+	// Verify source bucket access
+	srcBucket, srcObject, found := strings.Cut(copySource, "/")
+	if !found {
+		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	// Get source bucket ACL
+	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+	if err != nil {
+		return err
+	}
+
+	var srcBucketAcl ACL
+	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+		return err
+	}
+
+	if err := VerifyAccess(ctx, be, AccessOptions{
+		Acl:           srcBucketAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        opts.IsRoot,
+		Acc:           opts.Acc,
+		Bucket:        srcBucket,
+		Object:        srcObject,
+		Action:        GetObjectAction,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}

auth/bucket_policy.go

@@ -0,0 +1,145 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+)
+
+type BucketPolicy struct {
+	Statement []BucketPolicyItem `json:"Statement"`
+}
+
+func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
+	for _, statement := range bp.Statement {
+		err := statement.Validate(bucket, iam)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	for _, statement := range bp.Statement {
+		if statement.isAllowed(principal, action, resource) {
+			return true
+		}
+	}
+
+	return false
+}
+
+type BucketPolicyItem struct {
+	Effect     BucketPolicyAccessType `json:"Effect"`
+	Principals Principals             `json:"Principal"`
+	Actions    Actions                `json:"Action"`
+	Resources  Resources              `json:"Resource"`
+}
+
+func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
+	if err := bpi.Effect.Validate(); err != nil {
+		return err
+	}
+	if err := bpi.Principals.Validate(iam); err != nil {
+		return err
+	}
+	if err := bpi.Resources.Validate(bucket); err != nil {
+		return err
+	}
+
+	containsObjectAction := bpi.Resources.ContainsObjectPattern()
+	containsBucketAction := bpi.Resources.ContainsBucketPattern()
+
+	for action := range bpi.Actions {
+		isObjectAction := action.IsObjectAction()
+		if isObjectAction && !containsObjectAction {
+			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		}
+		if !isObjectAction && !containsBucketAction {
+			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		}
+	}
+
+	return nil
+}
+
+func (bpi *BucketPolicyItem) isAllowed(principal string, action Action, resource string) bool {
+	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
+		switch bpi.Effect {
+		case BucketPolicyAccessTypeAllow:
+			return true
+		case BucketPolicyAccessTypeDeny:
+			return false
+		}
+	}
+
+	return false
+}
+
+func getMalformedPolicyError(err error) error {
+	return s3err.APIError{
+		Code:           "MalformedPolicy",
+		Description:    err.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
+	var policy BucketPolicy
+	if err := json.Unmarshal(policyBin, &policy); err != nil {
+		return getMalformedPolicyError(err)
+	}
+
+	if err := policy.Validate(bucket, iam); err != nil {
+		return getMalformedPolicyError(err)
+	}
+
+	return nil
+}
+
+func verifyBucketPolicy(ctx context.Context, be backend.Backend, access, bucket, object string, action Action) error {
+	policyDoc, err := be.GetBucketPolicy(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	// If bucket policy is not set
+	if len(policyDoc) == 0 {
+		return nil
+	}
+
+	var bucketPolicy BucketPolicy
+	if err := json.Unmarshal(policyDoc, &bucketPolicy); err != nil {
+		return err
+	}
+
+	resource := bucket
+	if object != "" {
+		resource += "" + object
+	}
+
+	if !bucketPolicy.isAllowed(access, action, resource) {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}

auth/bucket_policy_actions.go

@@ -0,0 +1,218 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Action string
+
+const (
+	GetBucketAclAction               Action = "s3:GetBucketAcl"
+	CreateBucketAction               Action = "s3:CreateBucket"
+	PutBucketAclAction               Action = "s3:PutBucketAcl"
+	DeleteBucketAction               Action = "s3:DeleteBucket"
+	PutBucketVersioningAction        Action = "s3:PutBucketVersioning"
+	GetBucketVersioningAction        Action = "s3:GetBucketVersioning"
+	PutBucketPolicyAction            Action = "s3:PutBucketPolicy"
+	GetBucketPolicyAction            Action = "s3:GetBucketPolicy"
+	DeleteBucketPolicyAction         Action = "s3:DeleteBucketPolicy"
+	AbortMultipartUploadAction       Action = "s3:AbortMultipartUpload"
+	ListMultipartUploadPartsAction   Action = "s3:ListMultipartUploadParts"
+	ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
+	PutObjectAction                  Action = "s3:PutObject"
+	GetObjectAction                  Action = "s3:GetObject"
+	DeleteObjectAction               Action = "s3:DeleteObject"
+	GetObjectAclAction               Action = "s3:GetObjectAcl"
+	GetObjectAttributesAction        Action = "s3:GetObjectAttributes"
+	PutObjectAclAction               Action = "s3:PutObjectAcl"
+	RestoreObjectAction              Action = "s3:RestoreObject"
+	GetBucketTaggingAction           Action = "s3:GetBucketTagging"
+	PutBucketTaggingAction           Action = "s3:PutBucketTagging"
+	GetObjectTaggingAction           Action = "s3:GetObjectTagging"
+	PutObjectTaggingAction           Action = "s3:PutObjectTagging"
+	DeleteObjectTaggingAction        Action = "s3:DeleteObjectTagging"
+	ListBucketVersionsAction         Action = "s3:ListBucketVersions"
+	ListBucketAction                 Action = "s3:ListBucket"
+	AllActions                       Action = "s3:*"
+)
+
+var supportedActionList = map[Action]struct{}{
+	GetBucketAclAction:               {},
+	CreateBucketAction:               {},
+	PutBucketAclAction:               {},
+	DeleteBucketAction:               {},
+	PutBucketVersioningAction:        {},
+	GetBucketVersioningAction:        {},
+	PutBucketPolicyAction:            {},
+	GetBucketPolicyAction:            {},
+	DeleteBucketPolicyAction:         {},
+	AbortMultipartUploadAction:       {},
+	ListMultipartUploadPartsAction:   {},
+	ListBucketMultipartUploadsAction: {},
+	PutObjectAction:                  {},
+	GetObjectAction:                  {},
+	DeleteObjectAction:               {},
+	GetObjectAclAction:               {},
+	GetObjectAttributesAction:        {},
+	PutObjectAclAction:               {},
+	RestoreObjectAction:              {},
+	GetBucketTaggingAction:           {},
+	PutBucketTaggingAction:           {},
+	GetObjectTaggingAction:           {},
+	PutObjectTaggingAction:           {},
+	DeleteObjectTaggingAction:        {},
+	ListBucketVersionsAction:         {},
+	ListBucketAction:                 {},
+	AllActions:                       {},
+}
+
+var supportedObjectActionList = map[Action]struct{}{
+	AbortMultipartUploadAction:     {},
+	ListMultipartUploadPartsAction: {},
+	PutObjectAction:                {},
+	GetObjectAction:                {},
+	DeleteObjectAction:             {},
+	GetObjectAclAction:             {},
+	GetObjectAttributesAction:      {},
+	PutObjectAclAction:             {},
+	RestoreObjectAction:            {},
+	GetObjectTaggingAction:         {},
+	PutObjectTaggingAction:         {},
+	DeleteObjectTaggingAction:      {},
+	AllActions:                     {},
+}
+
+// Validates Action: it should either wildcard match with supported actions list or be in it
+func (a Action) IsValid() error {
+	if !strings.HasPrefix(string(a), "s3:") {
+		return fmt.Errorf("invalid action: %v", a)
+	}
+
+	if a == AllActions {
+		return nil
+	}
+
+	if a[len(a)-1] == '*' {
+		pattern := strings.TrimSuffix(string(a), "*")
+		for act := range supportedActionList {
+			if strings.HasPrefix(string(act), pattern) {
+				return nil
+			}
+		}
+
+		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+	}
+
+	_, found := supportedActionList[a]
+	if !found {
+		return fmt.Errorf("unsupported action: %v", a)
+	}
+	return nil
+}
+
+// Checks if the action is object action
+func (a Action) IsObjectAction() bool {
+	if a[len(a)-1] == '*' {
+		pattern := strings.TrimSuffix(string(a), "*")
+		for act := range supportedObjectActionList {
+			if strings.HasPrefix(string(act), pattern) {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	_, found := supportedObjectActionList[a]
+	return found
+}
+
+func (a Action) WildCardMatch(act Action) bool {
+	if strings.HasSuffix(string(a), "*") {
+		pattern := strings.TrimSuffix(string(a), "*")
+		return strings.HasPrefix(string(act), pattern)
+	}
+	return false
+}
+
+type Actions map[Action]struct{}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (a *Actions) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("actions can't be empty")
+		}
+		*a = make(Actions)
+		for _, s := range ss {
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("actions can't be empty")
+			}
+			*a = make(Actions)
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Validates and adds a new Action to Actions map
+func (a Actions) Add(str string) error {
+	action := Action(str)
+	err := action.IsValid()
+	if err != nil {
+		return err
+	}
+
+	a[action] = struct{}{}
+	return nil
+}
+
+func (a Actions) FindMatch(action Action) bool {
+	_, ok := a[AllActions]
+	if ok {
+		return true
+	}
+	// First O(1) check for non wildcard actions
+	_, found := a[action]
+	if found {
+		return true
+	}
+
+	for act := range a {
+		if strings.HasSuffix(string(act), "*") && act.WildCardMatch(action) {
+			return true
+		}
+	}
+
+	return false
+}

auth/bucket_policy_effect.go

@@ -0,0 +1,34 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import "fmt"
+
+type BucketPolicyAccessType string
+
+const (
+	BucketPolicyAccessTypeDeny  BucketPolicyAccessType = "Deny"
+	BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
+)
+
+// Checks policy statement Effect to be valid ("Deny", "Allow")
+func (bpat BucketPolicyAccessType) Validate() error {
+	switch bpat {
+	case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
+		return nil
+	}
+
+	return fmt.Errorf("invalid effect: %v", bpat)
+}

auth/bucket_policy_principals.go

@@ -0,0 +1,97 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type Principals map[string]struct{}
+
+func (p Principals) Add(key string) {
+	p[key] = struct{}{}
+}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (p *Principals) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("principals can't be empty")
+		}
+		*p = make(Principals)
+		for _, s := range ss {
+			p.Add(s)
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("principals can't be empty")
+			}
+			*p = make(Principals)
+			p.Add(s)
+		}
+	}
+
+	return err
+}
+
+// Converts Principals map to a slice, by omitting "*"
+func (p Principals) ToSlice() []string {
+	principals := []string{}
+	for p := range p {
+		if p == "*" {
+			continue
+		}
+		principals = append(principals, p)
+	}
+
+	return principals
+}
+
+// Validates Principals by checking user account access keys existence
+func (p Principals) Validate(iam IAMService) error {
+	_, containsWildCard := p["*"]
+	if containsWildCard {
+		if len(p) == 1 {
+			return nil
+		}
+		return fmt.Errorf("principals should either contain * or user access keys")
+	}
+
+	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return fmt.Errorf("user accounts don't exist: %v", accs)
+	}
+
+	return nil
+}
+
+func (p Principals) Contains(userAccess string) bool {
+	// "*" means it matches for any user account
+	_, ok := p["*"]
+	if ok {
+		return true
+	}
+
+	_, found := p[userAccess]
+	return found
+}

auth/bucket_policy_resources.go

@@ -0,0 +1,142 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Resources map[string]struct{}
+
+const ResourceArnPrefix = "arn:aws:s3:::"
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (r *Resources) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("resources can't be empty")
+		}
+		*r = make(Resources)
+		for _, s := range ss {
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("resources can't be empty")
+			}
+			*r = make(Resources)
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Adds and validates a new resource to Resources map
+func (r Resources) Add(rc string) error {
+	ok, pattern := isValidResource(rc)
+	if !ok {
+		return fmt.Errorf("invalid resource: %v", rc)
+	}
+
+	_, found := r[pattern]
+	if found {
+		return fmt.Errorf("duplicate resource: %v", rc)
+	}
+
+	r[pattern] = struct{}{}
+
+	return nil
+}
+
+// Checks if the resources contain object pattern
+func (r Resources) ContainsObjectPattern() bool {
+	for resource := range r {
+		if resource == "*" || strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Checks if the resources contain bucket pattern
+func (r Resources) ContainsBucketPattern() bool {
+	for resource := range r {
+		if resource == "*" || !strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
+func (r Resources) Validate(bucket string) error {
+	for resource := range r {
+		if !strings.HasPrefix(resource, bucket) {
+			return fmt.Errorf("incorrect bucket name in %v", resource)
+		}
+	}
+
+	return nil
+}
+
+func (r Resources) FindMatch(resource string) bool {
+	for res := range r {
+		if strings.HasSuffix(res, "*") {
+			pattern := strings.TrimSuffix(res, "*")
+			if strings.HasPrefix(resource, pattern) {
+				return true
+			}
+		} else {
+			if res == resource {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// Checks the resource to have arn prefix and not starting with /
+func isValidResource(rc string) (isValid bool, pattern string) {
+	if !strings.HasPrefix(rc, ResourceArnPrefix) {
+		return false, ""
+	}
+
+	res := strings.TrimPrefix(rc, ResourceArnPrefix)
+	if res == "" {
+		return false, ""
+	}
+	// The resource can't start with / (bucket name comes first)
+	if strings.HasPrefix(res, "/") {
+		return false, ""
+	}
+
+	return true, res
+}

backend/azure/azure.go

@@ -466,16 +466,6 @@ func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3
 		return nil, azureErrToS3Err(err)
 	}
 
-	dstContainerAcl, err := getAclFromMetadata(res.Metadata, aclKeyCapital)
-	if err != nil {
-		return nil, err
-	}
-
-	err = auth.VerifyACL(*dstContainerAcl, *input.ExpectedBucketOwner, types.PermissionWrite, false)
-	if err != nil {
-		return nil, err
-	}
-
 	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource && isMetaSame(res.Metadata, input.Metadata) {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
 	}

backend/backend.go

@@ -42,6 +42,8 @@ type Backend interface {
 	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
+	DeleteBucketPolicy(_ context.Context, bucket string) error
+
 	// multipart operations
 	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
@@ -125,6 +127,9 @@ func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, poli
 func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
 func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)

backend/posix/posix.go

@@ -61,6 +61,7 @@ const (
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
 	aclkey              = "user.acl"
 	etagkey             = "user.etag"
+	policykey           = "user.policy"
 )
 
 func New(rootdir string) (*Posix, error) {
@@ -74,6 +75,12 @@ func New(rootdir string) (*Posix, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
+	_, err = xattr.FGet(f, "user.test")
+	if errors.Is(err, syscall.ENOTSUP) {
+		f.Close()
+		return nil, fmt.Errorf("xattr not supported on %v", rootdir)
+	}
+
 	return &Posix{rootfd: f, rootdir: rootdir}, nil
 }
 
@@ -1422,7 +1429,6 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 	}
 	dstBucket := *input.Bucket
 	dstObject := *input.Key
-	owner := *input.ExpectedBucketOwner
 
 	_, err := os.Stat(srcBucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1440,22 +1446,6 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
-	dstBucketACLBytes, err := xattr.Get(dstBucket, aclkey)
-	if err != nil {
-		return nil, fmt.Errorf("get dst bucket acl tag: %w", err)
-	}
-
-	var dstBucketACL auth.ACL
-	err = json.Unmarshal(dstBucketACLBytes, &dstBucketACL)
-	if err != nil {
-		return nil, fmt.Errorf("parse dst bucket acl: %w", err)
-	}
-
-	err = auth.VerifyACL(dstBucketACL, owner, types.PermissionWrite, false)
-	if err != nil {
-		return nil, err
-	}
-
 	objPath := filepath.Join(srcBucket, srcObject)
 	f, err := os.Open(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1851,6 +1841,58 @@ func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object string)
 	return p.PutObjectTagging(ctx, bucket, object, nil)
 }
 
+func (p *Posix) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if policy == nil {
+		if err := xattr.Remove(bucket, policykey); err != nil {
+			if isNoAttr(err) {
+				return nil
+			}
+
+			return fmt.Errorf("remove policy: %w", err)
+		}
+
+		return nil
+	}
+
+	if err := xattr.Set(bucket, policykey, policy); err != nil {
+		return fmt.Errorf("set policy: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Posix) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	policy, err := xattr.Get(bucket, policykey)
+	if isNoAttr(err) {
+		return []byte{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get bucket policy: %w", err)
+	}
+
+	return policy, nil
+}
+
+func (p *Posix) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	return p.PutBucketPolicy(ctx, bucket, nil)
+}
+
 func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {

backend/scoutfs/scoutfs.go

@@ -139,7 +139,10 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	partsize := int64(0)
 	var totalsize int64
 	for i, p := range parts {
-		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
+		if p.PartNumber == nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
 		fi, err := os.Lstat(partPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
@@ -178,9 +181,9 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	defer f.cleanup()
 
 	for _, p := range parts {
-		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
+		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
+			return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
@@ -189,7 +192,7 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
+			return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
 		}
 	}
 
@@ -454,6 +457,13 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, err
 	}
 
+	objSize := fi.Size()
+	if fi.IsDir() {
+		// directory objects are always 0 len
+		objSize = 0
+		length = 0
+	}
+
 	if length == -1 {
 		length = fi.Size() - startOffset + 1
 	}
@@ -462,6 +472,11 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
+	var contentRange string
+	if acceptRange != "" {
+		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
+	}
+
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
@@ -519,6 +534,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		Metadata:        userMetaData,
 		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
+		ContentRange:    &contentRange,
 	}, nil
 }
 

cmd/versitygw/main.go

@@ -197,13 +197,13 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
-			EnvVars:     []string{"LOGFILE"},
+			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
 			Destination: &accessLog,
 		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
-			EnvVars:     []string{"WEBHOOK"},
+			EnvVars:     []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"},
 			Destination: &logWebhookURL,
 		},
 		&cli.StringFlag{
@@ -369,6 +369,10 @@ func initFlags() []cli.Flag {
 }
 
 func runGateway(ctx context.Context, be backend.Backend) error {
+	if rootUserAccess == "" || rootUserSecret == "" {
+		return fmt.Errorf("root user access and secret key must be provided")
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:           "versitygw",
 		ServerHeader:      "VERSITYGW",
@@ -492,7 +496,6 @@ Loop:
 	for {
 		select {
 		case <-ctx.Done():
-			err = ctx.Err()
 			break Loop
 		case err = <-c:
 			break Loop
@@ -512,12 +515,18 @@ Loop:
 
 	err = iam.Shutdown()
 	if err != nil {
+		if saveErr == nil {
+			saveErr = err
+		}
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
 	if logger != nil {
 		err := logger.Shutdown()
 		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
 			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
 		}
 	}

cmd/versitygw/s3.go

@@ -44,6 +44,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server access key id",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_ACCESS_KEY"},
 				Destination: &s3proxyAccess,
 				Aliases:     []string{"a"},
 			},
@@ -52,6 +53,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server secret access key",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_SECRET_KEY"},
 				Destination: &s3proxySecret,
 				Aliases:     []string{"s"},
 			},
@@ -59,23 +61,27 @@ to an s3 storage backend service.`,
 				Name:        "endpoint",
 				Usage:       "s3 service endpoint, default AWS if not specified",
 				Value:       "",
+				EnvVars:     []string{"VGW_S3_ENDPOINT"},
 				Destination: &s3proxyEndpoint,
 			},
 			&cli.StringFlag{
 				Name:        "region",
 				Usage:       "s3 service region, default 'us-east-1' if not specified",
 				Value:       "us-east-1",
+				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
+				EnvVars:     []string{"VGW_S3_SSL_SKIP_VERIFY"},
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
@@ -83,6 +89,7 @@ to an s3 storage backend service.`,
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DEBUG"},
 				Destination: &s3proxyDebug,
 			},
 		},

cmd/versitygw/scoutfs.go

@@ -48,6 +48,7 @@ move interfaces as well as support for tiered filesystems.`,
 				Name:        "glacier",
 				Usage:       "enable glacier emulation mode",
 				Aliases:     []string{"g"},
+				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
 		},

extra/example.conf

@@ -0,0 +1,272 @@
+###################################
+# VersityGW systemd configuration #
+###################################
+
+# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
+# For example, if the service name is "mygateway", then the file should be
+# named /etc/versitygw.d/mygateway.conf.
+# The systemd template file /lib/systemd/system/versitygw@.service will
+# automatically load the configuration file for the service name.
+# To start the gateway, use the following command:
+# systemctl start versitygw@mygateway
+# To enable the gateway to start on boot, use the following command:
+# systemctl enable versitygw@mygateway
+# To stop the gateway, use the following command:
+# systemctl stop versitygw@mygateway
+
+# There can be multiple gateway services running on the same host. Each
+# gateway service must have a unique service name with a unique configuration
+# file in /etc/versitygw.d/. They must also listen on different ports and/or
+# interfaces using the VGW_PORT option.
+
+##############################
+# VersityGW Required Options #
+##############################
+
+# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3
+# This defines the backend that the VGW will use for data access.
+VGW_BACKEND=posix
+
+# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
+# as the the top level directory for the gateway.
+# All sub directories of the top level directory are treated as buckets,
+# and all files/directories below the "bucket directory" are treated as
+# the objects. The object name is split on "/" separator to translate
+# to posix storage.
+# For example:
+# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+VGW_BACKEND_ARG=
+
+############################
+# VersityGW Global Options #
+############################
+
+# commented options are the default values
+
+# The following must be set, and do not have default values
+# The access and secret options will specify the root account credentials.
+# The root account is granted full authorization to all API requests after
+# authentication.
+ROOT_ACCESS_KEY_ID=
+ROOT_SECRET_ACCESS_KEY=
+
+# The following are optional, and have the default values as listed
+
+# The VGW_PORT option will specify the listening port for the S3 server.
+# This option can use either the form <ip>:<port> which will listen only
+# on the network interface that matches the IP on the specified port, or
+# :<port> which will listen on all network interfaces on the specified port.
+# The <ip> spec can either be IP dotted notation or a resolvable hostname.
+# The <port> spec can either be a numeric port or the service name typically
+# in /etc/services.
+#VGW_PORT=:7070
+
+# The VGW_REGION option will specify the region that the S3 server will
+# report to clients. This option is optional, and defaults to "us-east-1".
+#VGW_REGION=us-east-1
+
+# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
+# private key that the S3 server will use for SSL connections. This option
+# is optional, and defaults to not using SSL.
+#VGW_CERT=
+#VGW_KEY=
+
+# The VGW_ADMIN_PORT option will specify the listening port for the admin
+# server. The admin server endpoint can optionally be set to listen on a
+# different interface or port than the S3 service. This allows for better
+# control of firewall restrictions to the admin endpoint. The certs for this
+# can be different certs than specified for the S3 service. The default when
+# these are not specified is to have the admin server listen on the same
+# endpoint as the S3 service.
+# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
+# server will use SSL.
+#VGW_ADMIN_PORT=
+#VGW_ADMIN_CERT=
+#VGW_ADMIN_CERT_KEY=
+
+# The VGW_QUIET option when set will supress the S3 server request summary
+# logging to stdout.
+#VGW_QUIET=false
+
+# The VGW_HEALTH option when set will specify the URL to accept health checks
+# on. The health check endpoint is often used for load balancers to verify
+# gateway is alive. The health endpoint masks any bucket with this setting.
+# For example, if the health endpoint is set to /health, the gateway will not
+# allow creating or listing contents of a bucket called "health". The health
+# endpoint is unauthenticated, and returns a 200 status for GET.
+#VGW_HEALTH=
+
+###############
+# Access Logs #
+###############
+
+# The VGW_ACCESS_LOG option when set will specify the file to log all S3
+# server requests to. This option is optional, and defaults to not logging.
+# It is suggested to use absolute paths for the server log file because the
+# server may chdir into the backend root directory and change locations for
+# relative paths.
+# The log file format follows the AWS S3 access log format documented in
+# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
+#VGW_ACCESS_LOG=
+
+# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
+# S3 server request access logs to. The access logs are JSON encoded when
+# sent to the webhook.
+#VGW_LOG_WEBHOOK_URL=
+
+##############
+# Event Logs #
+##############
+
+# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
+# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
+# bucket events will be sent to the kafka service. The gateway events are
+# similar to AWS S3 events, and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_KAFKA_URL=
+#VGW_EVENT_KAFKA_TOPIC=
+#VGW_EVENT_KAFKA_KEY=
+
+# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
+# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the
+# the NATS messaging service. The gateway events are  similar to AWS S3 events,
+# and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_NATS_URL=
+#VGW_EVENT_NATS_TOPIC=
+
+# The VGW_DEBUG option enables verbose debug log output to stdout. This output
+# includes details for signature verification steps. This is generally only
+# useful for debugging the S3 server, and should not be used in production.
+#VGW_DEBUG=false
+
+################
+# IAM services #
+################
+
+# The VGW_IAM_DIR option will enable the internal IAM service with accounts
+# stored in a file under the specified directory. This is provided to minimize
+# dependencies on outside services for basic functionality. The local account
+# files are plain text and only protected with file permissions. This IAM
+# service is added for convenience, but is not considered as secure or scalable
+# as a dedicated IAM service.
+#VGW_IAM_DIR=
+
+# The ldap options will enable the LDAP IAM service with accounts stored in an
+# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
+# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
+# secret credentials and role respectively. The other options are used to
+# connect to the LDAP service.
+#VGW_IAM_LDAP_URL=
+#VGW_IAM_LDAP_BASE_DN=
+#VGW_IAM_LDAP_BIND_DN=
+#VGW_IAM_LDAP_BIND_PASS=
+#VGW_IAM_LDAP_QUERY_BASE=
+#VGW_IAM_LDAP_OBJECT_CLASSES=
+#VGW_IAM_LDAP_ACCESS_ATR=
+#VGW_IAM_LDAP_SECRET_ATR=
+#VGW_IAM_LDAP_ROLE_ATR=
+
+# The VGW_S3 IAM service is similar to the internal IAM service, but instead
+# stores the account information JSON encoded in an S3 object. This should use
+# a bucket that is not accessible to general users when using s3 backend to
+# prevent access to account credentials. This IAM service is added for
+# convenience, but is not considered as secure or scalable as a dedicated IAM
+# service.
+#VGW_S3_IAM_ACCESS_KEY=
+#VGW_S3_IAM_SECRET_KEY=
+#VGW_S3_IAM_REGION=
+#VGW_S3_IAM_ENDPOINT=
+#VGW_S3_IAM_BUCKET=
+#VGW_S3_IAM_NO_VERIFY=
+
+###############
+# IAM caching #
+###############
+
+# The IAM cache is intended to ease the load on the IAM service and increase
+# the Gateway performance by caching accounts and credentials for the TTL time
+# interval. Disabling this will cause a request to the configured IAM service
+# for each incoming request to retrieve the corresponding account credentials.
+# The cache is enabled by default. The TTL specifies how long to cache
+# credentials, and the prune value determines the interval for expired entries
+# to be removed from the cache. Increasing the TTL may lessen the load on the
+# IAM service backend, but may have out of date account info until the next
+# interval. Increasing the prune value may reduce memory use at the cost of
+# added CPU to check cache expirations.
+#VGW_IAM_CACHE_DISABLE=false
+#VGW_IAM_CACHE_TTL=120
+#VGW_IAM_CACHE_PRUNE=3600
+
+######################################
+# VersityGW Backend Specific Options #
+######################################
+
+#########
+# posix #
+#########
+
+# The posix backend translates S3 requests to file access in a local filesystem.
+# The posix backend requires a filesystem that supports extended attributes.
+# The top level directory for the gateway must be provided. All sub directories
+# of the top level directory are treated as buckets, and all files/directories
+# below the "bucket directory" are treated as the objects. The object
+# name is split on "/" separator to translate to posix storage.
+# For example:
+# top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+
+# There are currently no further options other than VGW_BACKEND_ARG for the
+# posix backend.
+
+###########
+# scoutfs #
+###########
+
+# The scoutfs backend requires a ScoutFS filesystem type for the backend
+# path. The glacier mode functionality requires ScoutAM to be configured
+# for tiering data from the ScoutFS filesystem to a mass stroage system.
+# The mass storage system is often one or more tape libraries. Due to the
+# high latency of tape, the glacier mode functionality is designed to
+# give feedback to clients about object state and offer ability to request
+# data to be staged back to disk without the client dealing with long
+# request timeout settings.
+
+# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
+# GET object:  if file offline, return invalid object state
+# HEAD object: if file offline, set obj storage class to GLACIER
+#              if file offline and staging, x-amz-restore: ongoing-request="true"
+#              if file offline and not staging, x-amz-restore: ongoing-request="false"
+#              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
+#              note: this expiry-date is not used but provided for client glacier compatibility
+# ListObjects: if file offline, set obj storage class to GLACIER
+# RestoreObject: add batch stage request to file
+#VGW_SCOUTFS_GLACIER=false
+
+######
+# s3 #
+######
+
+# The s3 backend allows the gateway to forward requests to an S3 compatible
+# service. This allows the gateway to act as a proxy for another S3 service.
+# The backend S3 access is all done with a single configured account. The
+# gateway will manage incoming multi-tenant access with the gateway configured
+# IAM service. This gives stroage admins the ability to manage local gateway
+# accounts while maintaining full control and a single account for the backend
+# S3 service.
+
+# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
+# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
+# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
+#VGW_S3_ACCESS_KEY=
+#VGW_S3_SECRET_KEY=
+#VGW_S3_REGION=
+#VGW_S3_ENDPOINT=
+#VGW_S3_DISABLE_CHECKSUM=false
+#VGW_S3_SSL_SKIP_VERIFY=false
+#VGW_S3_DEBUG=false

extra/posttrans.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+systemctl daemon-reload

extra/versitygw@.service

@@ -0,0 +1,33 @@
+[Unit]
+Description=VersityGW
+Documentation=https://github.com/versity/versitygw/wiki
+Wants=network-online.target
+After=network-online.target remote-fs.target
+AssertFileIsExecutable=/usr/bin/versitygw
+AssertPathExists=/etc/versitygw.d/%i.conf
+
+[Service]
+WorkingDirectory=/root
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=versitygw-%i
+
+User=root
+Group=root
+
+EnvironmentFile=/etc/versitygw.d/%i.conf
+
+ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"'
+
+# Let systemd restart this service always
+Restart=always
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65536
+
+# Specifies the maximum number of threads this process can create
+TasksMax=infinity
+
+[Install]
+WantedBy=multi-user.target
+

go.mod

@@ -6,14 +6,14 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
-	github.com/aws/aws-sdk-go-v2 v1.25.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4
+	github.com/aws/aws-sdk-go-v2 v1.26.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
 	github.com/aws/smithy-go v1.20.1
 	github.com/go-ldap/ldap/v3 v3.4.6
-	github.com/gofiber/fiber/v2 v2.52.2
+	github.com/gofiber/fiber/v2 v2.52.3
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.33.1
+	github.com/nats-io/nats.go v1.34.0
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/urfave/cli/v2 v2.27.1
@@ -26,11 +26,11 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -47,16 +47,16 @@ require (
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.7
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.7
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.9
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.9
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -16,42 +16,42 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.25.3 h1:xYiLpZTQs1mzvz5PaI6uR0Wh57ippuEthxS4iK5v0n0=
-github.com/aws/aws-sdk-go-v2 v1.25.3/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
+github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
+github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
-github.com/aws/aws-sdk-go-v2/config v1.27.7 h1:JSfb5nOQF01iOgxFI5OIKWwDiEXWTyTgg1Mm1mHi0A4=
-github.com/aws/aws-sdk-go-v2/config v1.27.7/go.mod h1:PH0/cNpoMO+B04qET699o5W92Ca79fVtbUnvMIZro4I=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.7 h1:WJd+ubWKoBeRh7A5iNMnxEOs982SyVKOJD+K8HIezu4=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.7/go.mod h1:UQi7LMR0Vhvs+44w5ec8Q+VS+cd10cjwgHwiVkE0YGU=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 h1:p+y7FvkK2dxS+FEwRIDHDe//ZX+jDhP8HHE50ppj4iI=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3/go.mod h1:/fYB+FZbDlwlAiynK9KDXlzZl3ANI9JkD0Uhz5FjNT4=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 h1:vXY/Hq1XdxHBIYgBUmug/AbMyIe1AKulPYS2/VE1X70=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9/go.mod h1:GyJJTZoHVuENM4TeJEl5Ffs4W9m19u+4wKJcDi/GZ4A=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 h1:ifbIbHZyGl1alsAhPIYsHOg5MuApgqOvVeI8wIugXfs=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3/go.mod h1:oQZXg3c6SNeY6OZrDY+xHcF4VGIEoNotX2B4PrDeoJI=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 h1:Qvodo9gHG9F3E8SfYOspPeBt0bjSbsevK8WhRAUHcoY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3/go.mod h1:vCKrdLXtybdf/uQd/YfVR2r5pcbNuEYKzMQpcxmeSJw=
+github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg=
+github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 h1:F+PUZee9mlfpEJVZdgyewRumKekS9O3fftj8fEMt0rQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13/go.mod h1:Rl7i2dEWGHGsBIJCpUxlRt7VwK/HyXxICxdvIRssQHE=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 h1:mDnFOE2sVkyphMWtTH+stv0eW3k0OTx94K63xpxHty4=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3/go.mod h1:V8MuRVcCRt5h1S+Fwu8KbC7l/gBGo3yBAyUbJM2IJOk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 h1:mbWNpfRUTT6bnacmvOTKXZjR/HycibdWzNpfbrbLDIs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5/go.mod h1:FCOPWGjsshkkICJIn9hq9xr6dLKtyaWpuUojiN3W1/8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 h1:K/NXvIftOlX+oGgWGIa3jDyYLDNsdVhsjHmsBH2GLAQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5/go.mod h1:cl9HGLV66EnCmMNzq4sYOti+/xo8w34CsgzVtm2GgsY=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 h1:4t+QEX7BsXz98W8W1lNvMAG+NX8qHz2CjLBxQKku40g=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3/go.mod h1:oFcjjUq5Hm09N9rpxTdeMeLeQcxS7mIkBkL8qUKng+A=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 h1:lW5xUzOPGAMY7HPuNF4FdyBwRc3UJ/e8KsapbesVeNU=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4/go.mod h1:MGTaf3x/+z7ZGugCGvepnx2DS6+caCYYqKhzVoLNYPk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 h1:XOPfar83RIRPEzfihnp+U6udOveKZJvPQ76SKWrLRHc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.2/go.mod h1:Vv9Xyk1KMHXrR3vNQe8W5LMFdTjSeWk0gBZBzvf3Qa0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 h1:pi0Skl6mNl2w8qWZXcdOyg197Zsf4G97U7Sso9JXGZE=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2/go.mod h1:JYzLoEVeLXk+L4tn1+rrkfhkxl6mLDEVaDSvGq9og90=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 h1:Ppup1nVNAOWbBOrcoOxaxPeEnSFB2RnnQdguhXpmeQk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.4/go.mod h1:+K1rNPVyGxkRuv9NNiaZ4YhBFuyw2MMA9SlIJ1Zlpz8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
 github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
@@ -65,8 +65,8 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
 github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
-github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo=
-github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/fiber/v2 v2.52.3 h1:bgAZwPv0aHIfRwIUdkWhg6U8D3MEYnoJjT+HfW/dDTo=
+github.com/gofiber/fiber/v2 v2.52.3/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
 github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -90,8 +90,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
-github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk=
+github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=

s3api/controllers/backend_moq_test.go

@@ -44,6 +44,9 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketPolicy method")
+//			},
 //			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketTagging method")
 //			},
@@ -53,7 +56,7 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error {
 //				panic("mock out the DeleteObjectTagging method")
 //			},
-//			DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+//			DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 //				panic("mock out the DeleteObjects method")
 //			},
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
@@ -174,6 +177,9 @@ type BackendMock struct {
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
+	// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
+	DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketTaggingFunc mocks the DeleteBucketTagging method.
 	DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -331,6 +337,13 @@ type BackendMock struct {
 			// DeleteBucketInput is the deleteBucketInput argument value.
 			DeleteBucketInput *s3.DeleteBucketInput
 		}
+		// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
+		DeleteBucketPolicy []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketTagging holds details about calls to the DeleteBucketTagging method.
 		DeleteBucketTagging []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -585,6 +598,7 @@ type BackendMock struct {
 	lockCreateBucket            sync.RWMutex
 	lockCreateMultipartUpload   sync.RWMutex
 	lockDeleteBucket            sync.RWMutex
+	lockDeleteBucketPolicy      sync.RWMutex
 	lockDeleteBucketTagging     sync.RWMutex
 	lockDeleteObject            sync.RWMutex
 	lockDeleteObjectTagging     sync.RWMutex
@@ -881,6 +895,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketPolicy calls DeleteBucketPolicyFunc.
+func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketPolicyFunc == nil {
+		panic("BackendMock.DeleteBucketPolicyFunc: method is nil but Backend.DeleteBucketPolicy was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketPolicy.Lock()
+	mock.calls.DeleteBucketPolicy = append(mock.calls.DeleteBucketPolicy, callInfo)
+	mock.lockDeleteBucketPolicy.Unlock()
+	return mock.DeleteBucketPolicyFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketPolicyCalls gets all the calls that were made to DeleteBucketPolicy.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketPolicyCalls())
+func (mock *BackendMock) DeleteBucketPolicyCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketPolicy.RLock()
+	calls = mock.calls.DeleteBucketPolicy
+	mock.lockDeleteBucketPolicy.RUnlock()
+	return calls
+}
+
 // DeleteBucketTagging calls DeleteBucketTaggingFunc.
 func (mock *BackendMock) DeleteBucketTagging(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketTaggingFunc == nil {

s3api/controllers/base.go

@@ -80,7 +80,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectTaggingAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -139,7 +147,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			}
 		}
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.ListMultipartUploadPartsAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -169,7 +185,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionReadAcp,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectAclAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -191,7 +215,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectAttributesAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -218,7 +250,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.GetObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -341,7 +381,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketTaggingAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -378,7 +425,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketVersioningAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -407,7 +461,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("policy") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketPolicyAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -427,7 +488,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("versions") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.ListBucketVersionsAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -464,7 +532,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionReadAcp,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketAclAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -490,7 +565,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("uploads") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.ListBucketMultipartUploadsAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -525,7 +607,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.QueryInt("list-type") == 2 {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.ListBucketAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -560,7 +649,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.ListBucketAction,
+	})
 	if err != nil {
 		return SendXMLResponse(ctx, nil, err,
 			&MetaOpts{
@@ -640,7 +736,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			tags[tag.Key] = tag.Value
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -661,7 +764,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketVersioningAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -698,7 +808,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 
 	if ctx.Request().URI().QueryArgs().Has("policy") {
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketPolicyAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -708,6 +825,17 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 				})
 		}
 
+		err = auth.ValidatePolicyDocument(ctx.Body(), bucket, c.iam)
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "PutBucketPolicy",
+					BucketOwner: parsedAcl.Owner,
+				},
+			)
+		}
+
 		err = c.be.PutBucketPolicy(ctx.Context(), bucket, ctx.Body())
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -724,7 +852,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		var accessControlPolicy auth.AccessControlPolicy
 
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE_ACP", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWriteAcp,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketAclAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -844,9 +979,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	}
 
 	defACL := auth.ACL{
-		ACL:      "private",
-		Owner:    acct.Access,
-		Grantees: []auth.Grantee{},
+		Owner: acct.Access,
 	}
 
 	updAcl, err := auth.UpdateACL(&s3.PutBucketAclInput{
@@ -950,7 +1083,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			tags[tag.Key] = tag.Value
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutBucketTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -984,6 +1125,24 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				})
 		}
 
+		err := auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutObjectAction,
+		})
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "UploadPartCopy",
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
 		resp, err := c.be.UploadPartCopy(ctx.Context(), &s3.UploadPartCopyInput{
 			Bucket:              &bucket,
 			Key:                 &keyStart,
@@ -1013,7 +1172,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutObjectAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1152,7 +1319,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if copySource != "" {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutObjectAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -1225,7 +1400,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 
 	metadata := utils.GetUserMetaData(&ctx.Request().Header)
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        keyStart,
+		Action:        auth.PutObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1281,7 +1464,14 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1301,7 +1491,42 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	if ctx.Request().URI().QueryArgs().Has("policy") {
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.DeleteBucketPolicyAction,
+		})
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "DeleteBucketPolicy",
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = c.be.DeleteBucketPolicy(ctx.Context(), bucket)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				Action:      "DeleteBucketPolicy",
+				BucketOwner: parsedAcl.Owner,
+				Status:      http.StatusNoContent,
+			})
+	}
+
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.DeleteBucketAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1340,7 +1565,14 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.DeleteObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1380,12 +1612,20 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.DeleteObjectTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
 					Logger:      c.logger,
-					Action:      "RemoveObjectTagging",
+					Action:      "DeleteObjectTagging",
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
@@ -1405,7 +1645,15 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 		expectedBucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner")
 		requestPayer := ctx.Get("X-Amz-Request-Payer")
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.AbortMultipartUploadAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1432,7 +1680,15 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.DeleteObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1465,7 +1721,14 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.ListBucketAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1502,7 +1765,15 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.GetObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1608,7 +1879,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.RestoreObjectAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1646,7 +1925,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -1688,7 +1975,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.PutObjectAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -1727,7 +2022,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.PutObjectAction,
+	})
 	if err != nil {
 		return SendXMLResponse(ctx, nil, err,
 			&MetaOpts{

s3api/controllers/base_test.go

@@ -570,6 +570,19 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</VersioningConfiguration>
 	`
 
+	policyBody := `
+	{
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:GetObject",
+				"Resource": "arn:aws:s3:::my-bucket/*"
+			}
+		]
+	}
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -667,12 +680,21 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			statusCode: 200,
 		},
 		{
-			name: "Put-bucket-policy-success",
+			name: "Put-bucket-policy-invalid-body",
 			app:  app,
 			args: args{
 				req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", nil),
 			},
 			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", strings.NewReader(policyBody)),
+			},
+			wantErr:    false,
 			statusCode: 200,
 		},
 		{

s3api/utils/auth_test.go

@@ -84,7 +84,7 @@ func Test_Client_UserAgent(t *testing.T) {
 		}
 
 		req.Host = host
-		req.Header.Add("X-Amz-Content-Sha256", zeroLenSig)
+		req.Header.Set("X-Amz-Content-Sha256", zeroLenSig)
 
 		signer := v4.NewSigner()
 

s3api/utils/utils.go

@@ -20,11 +20,13 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/aws/smithy-go/encoding/httpbinding"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/s3err"
@@ -73,6 +75,13 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		}
 	})
 
+	// make sure all headers in the signed headers are present
+	for _, header := range signedHdrs {
+		if httpReq.Header.Get(header) == "" {
+			httpReq.Header.Set(header, "")
+		}
+	}
+
 	// Check if Content-Length in signed headers
 	// If content length is non 0, then the header will be included
 	if !includeHeader("Content-Length", signedHdrs) {
@@ -107,16 +116,18 @@ func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, cont
 	}
 
 	uri := string(ctx.Request().URI().Path())
+	uri = httpbinding.EscapePath(uri, false)
 	isFirst := true
 
 	ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
 		_, ok := signedQueryArgs[string(key)]
 		if !ok {
+			escapeValue := url.QueryEscape(string(value))
 			if isFirst {
-				uri += fmt.Sprintf("?%s=%s", key, value)
+				uri += fmt.Sprintf("?%s=%s", key, escapeValue)
 				isFirst = false
 			} else {
-				uri += fmt.Sprintf("&%s=%s", key, value)
+				uri += fmt.Sprintf("&%s=%s", key, escapeValue)
 			}
 		}
 	})

s3api/utils/utils_test.go

@@ -35,6 +35,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 		args    args
 		want    *http.Request
 		wantErr bool
+		hdrs    []string
 	}{
 		{
 			name: "Success-response",
@@ -43,6 +44,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request,
 			wantErr: false,
+			hdrs:    []string{},
 		},
 		{
 			name: "Success-response-With-Headers",
@@ -51,11 +53,12 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request2,
 			wantErr: false,
+			hdrs:    []string{"X-Amz-Mfa"},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
+			got, err := createHttpRequestFromCtx(tt.args.ctx, tt.hdrs, 0)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
 				return

tests/.env.default

@@ -1,6 +1,7 @@
 AWS_PROFILE=versity
 AWS_ENDPOINT_URL=https://127.0.0.1:7070
 VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
 BACKEND=posix
 LOCAL_FOLDER=/tmp/gw
 BUCKET_ONE_NAME=versity-gwtest-bucket-one
@@ -8,4 +9,6 @@ BUCKET_TWO_NAME=versity-gwtest-bucket-two
 #RECREATE_BUCKETS=true
 CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
-S3CMD_CONFIG=./tests/s3cfg.local.default
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets
+MC_ALIAS=versity

tests/.env.s3.default

@@ -0,0 +1,14 @@
+AWS_PROFILE=versity_s3
+AWS_ENDPOINT_URL=https://127.0.0.1:7070
+VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
+BACKEND=s3
+LOCAL_FOLDER=/tmp/gw
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+#RECREATE_BUCKETS=true
+CERT=$PWD/cert.pem
+KEY=$PWD/versitygw.pem
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets.s3
+MC_ALIAS=versity

tests/.env.static

@@ -1,8 +0,0 @@
-AWS_PROFILE=versity
-AWS_ENDPOINT_URL=http://127.0.0.1:7070
-VERSITY_EXE=./versitygw
-BACKEND=posix
-LOCAL_FOLDER=/tmp/gw
-BUCKET_ONE_NAME=versity-gwtest-bucket-one-static
-BUCKET_TWO_NAME=versity-gwtest-bucket-two-static
-RECREATE_BUCKETS=false

tests/integration/group-tests.go

@@ -43,6 +43,7 @@ func TestPresignedAuthentication(s *S3Conf) {
 	PresignedAuth_incorrect_secret_key(s)
 	PresignedAuth_PutObject_success(s)
 	PresignedAuth_Put_GetObject_with_data(s)
+	PresignedAuth_Put_GetObject_with_UTF8_chars(s)
 	PresignedAuth_UploadPart(s)
 }
 
@@ -241,6 +242,41 @@ func TestGetBucketAcl(s *S3Conf) {
 	GetBucketAcl_success(s)
 }
 
+func TestPutBucketPolicy(s *S3Conf) {
+	PutBucketPolicy_non_existing_bucket(s)
+	PutBucketPolicy_invalid_effect(s)
+	PutBucketPolicy_empty_actions_string(s)
+	PutBucketPolicy_empty_actions_array(s)
+	PutBucketPolicy_invalid_action(s)
+	PutBucketPolicy_unsupported_action(s)
+	PutBucketPolicy_incorrect_action_wildcard_usage(s)
+	PutBucketPolicy_empty_principals_string(s)
+	PutBucketPolicy_empty_principals_array(s)
+	PutBucketPolicy_principals_incorrect_wildcard_usage(s)
+	PutBucketPolicy_non_existing_principals(s)
+	PutBucketPolicy_empty_resources_string(s)
+	PutBucketPolicy_empty_resources_array(s)
+	PutBucketPolicy_invalid_resource_prefix(s)
+	PutBucketPolicy_invalid_resource_with_starting_slash(s)
+	PutBucketPolicy_duplicate_resource(s)
+	PutBucketPolicy_incorrect_bucket_name(s)
+	PutBucketPolicy_object_action_on_bucket_resource(s)
+	PutBucketPolicy_bucket_action_on_object_resource(s)
+	PutBucketPolicy_success(s)
+}
+
+func TestGetBucketPolicy(s *S3Conf) {
+	GetBucketPolicy_non_existing_bucket(s)
+	GetBucketPolicy_default_empty_policy(s)
+	GetBucketPolicy_success(s)
+}
+
+func TestDeleteBucketPolicy(s *S3Conf) {
+	DeleteBucketPolicy_non_existing_bucket(s)
+	DeleteBucketPolicy_remove_before_setting(s)
+	DeleteBucketPolicy_success(s)
+}
+
 func TestFullFlow(s *S3Conf) {
 	TestAuthentication(s)
 	TestPresignedAuthentication(s)
@@ -270,6 +306,9 @@ func TestFullFlow(s *S3Conf) {
 	TestCompleteMultipartUpload(s)
 	TestPutBucketAcl(s)
 	TestGetBucketAcl(s)
+	TestPutBucketPolicy(s)
+	TestGetBucketPolicy(s)
+	TestDeleteBucketPolicy(s)
 }
 
 func TestPosix(s *S3Conf) {
@@ -329,6 +368,7 @@ func GetIntTests() IntTests {
 		"PresignedAuth_incorrect_secret_key":                    PresignedAuth_incorrect_secret_key,
 		"PresignedAuth_PutObject_success":                       PresignedAuth_PutObject_success,
 		"PresignedAuth_Put_GetObject_with_data":                 PresignedAuth_Put_GetObject_with_data,
+		"PresignedAuth_Put_GetObject_with_UTF8_chars":           PresignedAuth_Put_GetObject_with_UTF8_chars,
 		"PresignedAuth_UploadPart":                              PresignedAuth_UploadPart,
 		"CreateBucket_invalid_bucket_name":                      CreateBucket_invalid_bucket_name,
 		"CreateBucket_existing_bucket":                          CreateBucket_existing_bucket,
@@ -443,6 +483,32 @@ func GetIntTests() IntTests {
 		"GetBucketAcl_non_existing_bucket":                      GetBucketAcl_non_existing_bucket,
 		"GetBucketAcl_access_denied":                            GetBucketAcl_access_denied,
 		"GetBucketAcl_success":                                  GetBucketAcl_success,
+		"PutBucketPolicy_non_existing_bucket":                   PutBucketPolicy_non_existing_bucket,
+		"PutBucketPolicy_invalid_effect":                        PutBucketPolicy_invalid_effect,
+		"PutBucketPolicy_empty_actions_string":                  PutBucketPolicy_empty_actions_string,
+		"PutBucketPolicy_empty_actions_array":                   PutBucketPolicy_empty_actions_array,
+		"PutBucketPolicy_invalid_action":                        PutBucketPolicy_invalid_action,
+		"PutBucketPolicy_unsupported_action":                    PutBucketPolicy_unsupported_action,
+		"PutBucketPolicy_incorrect_action_wildcard_usage":       PutBucketPolicy_incorrect_action_wildcard_usage,
+		"PutBucketPolicy_empty_principals_string":               PutBucketPolicy_empty_principals_string,
+		"PutBucketPolicy_empty_principals_array":                PutBucketPolicy_empty_principals_array,
+		"PutBucketPolicy_principals_incorrect_wildcard_usage":   PutBucketPolicy_principals_incorrect_wildcard_usage,
+		"PutBucketPolicy_non_existing_principals":               PutBucketPolicy_non_existing_principals,
+		"PutBucketPolicy_empty_resources_string":                PutBucketPolicy_empty_resources_string,
+		"PutBucketPolicy_empty_resources_array":                 PutBucketPolicy_empty_resources_array,
+		"PutBucketPolicy_invalid_resource_prefix":               PutBucketPolicy_invalid_resource_prefix,
+		"PutBucketPolicy_invalid_resource_with_starting_slash":  PutBucketPolicy_invalid_resource_with_starting_slash,
+		"PutBucketPolicy_duplicate_resource":                    PutBucketPolicy_duplicate_resource,
+		"PutBucketPolicy_incorrect_bucket_name":                 PutBucketPolicy_incorrect_bucket_name,
+		"PutBucketPolicy_object_action_on_bucket_resource":      PutBucketPolicy_object_action_on_bucket_resource,
+		"PutBucketPolicy_bucket_action_on_object_resource":      PutBucketPolicy_bucket_action_on_object_resource,
+		"PutBucketPolicy_success":                               PutBucketPolicy_success,
+		"GetBucketPolicy_non_existing_bucket":                   GetBucketPolicy_non_existing_bucket,
+		"GetBucketPolicy_default_empty_policy":                  GetBucketPolicy_default_empty_policy,
+		"GetBucketPolicy_success":                               GetBucketPolicy_success,
+		"DeleteBucketPolicy_non_existing_bucket":                DeleteBucketPolicy_non_existing_bucket,
+		"DeleteBucketPolicy_remove_before_setting":              DeleteBucketPolicy_remove_before_setting,
+		"DeleteBucketPolicy_success":                            DeleteBucketPolicy_success,
 		"PutObject_overwrite_dir_obj":                           PutObject_overwrite_dir_obj,
 		"PutObject_overwrite_file_obj":                          PutObject_overwrite_file_obj,
 		"PutObject_dir_obj_with_data":                           PutObject_dir_obj_with_data,

tests/integration/tests.go

@@ -1448,8 +1448,6 @@ func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
 			return fmt.Errorf("read get object response body %w", err)
 		}
 
-		fmt.Println(resp.Request.Method, resp.ContentLength, string(respBody))
-
 		if string(respBody) != data {
 			return fmt.Errorf("expected get object response body to be %v, instead got %s", data, respBody)
 		}
@@ -1463,6 +1461,72 @@ func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
 	})
 }
 
+func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error {
+	testName := "PresignedAuth_Put_GetObject_with_data"
+	return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
+		bucket, obj := getBucketName(), "my-$%^&*;"
+		err := setup(s, bucket)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		httpClient := http.Client{
+			Timeout: shortTimeout,
+		}
+
+		req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
+		if err != nil {
+			return err
+		}
+
+		req.Header = v4req.SignedHeader
+
+		resp, err := httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected my-obj to be successfully uploaded and get %v response status, instead got %v", http.StatusOK, resp.StatusCode)
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		v4GetReq, err := client.PresignGetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &obj})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		req, err = http.NewRequest(v4GetReq.Method, v4GetReq.URL, nil)
+		if err != nil {
+			return err
+		}
+
+		resp, err = httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected get object response status to be %v, instead got %v", http.StatusOK, resp.StatusCode)
+		}
+
+		err = teardown(s, bucket)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func PresignedAuth_UploadPart(s *S3Conf) error {
 	testName := "PresignedAuth_UploadPart"
 	return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
@@ -3201,28 +3265,35 @@ func CopyObject_not_owned_source_bucket(s *S3Conf) error {
 		}
 
 		usr := user{
-			access: "admin1",
-			secret: "admin1secret",
-			role:   "admin",
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
 		}
 
 		cfg := *s
 		cfg.awsID = usr.access
 		cfg.awsSecret = usr.secret
 
+		userS3Client := s3.NewFromConfig(cfg.Config())
+
 		err = createUsers(s, []user{usr})
 		if err != nil {
 			return err
 		}
 
 		dstBucket := getBucketName()
-		err = setup(&cfg, dstBucket)
+		err = setup(s, dstBucket)
+		if err != nil {
+			return err
+		}
+
+		err = changeBucketsOwner(s, []string{bucket}, usr.access)
 		if err != nil {
 			return err
 		}
 
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-		_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
+		_, err = userS3Client.CopyObject(ctx, &s3.CopyObjectInput{
 			Bucket:     &dstBucket,
 			Key:        getPtr("obj-1"),
 			CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
@@ -5177,6 +5248,555 @@ func GetBucketAcl_success(s *S3Conf) error {
 	})
 }
 
+func PutBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "PutBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_effect(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_effect"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("invalid_effect", `"*"`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid effect: invalid_effect")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_actions_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_actions_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `""`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_actions_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_actions_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `[]`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_action(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"ListObjects"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid action: ListObjects")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_unsupported_action(s *S3Conf) error {
+	testName := "PutBucketPolicy_unsupported_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:PutLifecycleConfiguration"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported action: s3:PutLifecycleConfiguration")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_incorrect_action_wildcard_usage(s *S3Conf) error {
+	testName := "PutBucketPolicy_incorrect_action_wildcard_usage"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:hello*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid wildcard usage: s3:hello prefix is not in the supported actions list")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_principals_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_principals_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `""`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_principals_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_principals_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `[]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_principals_incorrect_wildcard_usage(s *S3Conf) error {
+	testName := "PutBucketPolicy_principals_incorrect_wildcard_usage"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*", "grt1"]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals should either contain * or user access keys")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_non_existing_principals(s *S3Conf) error {
+	testName := "PutBucketPolicy_non_existing_principals"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		apiErr1 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"}))
+		apiErr2 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_2", "a_rarely_existing_user_account_1"}))
+
+		err1 := checkApiErr(err, apiErr1)
+		err2 := checkApiErr(err, apiErr2)
+
+		if err1 != nil && err2 != nil {
+			return err1
+		}
+
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_resources_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_resources_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, `""`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_resources_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_resources_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, `[]`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_resource_prefix(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_resource_prefix"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:iam:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_resource_with_starting_slash(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_resource_with_starting_slash"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::/%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_duplicate_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_duplicate_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, fmt.Sprintf("[%v, %v]", resource, resource))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("duplicate resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_incorrect_bucket_name(s *S3Conf) error {
+	testName := "PutBucketPolicy_incorrect_bucket_name"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::prefix-%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("incorrect bucket name in prefix-%v", bucket))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_object_action_on_bucket_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_object_action_on_bucket_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:PutObjectTagging"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported object action 's3:PutObjectTagging' on the specified resources")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_bucket_action_on_object_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_object_action_on_bucket_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:DeleteBucket"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported bucket action 's3:DeleteBucket' on the specified resources")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_success(s *S3Conf) error {
+	testName := "PutBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		err := createUsers(s, []user{
+			{"grt1", "grt1secret", "user"},
+			{"grt2", "grt2secret", "user"},
+		})
+		if err != nil {
+			return err
+		}
+
+		bucketResource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		objectResource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+
+		for _, doc := range []string{
+			genPolicyDoc("Allow", `["grt1", "grt2"]`, `["s3:DeleteBucket", "s3:GetBucketAcl"]`, bucketResource),
+			genPolicyDoc("Deny", `"*"`, `"s3:DeleteBucket"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)),
+			genPolicyDoc("Allow", `"grt1"`, `["s3:PutBucketVersioning", "s3:ListMultipartUploadParts", "s3:ListBucket"]`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+			genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+			genPolicyDoc("Allow", `"*"`, `"s3:Get*"`, objectResource),
+			genPolicyDoc("Deny", `"*"`, `"s3:Create*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+		} {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+				Bucket: &bucket,
+				Policy: &doc,
+			})
+			cancel()
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func GetBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "GetBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func GetBucketPolicy_default_empty_policy(s *S3Conf) error {
+	testName := "GetBucketPolicy_default_empty_policy"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy != nil {
+			return fmt.Errorf("expected policy to be nil, instead got %s", *out.Policy)
+		}
+
+		return nil
+	})
+}
+
+func GetBucketPolicy_success(s *S3Conf) error {
+	testName := "GetBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `["s3:DeleteBucket", "s3:GetBucketTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy == nil {
+			return fmt.Errorf("expected non nil policy result")
+		}
+
+		if *out.Policy != doc {
+			return fmt.Errorf("expected the bucket policy to be %v, instead got %v", doc, *out.Policy)
+		}
+
+		return nil
+	})
+}
+
+func DeleteBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func DeleteBucketPolicy_remove_before_setting(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_remove_before_setting"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+
+		return err
+	})
+}
+
+func DeleteBucketPolicy_success(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `["s3:DeleteBucket", "s3:GetBucketTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy != nil {
+			return fmt.Errorf("expected policy to be nil, instead got %s", *out.Policy)
+		}
+
+		return err
+	})
+}
+
 // IAM related tests
 // multi-user iam tests
 func IAM_user_access_denied(s *S3Conf) error {

tests/integration/utils.go

@@ -219,12 +219,17 @@ func checkApiErr(err error, apiErr s3err.APIError) error {
 	}
 	var ae smithy.APIError
 	if errors.As(err, &ae) {
-		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
-			return nil
+		if ae.ErrorCode() != apiErr.Code {
+			return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, ae.ErrorCode())
 		}
 
-		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+		if ae.ErrorMessage() != apiErr.Description {
+			return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, ae.ErrorMessage())
+		}
+
+		return nil
 	}
+
 	return fmt.Errorf("expected aws api error, instead got: %w", err)
 }
 
@@ -603,3 +608,28 @@ func changeAuthCred(uri, newVal string, index int) (string, error) {
 
 	return urlParsed.String(), nil
 }
+
+func genPolicyDoc(effect, principal, action, resource string) string {
+	jsonTemplate := `
+	{
+		"Statement": [
+			{
+				"Effect":  "%s",
+				"Principal": %s,
+				"Action":  %s,
+				"Resource":  %s
+			}
+		]
+	}
+	`
+
+	return fmt.Sprintf(jsonTemplate, effect, principal, action, resource)
+}
+
+func getMalformedPolicyError(msg string) s3err.APIError {
+	return s3err.APIError{
+		Code:           "MalformedPolicy",
+		Description:    msg,
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}

tests/s3cfg.local.default

@@ -3,6 +3,7 @@ host_base = 127.0.0.1:7070
 host_bucket = 127.0.0.1:7070
 bucket_location = us-east-1
 use_https = True
+signurl_use_https = True
 
 # Enable S3 v4 signature APIs
 signature_v2 = False

tests/setup.sh

@@ -17,22 +17,25 @@ setup() {
     return 1
   fi
 
-  S3CMD_OPTS=()
-  S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
-  S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
-  S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
+  if [[ $RUN_S3CMD == true ]]; then
+    S3CMD_OPTS=()
+    S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
+    S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
+    S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
+    export S3CMD_CONFIG S3CMD_OPTS
+  fi
 
-  check_add_mc_alias || check_result=$?
-  if [[ $check_result -ne 0 ]]; then
-    echo "mc alias check/add failed"
-    return 1
+  if [[ $RUN_MC == true ]]; then
+    check_add_mc_alias || check_result=$?
+    if [[ $check_result -ne 0 ]]; then
+      echo "mc alias check/add failed"
+      return 1
+    fi
   fi
 
   export AWS_PROFILE \
     BUCKET_ONE_NAME \
-    BUCKET_TWO_NAME \
-    S3CMD_CONFIG \
-    S3CMD_OPTS
+    BUCKET_TWO_NAME
 }
 
 # make sure required environment variables for tests are defined properly

tests/setup_mc.sh

@@ -8,7 +8,8 @@ check_for_alias() {
     return 2
   fi
   while IFS= read -r line; do
-    if [[ $line =~ ^versity$ ]]; then
+    error=$(echo "$line" | grep -w "$MC_ALIAS ")
+    if [[ $? -eq 0 ]]; then
       return 0
     fi
   done <<< "$aliases"
@@ -25,7 +26,7 @@ check_add_mc_alias() {
     return 0
   fi
   local set_result
-  error=$(mc alias set --insecure versity "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$?
+  error=$(mc alias set --insecure "$MC_ALIAS" "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$?
   if [[ $set_result -ne 0 ]]; then
     echo "error setting alias: $error"
     return 1

tests/test_aws.sh

@@ -11,8 +11,12 @@ source ./tests/test_common.sh
 }
 
 # test adding and removing an object on versitygw
-@test "test_put_object" {
-  test_common_put_object "aws"
+@test "test_put_object-with-data" {
+  test_common_put_object_with_data "aws"
+}
+
+@test "test_put_object-no-data" {
+  test_common_put_object_no_data "aws"
 }
 
 # test listing buckets on versitygw
@@ -97,27 +101,7 @@ source ./tests/test_common.sh
 
 # test abilty to set and retrieve bucket tags
 @test "test-set-get-bucket-tags" {
-
-  local key="test_key"
-  local value="test_value"
-
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-
-  get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
-  tag_set=$(echo "$tags" | jq '.TagSet')
-  [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
-
-  put_bucket_tag "$BUCKET_ONE_NAME" $key $value
-  get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$?
-  [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
-  tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
-  tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
-  [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
-  [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  test_common_set_get_bucket_tags "aws"
 }
 
 # test v1 s3api list objects command
@@ -184,34 +168,7 @@ source ./tests/test_common.sh
 
 # test abilty to set and retrieve object tags
 @test "test-set-get-object-tags" {
-
-  local bucket_file="bucket-file"
-  local key="test_key"
-  local value="test_value"
-
-  create_test_files "$bucket_file" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating test files"
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-  local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
-  put_object "aws" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
-  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
-
-  get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting object tags"
-  tag_set=$(echo "$tags" | jq '.TagSet')
-  [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
-
-  put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value
-  get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
-  [[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
-  tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
-  tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
-  [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
-  [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
-  delete_test_files $bucket_file
+  test_common_set_get_object_tags "aws"
 }
 
 # test multi-part upload
@@ -358,3 +315,7 @@ source ./tests/test_common.sh
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
   delete_test_files $bucket_file
 }
+
+@test "test-presigned-url-utf8-chars" {
+  test_common_presigned_url_utf8_chars "aws"
+}

tests/test_common.sh

@@ -25,24 +25,38 @@ test_common_create_delete_bucket() {
   [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
 }
 
-test_common_put_object() {
-
+test_common_put_object_with_data() {
   if [[ $# -ne 1 ]]; then
     fail "put object test requires command type"
   fi
 
   local object_name="test-object"
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+  echo "test data" > "$test_file_folder"/"$object_name"
+}
+
+test_common_put_object_no_data() {
+  if [[ $# -ne 1 ]]; then
+    fail "put object test requires command type"
+  fi
+
+  local object_name="test-object"
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+  test_common_put_object "$1" "$object_name"
+}
+
+test_common_put_object() {
+  if [[ $# -ne 2 ]]; then
+    fail "put object test requires command type, file"
+  fi
 
   setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
   [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
 
-  create_test_files "$object_name" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Error creating test file"
-
-  echo "test data" > "$test_file_folder"/"$object_name"
-
-  object="$BUCKET_ONE_NAME"/$object_name
-  put_object "$1" "$test_file_folder"/"$object_name" "$object" || local put_object=$?
+  object="$BUCKET_ONE_NAME"/"$2"
+  put_object "$1" "$test_file_folder"/"$2" "$object" || local put_object=$?
   [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
   object_exists "$1" "$object" || local exists_result_one=$?
   [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
@@ -53,7 +67,7 @@ test_common_put_object() {
   [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
-  delete_test_files "$object_name"
+  delete_test_files "$2"
 }
 
 # common test for listing buckets
@@ -95,7 +109,6 @@ test_common_list_buckets() {
 }
 
 test_common_list_objects() {
-
   if [[ $# -ne 1 ]]; then
     echo "common test function for listing objects requires command type"
     return 1
@@ -133,3 +146,150 @@ test_common_list_objects() {
     fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
   fi
 }
+
+test_common_set_get_bucket_tags() {
+
+  if [[ $# -ne 1 ]]; then
+    fail "set/get bucket tags test requires command type"
+  fi
+
+  local key="test_key"
+  local value="test_value"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  get_bucket_tags "$1" "$BUCKET_ONE_NAME" || local get_result=$?
+  [[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
+
+  if [[ $1 == 'aws' ]]; then
+    if [[ $tags != "" ]]; then
+      tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet')
+      [[ $tag_set == "[]" ]] || fail "Error:  tags not empty: $tags"
+    fi
+  else
+    [[ $tags == "" ]] || [[ $tags =~ "No tags found" ]] || fail "Error:  tags not empty: $tags"
+  fi
+
+  put_bucket_tag "$1" "$BUCKET_ONE_NAME" $key $value
+  get_bucket_tags "$1" "$BUCKET_ONE_NAME" || local get_result_two=$?
+  [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
+
+  local tag_set_key
+  local tag_set_value
+  if [[ $1 == 'aws' ]]; then
+    tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key')
+    tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value')
+    [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
+    [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
+  else
+    read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')"
+    [[ $tag_set_key == "$key" ]] || fail "Key mismatch"
+    [[ $tag_set_value == "$value" ]] || fail "Value mismatch"
+  fi
+  delete_bucket_tags "$1" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+}
+
+test_common_set_get_object_tags() {
+
+  if [[ $# -ne 1 ]]; then
+    echo "get/set object tags missing command type"
+    return 1
+  fi
+
+  local bucket_file="bucket-file"
+  local key="test_key"
+  local value="test_value"
+
+  create_test_files "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
+  put_object "$1" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
+
+  get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
+  [[ $get_result -eq 0 ]] || fail "Error getting object tags"
+  if [[ $1 == 'aws' ]]; then
+    tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet')
+    [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
+  elif [[ ! $tags == *"No tags found"* ]]; then
+    fail "no tags found (tags: $tags)"
+  fi
+
+  put_object_tag "$1" "$BUCKET_ONE_NAME" $bucket_file $key $value
+  get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
+  [[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
+  if [[ $1 == 'aws' ]]; then
+    tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key')
+    tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value')
+    [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
+    [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
+  else
+    read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')"
+    [[ $tag_set_key == "$key" ]] || fail "Key mismatch"
+    [[ $tag_set_value == "$value" ]] || fail "Value mismatch"
+  fi
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}
+
+test_common_multipart_upload() {
+  if [[ $# -ne 1 ]]; then
+    echo "multipart upload command missing command type"
+    return 1
+  fi
+  bucket_file="largefile"
+
+  create_large_file "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  put_object "$1" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "failed to copy file"
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}
+
+test_common_presigned_url_utf8_chars() {
+
+  if [[ $# -ne 1 ]]; then
+    echo "presigned url command missing command type"
+    return 1
+  fi
+
+  local bucket_file="my-$%^&*;"
+  local bucket_file_copy="bucket-file-copy"
+  bucket_file_data="test file\n"
+
+  create_test_files "$bucket_file" || local created=$?
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/"$bucket_file"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || put_result=$?
+  [[ $put_result -eq 0 ]] || fail "Failed to add object $bucket_file"
+
+  create_presigned_url "$1" "$BUCKET_ONE_NAME" "$bucket_file" || presigned_result=$?
+  [[ $presigned_result -eq 0 ]] || fail "presigned url creation failure"
+
+  error=$(curl -k -v "$presigned_url" -o "$test_file_folder"/"$bucket_file_copy") || curl_result=$?
+  if [[ $curl_result -ne 0 ]]; then
+    fail "error downloading file with curl: $error"
+  fi
+  compare_files "$test_file_folder"/"$bucket_file" "$test_file_folder"/"$bucket_file_copy" || compare_result=$?
+  if [[ $compare_result -ne 0 ]]; then
+    echo "file one: $(cat "$test_file_folder"/"$bucket_file")"
+    echo "file two: $(cat "$test_file_folder"/"$bucket_file_copy")"
+    fail "files don't match"
+  fi
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files "$bucket_file" "$bucket_file_copy"
+}

tests/test_mc.sh

@@ -3,7 +3,41 @@
 source ./tests/test_common.sh
 source ./tests/setup.sh
 
+export RUN_MC=true
+
 # test mc bucket creation/deletion
 @test "test_create_delete_bucket_mc" {
   test_common_create_delete_bucket "mc"
-}
+}
+
+@test "test_put_object-with-data-mc" {
+  test_common_put_object_with_data "mc"
+}
+
+@test "test_put_object-no-data-mc" {
+  test_common_put_object_no_data "mc"
+}
+
+@test "test_list_buckets_mc" {
+  test_common_list_buckets "mc"
+}
+
+@test "test_list_objects_mc" {
+  test_common_list_objects "mc"
+}
+
+@test "test_set_get_bucket_tags_mc" {
+  test_common_set_get_bucket_tags "mc"
+}
+
+@test "test_set_get_object_tags_mc" {
+  test_common_set_get_object_tags "mc"
+}
+
+@test "test_multipart_upload_mc" {
+  test_common_multipart_upload "mc"
+}
+
+@test "test_presigned_url_utf8_chars_mc" {
+  test_common_presigned_url_utf8_chars "mc"
+}

tests/test_s3cmd.sh

@@ -4,14 +4,20 @@ source ./tests/setup.sh
 source ./tests/test_common.sh
 source ./tests/util.sh
 
+export RUN_S3CMD=true
+
 # test s3cmd bucket creation/deletion
 @test "test_create_delete_bucket_s3cmd" {
   test_common_create_delete_bucket "s3cmd"
 }
 
 # test s3cmd put object
-@test "test_put_object_s3cmd" {
-  test_common_put_object "s3cmd"
+@test "test_put_object_with_data_s3cmd" {
+  test_common_put_object_with_data "s3cmd"
+}
+
+@test "test_put_object_no_data_s3cmd" {
+  test_common_put_object_no_data "s3cmd"
 }
 
 # test listing buckets on versitygw
@@ -24,18 +30,9 @@ source ./tests/util.sh
 }
 
 @test "test_multipart_upload_s3cmd" {
+  test_common_multipart_upload "s3cmd"
+}
 
-  bucket_file="largefile"
-
-  create_large_file "$bucket_file" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload"
-
-  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-
-  put_object "s3cmd" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$?
-  [[ $put_result -eq 0 ]] || fail "failed to copy file"
-
-  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
-  delete_test_files $bucket_file
-}
+#@test "test_presigned_url_utf8_chars_s3cmd" {
+#  test_common_presigned_url_utf8_chars "s3cmd"
+#}

tests/util.sh

@@ -18,7 +18,7 @@ create_bucket() {
   elif [[ $1 == "s3cmd" ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure mb versity/"$2" 2>&1) || exit_code=$?
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -34,14 +34,21 @@ create_bucket() {
 # param:  bucket name
 # return 0 for success, 1 for failure
 delete_bucket() {
-  if [ $# -ne 1 ]; then
-    echo "delete bucket missing bucket name"
+  if [ $# -ne 2 ]; then
+    echo "delete bucket missing command type, bucket name"
     return 1
   fi
 
   local exit_code=0
   local error
-  error=$(aws --no-verify-ssl s3 rb s3://"$1" 2>&1) || exit_code="$?"
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3 rb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
+  else
+    echo "Invalid command type $1"
+    return 1
+  fi
   if [ $exit_code -ne 0 ]; then
     if [[ "$error" == *"The specified bucket does not exist"* ]]; then
       return 0
@@ -79,7 +86,7 @@ delete_bucket_recursive() {
     if [[ "$error" == *"The specified bucket does not exist"* ]]; then
       return 0
     else
-      echo "error deleting bucket: $error"
+      echo "error deleting bucket recursively: $error"
       return 1
     fi
   fi
@@ -91,7 +98,7 @@ delete_bucket_recursive() {
 # return 0 for success, 1 for failure
 delete_bucket_contents() {
   if [ $# -ne 2 ]; then
-    echo "delete bucket missing bucket name"
+    echo "delete bucket missing command id, bucket name"
     return 1
   fi
 
@@ -102,7 +109,7 @@ delete_bucket_contents() {
   elif [[ $1 == "s3cmd" ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?"
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure rm --force --recursive versity/"$2" 2>&1) || exit_code="$?"
+    error=$(mc --insecure rm --force --recursive "$MC_ALIAS"/"$2" 2>&1) || exit_code="$?"
   else
     echo "invalid command type $1"
     return 1
@@ -126,18 +133,20 @@ bucket_exists() {
   local exit_code=0
   local error
   if [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?"
+    error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?"
+    # NOTE:  s3cmd sometimes takes longer with direct connection
+    sleep 1
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure ls versity)
+    error=$(mc --insecure ls "$MC_ALIAS/$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type: $1"
     return 2
   fi
 
   if [ $exit_code -ne 0 ]; then
-    if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
+    if [[ "$error" == *"does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
       return 1
     else
       echo "error checking if bucket exists: $error"
@@ -193,6 +202,7 @@ setup_bucket() {
       return 1
     fi
     if [[ $RECREATE_BUCKETS == "false" ]]; then
+      echo "bucket data deletion success"
       return 0
     fi
   fi
@@ -219,11 +229,13 @@ object_exists() {
     return 2
   fi
   local exit_code=0
-  local error
+  local error=""
   if [[ $1 == 'aws' ]]; then
     error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?"
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 2
@@ -235,8 +247,8 @@ object_exists() {
       echo "error checking if object exists: $error"
       return 2
     fi
-  # s3cmd returns empty when object doesn't exist, rather than error
-  elif [[ $1 == 's3cmd' ]] && [[ $error == "" ]]; then
+  # s3cmd, mc return empty when object doesn't exist, rather than error
+  elif [[ ( $1 == 's3cmd' ) || ( $1 == 'mc' ) ]] && [[ $error == "" ]]; then
     return 1
   fi
   return 0
@@ -255,8 +267,9 @@ put_object() {
   if [[ $1 == 'aws' ]]; then
     error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    echo "2: $2 3: $(dirname $3)"
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$(dirname "$3")" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure cp "$2" "$MC_ALIAS"/"$(dirname "$3")" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -305,7 +318,8 @@ delete_object() {
     error=$(aws --no-verify-ssl s3 rm s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm s3://"$2" 2>&1) || exit_code=$?
-    echo "$error"
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure rm "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -322,7 +336,7 @@ delete_object() {
 # export bucket_array (bucket names) on success, return 1 for failure
 list_buckets() {
   if [[ $# -ne 1 ]]; then
-    echo "List buckets command mssing format"
+    echo "List buckets command missing format"
     return 1
   fi
 
@@ -331,7 +345,9 @@ list_buckets() {
   if [[ $1 == "aws" ]]; then
     output=$(aws --no-verify-ssl s3 ls s3:// 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://) || exit_code=$?
+    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    output=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
   else
     echo "invalid format:  $1"
     return 1
@@ -345,7 +361,7 @@ list_buckets() {
   bucket_array=()
   while IFS= read -r line; do
     bucket_name=$(echo "$line" | awk '{print $NF}')
-    bucket_array+=("$bucket_name")
+    bucket_array+=("${bucket_name%/}")
   done <<< "$output"
 
   export bucket_array
@@ -365,6 +381,8 @@ list_objects() {
     output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -463,13 +481,20 @@ get_object_acl() {
 # params:  bucket, key, value
 # return:  0 for success, 1 for error
 put_bucket_tag() {
-  if [ $# -ne 3 ]; then
-    echo "bucket tag command missing bucket name, key, value"
+  if [ $# -ne 4 ]; then
+    echo "bucket tag command missing command type, bucket name, key, value"
     return 1
   fi
   local error
   local result
-  error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
     echo "Error adding bucket tag: $error"
     return 1
@@ -481,30 +506,65 @@ put_bucket_tag() {
 # params:  bucket
 # export 'tags' on success, return 1 for error
 get_bucket_tags() {
-  if [ $# -ne 1 ]; then
-    echo "get bucket tag command missing bucket name"
+  if [ $# -ne 2 ]; then
+    echo "get bucket tag command missing command type, bucket name"
     return 1
   fi
   local result
-  tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$1") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
+    if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
+      export tags=
+      return 0
+    fi
     echo "error getting bucket tags: $tags"
     return 1
   fi
   export tags
 }
 
+delete_bucket_tags() {
+  if [ $# -ne 2 ]; then
+    echo "delete bucket tag command missing command type, bucket name"
+    return 1
+  fi
+  local result
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  return 0
+}
+
 # add tags to object
 # params:  object, key, value
 # return:  0 for success, 1 for error
 put_object_tag() {
-  if [ $# -ne 4 ]; then
-    echo "object tag command missing object name, file, key, and/or value"
+  if [ $# -ne 5 ]; then
+    echo "object tag command missing command type, object name, file, key, and/or value"
     return 1
   fi
   local error
   local result
-  error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$1" --key "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
     echo "Error adding object tag: $error"
     return 1
@@ -516,12 +576,19 @@ put_object_tag() {
 # params:  bucket
 # export 'tags' on success, return 1 for error
 get_object_tags() {
-  if [ $# -ne 2 ]; then
-    echo "get object tag command missing bucket and/or key"
+  if [ $# -ne 3 ]; then
+    echo "get object tag command missing command type, bucket, and/or key"
     return 1
   fi
   local result
-  tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$1" --key "$2") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
     echo "error getting object tags: $tags"
     return 1
@@ -834,3 +901,28 @@ upload_part_copy() {
   etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag')
   export etag
 }
+
+create_presigned_url() {
+  if [[ $# -ne 3 ]]; then
+    echo "create presigned url function requires command type, bucket, and filename"
+    return 1
+  fi
+
+  local presign_result=0
+  if [[ $1 == 'aws' ]]; then
+    presigned_url=$(aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    presigned_url=$(s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    presigned_url_data=$(mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
+    presigned_url="${presigned_url_data#*Share: }"
+  else
+    echo "unrecognized command type $1"
+    return 1
+  fi
+  if [[ $presign_result -ne 0 ]]; then
+    echo "error generating presigned url: $presigned_url"
+    return 1
+  fi
+  export presigned_url
+}

tests/util_mc.sh

@@ -10,12 +10,12 @@ delete_bucket_recursive_mc() {
   fi
   local exit_code=0
   local error
-  error=$(mc --insecure rm --recursive --force versity/"$1" 2>&1) || exit_code="$?"
+  error=$(mc --insecure rm --recursive --force "$MC_ALIAS"/"$1" 2>&1) || exit_code="$?"
   if [[ $exit_code -ne 0 ]]; then
     echo "error deleting bucket contents: $error"
     return 1
   fi
-  error=$(mc --insecure rb versity/"$1" 2>&1) || exit_code="$?"
+  error=$(mc --insecure rb "$MC_ALIAS"/"$1" 2>&1) || exit_code="$?"
   if [[ $exit_code -ne 0 ]]; then
     echo "error deleting bucket: $error"
     return 1

tests/versity.sh

@@ -1,5 +1,30 @@
 #!/bin/bash
 
+source ./tests/util_file.sh
+
+check_exe_params_versity() {
+  if [ -z "$LOCAL_FOLDER" ]; then
+    echo "No local storage folder set"
+    return 1
+  elif [ -z "$VERSITY_EXE" ]; then
+    echo "No versity executable location set"
+    return 1
+  elif [ -z "$BACKEND" ]; then
+    echo "No backend parameter set (options: 'posix')"
+    return 1
+  fi
+  if [ "$BACKEND" == 's3' ]; then
+    if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then
+      echo "missing second AWS access key ID for s3 backend"
+      return 1
+    fi
+    if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then
+      echo "missing second AWS secret access key for s3 backend"
+      return 1
+    fi
+  fi
+}
+
 check_exe_params() {
   if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     echo "No AWS access key set"
@@ -7,30 +32,29 @@ check_exe_params() {
   elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
     echo "No AWS secret access key set"
     return 1
-  elif [ -z "$VERSITY_EXE" ]; then
-    echo "No versity executable location set"
-    return 1
-  elif [ -z "$BACKEND" ]; then
-    echo "No backend parameter set (options: 'posix')"
-    return 1
   elif [ -z "$AWS_PROFILE" ]; then
     echo "No AWS profile set"
     return 1
-  elif [ -z "$LOCAL_FOLDER" ]; then
-    echo "No local storage folder set"
-    return 1
   elif [ -z "$AWS_ENDPOINT_URL" ]; then
     echo "No AWS endpoint URL set"
     return 1
+  elif [ -z "$MC_ALIAS" ]; then
+    echo "No mc alias set"
+    return 1
+  elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then
+    echo "RUN_VERSITYGW must be 'true' or 'false'"
+    return 1
+  fi
+  if [[ $RUN_VERSITYGW == "true" ]]; then
+    local check_result
+    check_exe_params_versity || check_result=$?
+    if [[ $check_result -ne 0 ]]; then
+      return 1
+    fi
   fi
 }
 
 start_versity() {
-  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then
-    source tests/.secrets
-  else
-    echo "Warning: no secrets file found"
-  fi
   if [ -z "$VERSITYGW_TEST_ENV" ]; then
     if [ -r tests/.env ]; then
       source tests/.env
@@ -41,6 +65,12 @@ start_versity() {
     # shellcheck source=./.env.default
     source "$VERSITYGW_TEST_ENV"
   fi
+  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
+    # shellcheck source=/.secrets
+    source "$SECRETS_FILE"
+  else
+    echo "Warning: no secrets file found"
+  fi
 
   check_exe_params || check_result=$?
   if [[ $check_result -ne 0 ]]; then
@@ -48,33 +78,143 @@ start_versity() {
     return 1
   fi
 
-  base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE"
-  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
-    base_command+=" --cert $CERT --key $KEY"
+  if [ "$RUN_VERSITYGW" == "true" ]; then
+    run_versity_app || run_result=$?
+    if [[ $run_result -ne 0 ]]; then
+      echo "error starting versity apps"
+      return 1
+    fi
   fi
-  base_command+=" $BACKEND $LOCAL_FOLDER &"
-  eval "$base_command"
 
-  versitygw_pid=$!
-  export versitygw_pid \
-    AWS_ACCESS_KEY_ID \
-    AWS_SECRET_ACCESS_KEY \
-    VERSITY_EXE \
-    BACKEND \
-    AWS_PROFILE \
-    LOCAL_FOLDER \
-    AWS_ENDPOINT_URL
+  export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
+}
+
+start_versity_process() {
+  if [[ $# -ne 1 ]]; then
+    echo "start versity process function requires number"
+    return 1
+  fi
+  create_test_file_folder || create_result=$?
+  if [[ $create_result -ne 0 ]]; then
+    echo "error creating test log folder"
+    return 1
+  fi
+  base_command+=(">" "$test_file_folder/versity_log_$1.txt" "2>&1")
+  ("${base_command[@]}") &
+  if [[ $? -ne 0 ]]; then
+    echo "error running versitygw command: $(cat "$test_file_folder/versity_log_$1.txt")"
+    return 1
+  fi
+  eval versitygw_pid_"$1"=$!
+  local pid
+  eval pid=\$versitygw_pid_"$1"
+  sleep 1
+
+  local proc_check
+  check_result=$(kill -0 $pid 2>&1) || proc_check=$?
+  if [[ $proc_check -ne 0 ]]; then
+    echo "versitygw failed to start: $check_result"
+    echo "log data: $(cat "$test_file_folder/versity_log_$1.txt")"
+    return 1
+  fi
+  export versitygw_pid_"$1"
+}
+
+run_versity_app_posix() {
+  if [[ $# -ne 3 ]]; then
+    echo "run versity app w/posix command requires access ID, secret key, process number"
+    return 1
+  fi
+  base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION")
+  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
+    base_command+=(--cert "$CERT" --key "$KEY")
+  fi
+  base_command+=(posix "$LOCAL_FOLDER")
+  export base_command
+
+  local versity_result
+  start_versity_process "$3" || versity_result=$?
+  if [[ $versity_result -ne 0 ]]; then
+    echo "error starting versity process"
+    return 1
+  fi
+  return 0
+}
+
+run_versity_app_s3() {
+  if [[ $# -ne 1 ]]; then
+    echo "run versity app w/s3 command requires process number"
+    return 1
+  fi
+  base_command=("$VERSITY_EXE" --port=":7071" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY")
+  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
+    base_command+=(--cert "$CERT" --key "$KEY")
+  fi
+  base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com)
+  export base_command
+
+  local versity_result
+  start_versity_process "$1" || versity_result=$?
+  if [[ $versity_result -ne 0 ]]; then
+    echo "error starting versity process"
+    return 1
+  fi
+  return 0
+}
+
+run_versity_app() {
+  if [[ $BACKEND == 'posix' ]]; then
+    run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
+    if [[ $result_one -ne 0 ]]; then
+      echo "error starting versity app"
+      return 1
+    fi
+  elif [[ $BACKEND == 's3' ]]; then
+    run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
+    if [[ $result_one -ne 0 ]]; then
+      echo "error starting versity app"
+      return 1
+    fi
+    run_versity_app_s3 "2" || result_two=$?
+    if [[ $result_two -ne 0 ]]; then
+      echo "error starting second versity app"
+      return 1
+    fi
+  else
+    echo "unrecognized backend type $BACKEND"
+    return 1
+  fi
+}
+
+stop_single_process() {
+  if [[ $# -ne 1 ]]; then
+    echo "stop single process function requires process ID"
+    return 1
+  fi
+  if ps -p "$1" > /dev/null; then
+    kill "$1"
+    wait "$1" || true
+  else
+    echo "Process with PID $1 does not exist."
+  fi
 }
 
 stop_versity() {
-  if [ -n "$versitygw_pid" ]; then
-    if ps -p "$versitygw_pid" > /dev/null; then
-      kill "$versitygw_pid"
-      wait "$versitygw_pid" || true
-    else
-      echo "Process with PID $versitygw_pid does not exist."
+  if [ "$RUN_VERSITYGW" == "false" ]; then
+    return
+  fi
+  local result_one
+  local result_two
+  # shellcheck disable=SC2154
+  stop_single_process "$versitygw_pid_1" || result_one=$?
+  if [[ $result_one -ne 0 ]]; then
+    echo "error stopping versity process"
+  fi
+  if [[ $BACKEND == 's3' ]]; then
+    # shellcheck disable=SC2154
+    stop_single_process "$versitygw_pid_2" || result_two=$?
+    if [[ $result_two -ne 0 ]]; then
+      echo "error stopping versity process two"
     fi
-  else
-    echo "versitygw_pid is not set or empty."
   fi
 }

versitygw.spec.in

@@ -1,31 +0,0 @@
-%global debug_package %{nil}
-%define pkg_version @@VERSION@@
-
-Name:		versitygw
-Version:	%{pkg_version}
-Release:	1%{?dist}
-Summary:	Versity S3 Gateway
-
-License:	Apache-2.0
-URL:		https://github.com/versity/versitygw
-Source0:	%{name}-%{version}.tar.gz
-%description
-The S3 gateway is an S3 protocol translator that allows an S3 client
-to access the supported backend storage as if it was a native S3 service.
-
-BuildRequires:	golang >= 1.20
-
-%prep
-%setup
-
-%build
-make
-
-%install
-mkdir -p %{buildroot}%{_bindir}
-install -m 0755 %{name} %{buildroot}%{_bindir}/
-
-%post
-
-%files
-%{_bindir}/%{name}