Merge pull request #473 from versity/test_cmdline_presign

Test cmdline presign

.github/workflows/system.yml

@@ -33,11 +33,16 @@ jobs:
         run: |
           sudo apt-get install s3cmd
 
-      - name: Build and run
+      - name: Install mc
+        run: |
+          curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
+          chmod 755 /usr/local/bin/mc
+
+      - name: Build and run, posix backend
         run: |
           make testbin
-          export AWS_ACCESS_KEY_ID=user
-          export AWS_SECRET_ACCESS_KEY=pass
+          export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+          export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
@@ -46,4 +51,23 @@ jobs:
           export WORKSPACE=$GITHUB_WORKSPACE
           openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
+          mkdir /tmp/cover
+          VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+
+      #- name: Build and run, s3 backend
+      #  run: |
+      #    make testbin
+      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+      #    export AWS_REGION=us-east-1
+      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
+      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
+      #    aws configure set aws_region $AWS_REGION --profile versity_s3
+      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
+      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+      #    export WORKSPACE=$GITHUB_WORKSPACE
+      #    VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+
+      - name: Coverage report
+        run: |
+          go tool covdata percent -i=/tmp/cover

.gitignore

@@ -41,7 +41,7 @@ VERSION
 dist/
 
 # secrets file for local github-actions testing
-tests/.secrets
+tests/.secrets*
 
 # IAM users files often created in testing
 users.json

.goreleaser.yaml

@@ -6,6 +6,7 @@ builds:
   - goos:
       - linux
       - darwin
+      - freebsd
       # windows is untested, we can start doing windows releases
       # if someone is interested in taking on testing
       # - windows
@@ -31,11 +32,28 @@ archives:
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
+
+    # Set this to true if you want all files in the archive to be in a single directory.
+    # If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
+    # you'll get a folder 'goreleaser_Linux_arm64'.
+    # If set to false, all files are extracted separately.
+    # You can also set it to a custom folder name (templating is supported).
+    wrap_in_directory: true
+
     # use zip for windows archives
     format_overrides:
     - goos: windows
       format: zip
 
+    # Additional files/globs you want to add to the archive.
+    #
+    # Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
+    # Templates: allowed
+    files:
+      - README.md
+      - LICENSE
+      - NOTICE
+
 checksum:
   name_template: 'checksums.txt'
 
@@ -82,6 +100,27 @@ nfpms:
 
     rpm:
       group: "System Environment/Daemons"
+      # RPM specific scripts.
+      scripts:
+        # The pretrans script runs before all RPM package transactions / stages.
+        #pretrans: ./extra/pretrans.sh
+        # The posttrans script runs after all RPM package transactions / stages.
+        posttrans: ./extra/posttrans.sh
+
+    contents:
+      - src: extra/versitygw@.service
+        dst: /lib/systemd/system/versitygw@.service
+
+      - src: extra/example.conf
+        dst: /etc/versitygw.d/example.conf
+        type: config
+  
+      - dst: /etc/versitygw.d
+        type: dir
+        file_info:
+          mode: 0700
+
+
 
 # yaml-language-server: $schema=https://goreleaser.com/static/schema.json
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile_test_bats

@@ -1,6 +1,9 @@
 FROM --platform=linux/arm64 ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG SECRETS_FILE=tests/.secrets
+ARG CONFIG_FILE=tests/.env.docker
+
 ENV TZ=Etc/UTC
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -12,6 +15,7 @@ RUN apt-get update && \
     tzdata \
     s3cmd \
     jq \
+    bc \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*
@@ -19,8 +23,16 @@ RUN apt-get update && \
 # Set working directory
 WORKDIR /tmp
 
+# Install AWS cli
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
 
+# Install mc
+RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
+  --create-dirs \
+  -o /usr/local/minio-binaries/mc && \
+chmod -R 755 /usr/local/minio-binaries
+ENV PATH="/usr/local/minio-binaries":${PATH}
+
 # Download Go 1.21 (adjust the version and platform as needed)
 RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
 
@@ -40,6 +52,7 @@ RUN groupadd -r tester && useradd -r -g tester tester
 RUN mkdir /home/tester && chown tester:tester /home/tester
 ENV HOME=/home/tester
 
+# install bats
 RUN git clone https://github.com/bats-core/bats-core.git && \
     cd bats-core && \
     ./install.sh /home/tester
@@ -48,11 +61,12 @@ USER tester
 COPY --chown=tester:tester . /home/tester
 
 WORKDIR /home/tester
-RUN cp tests/.env.docker.default tests/.env.docker
+RUN cp ${CONFIG_FILE}.default $CONFIG_FILE
+#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
 RUN cp tests/s3cfg.local.default tests/s3cfg.local
 RUN make
 
-RUN . tests/.secrets && \
+RUN . $SECRETS_FILE && \
     export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
@@ -65,6 +79,6 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
         -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
 
 ENV WORKSPACE=.
-ENV VERSITYGW_TEST_ENV=tests/.env.docker
+ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
 
-CMD ["tests/run_all.sh"]
+CMD ["tests/run_all.sh"]

Makefile

@@ -59,20 +59,13 @@ cleanall: clean
 	rm -f $(BIN)
 	rm -f versitygw-*.tar
 	rm -f versitygw-*.tar.gz
-	rm -f versitygw.spec
-
-%.spec: %.spec.in
-	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
-	mv $@+ $@
 
 TARFILE = $(BIN)-$(VERSION).tar
 
-dist: $(BIN).spec
+dist:
 	echo $(VERSION) >VERSION
 	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
-	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
 	rm -f VERSION
-	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
 
 # Creates and runs S3 gateway instance in a docker container

README.md

@@ -8,19 +8,33 @@
 
  [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-**Current status:** Ready for general testing, Issue reports welcome.
-
-**News:**<br>
+### Binary release builds
+Download [latest release](https://github.com/versity/versitygw/releases)
+ | Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
+ |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
+ |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
+ 
+### News
 * New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
 
+### Mailing List
+Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
 
+### Documentation
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
+### Need help?
+Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
+<br>
+Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
+
+### Use Cases
 * Share filesystem directory via S3 protocol
 * Proxy S3 requests to S3 storage
 * Simple to deploy S3 server with a single command
 * Protocol compatibility in `posix` allows common access to files via posix or S3 
 
+### Overview
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.

auth/acl.go

@@ -15,12 +15,14 @@
 package auth
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -203,13 +205,14 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
-	if isRoot {
-		return nil
-	}
+func verifyACL(acl ACL, access string, permission types.Permission) error {
+	// Default disabled ACL case
+	if acl.ACL == "" && len(acl.Grantees) == 0 {
+		if acl.Owner == access {
+			return nil
+		}
 
-	if acl.Owner == access {
-		return nil
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
 	}
 
 	if acl.ACL != "" {
@@ -273,3 +276,78 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 	// Return access denied in all other cases
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
+
+type AccessOptions struct {
+	Acl           ACL
+	AclPermission types.Permission
+	IsRoot        bool
+	Acc           Account
+	Bucket        string
+	Object        string
+	Action        Action
+}
+
+func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+	if opts.Acc.Access == opts.Acl.Owner {
+		return nil
+	}
+
+	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
+		return err
+	}
+	if err := verifyBucketPolicy(ctx, be, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
+	if opts.IsRoot {
+		return nil
+	}
+	if opts.Acc.Role == RoleAdmin {
+		return nil
+	}
+
+	// Verify destination bucket access
+	if err := VerifyAccess(ctx, be, opts); err != nil {
+		return err
+	}
+	// Verify source bucket access
+	srcBucket, srcObject, found := strings.Cut(copySource, "/")
+	if !found {
+		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	}
+
+	// Get source bucket ACL
+	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
+	if err != nil {
+		return err
+	}
+
+	var srcBucketAcl ACL
+	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
+		return err
+	}
+
+	if err := VerifyAccess(ctx, be, AccessOptions{
+		Acl:           srcBucketAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        opts.IsRoot,
+		Acc:           opts.Acc,
+		Bucket:        srcBucket,
+		Object:        srcObject,
+		Action:        GetObjectAction,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}

auth/bucket_policy.go

@@ -0,0 +1,145 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3err"
+)
+
+type BucketPolicy struct {
+	Statement []BucketPolicyItem `json:"Statement"`
+}
+
+func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
+	for _, statement := range bp.Statement {
+		err := statement.Validate(bucket, iam)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
+	for _, statement := range bp.Statement {
+		if statement.isAllowed(principal, action, resource) {
+			return true
+		}
+	}
+
+	return false
+}
+
+type BucketPolicyItem struct {
+	Effect     BucketPolicyAccessType `json:"Effect"`
+	Principals Principals             `json:"Principal"`
+	Actions    Actions                `json:"Action"`
+	Resources  Resources              `json:"Resource"`
+}
+
+func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
+	if err := bpi.Effect.Validate(); err != nil {
+		return err
+	}
+	if err := bpi.Principals.Validate(iam); err != nil {
+		return err
+	}
+	if err := bpi.Resources.Validate(bucket); err != nil {
+		return err
+	}
+
+	containsObjectAction := bpi.Resources.ContainsObjectPattern()
+	containsBucketAction := bpi.Resources.ContainsBucketPattern()
+
+	for action := range bpi.Actions {
+		isObjectAction := action.IsObjectAction()
+		if isObjectAction && !containsObjectAction {
+			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		}
+		if !isObjectAction && !containsBucketAction {
+			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		}
+	}
+
+	return nil
+}
+
+func (bpi *BucketPolicyItem) isAllowed(principal string, action Action, resource string) bool {
+	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
+		switch bpi.Effect {
+		case BucketPolicyAccessTypeAllow:
+			return true
+		case BucketPolicyAccessTypeDeny:
+			return false
+		}
+	}
+
+	return false
+}
+
+func getMalformedPolicyError(err error) error {
+	return s3err.APIError{
+		Code:           "MalformedPolicy",
+		Description:    err.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
+func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
+	var policy BucketPolicy
+	if err := json.Unmarshal(policyBin, &policy); err != nil {
+		return getMalformedPolicyError(err)
+	}
+
+	if err := policy.Validate(bucket, iam); err != nil {
+		return getMalformedPolicyError(err)
+	}
+
+	return nil
+}
+
+func verifyBucketPolicy(ctx context.Context, be backend.Backend, access, bucket, object string, action Action) error {
+	policyDoc, err := be.GetBucketPolicy(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	// If bucket policy is not set
+	if len(policyDoc) == 0 {
+		return nil
+	}
+
+	var bucketPolicy BucketPolicy
+	if err := json.Unmarshal(policyDoc, &bucketPolicy); err != nil {
+		return err
+	}
+
+	resource := bucket
+	if object != "" {
+		resource += "" + object
+	}
+
+	if !bucketPolicy.isAllowed(access, action, resource) {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}

auth/bucket_policy_actions.go

@@ -0,0 +1,218 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Action string
+
+const (
+	GetBucketAclAction               Action = "s3:GetBucketAcl"
+	CreateBucketAction               Action = "s3:CreateBucket"
+	PutBucketAclAction               Action = "s3:PutBucketAcl"
+	DeleteBucketAction               Action = "s3:DeleteBucket"
+	PutBucketVersioningAction        Action = "s3:PutBucketVersioning"
+	GetBucketVersioningAction        Action = "s3:GetBucketVersioning"
+	PutBucketPolicyAction            Action = "s3:PutBucketPolicy"
+	GetBucketPolicyAction            Action = "s3:GetBucketPolicy"
+	DeleteBucketPolicyAction         Action = "s3:DeleteBucketPolicy"
+	AbortMultipartUploadAction       Action = "s3:AbortMultipartUpload"
+	ListMultipartUploadPartsAction   Action = "s3:ListMultipartUploadParts"
+	ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
+	PutObjectAction                  Action = "s3:PutObject"
+	GetObjectAction                  Action = "s3:GetObject"
+	DeleteObjectAction               Action = "s3:DeleteObject"
+	GetObjectAclAction               Action = "s3:GetObjectAcl"
+	GetObjectAttributesAction        Action = "s3:GetObjectAttributes"
+	PutObjectAclAction               Action = "s3:PutObjectAcl"
+	RestoreObjectAction              Action = "s3:RestoreObject"
+	GetBucketTaggingAction           Action = "s3:GetBucketTagging"
+	PutBucketTaggingAction           Action = "s3:PutBucketTagging"
+	GetObjectTaggingAction           Action = "s3:GetObjectTagging"
+	PutObjectTaggingAction           Action = "s3:PutObjectTagging"
+	DeleteObjectTaggingAction        Action = "s3:DeleteObjectTagging"
+	ListBucketVersionsAction         Action = "s3:ListBucketVersions"
+	ListBucketAction                 Action = "s3:ListBucket"
+	AllActions                       Action = "s3:*"
+)
+
+var supportedActionList = map[Action]struct{}{
+	GetBucketAclAction:               {},
+	CreateBucketAction:               {},
+	PutBucketAclAction:               {},
+	DeleteBucketAction:               {},
+	PutBucketVersioningAction:        {},
+	GetBucketVersioningAction:        {},
+	PutBucketPolicyAction:            {},
+	GetBucketPolicyAction:            {},
+	DeleteBucketPolicyAction:         {},
+	AbortMultipartUploadAction:       {},
+	ListMultipartUploadPartsAction:   {},
+	ListBucketMultipartUploadsAction: {},
+	PutObjectAction:                  {},
+	GetObjectAction:                  {},
+	DeleteObjectAction:               {},
+	GetObjectAclAction:               {},
+	GetObjectAttributesAction:        {},
+	PutObjectAclAction:               {},
+	RestoreObjectAction:              {},
+	GetBucketTaggingAction:           {},
+	PutBucketTaggingAction:           {},
+	GetObjectTaggingAction:           {},
+	PutObjectTaggingAction:           {},
+	DeleteObjectTaggingAction:        {},
+	ListBucketVersionsAction:         {},
+	ListBucketAction:                 {},
+	AllActions:                       {},
+}
+
+var supportedObjectActionList = map[Action]struct{}{
+	AbortMultipartUploadAction:     {},
+	ListMultipartUploadPartsAction: {},
+	PutObjectAction:                {},
+	GetObjectAction:                {},
+	DeleteObjectAction:             {},
+	GetObjectAclAction:             {},
+	GetObjectAttributesAction:      {},
+	PutObjectAclAction:             {},
+	RestoreObjectAction:            {},
+	GetObjectTaggingAction:         {},
+	PutObjectTaggingAction:         {},
+	DeleteObjectTaggingAction:      {},
+	AllActions:                     {},
+}
+
+// Validates Action: it should either wildcard match with supported actions list or be in it
+func (a Action) IsValid() error {
+	if !strings.HasPrefix(string(a), "s3:") {
+		return fmt.Errorf("invalid action: %v", a)
+	}
+
+	if a == AllActions {
+		return nil
+	}
+
+	if a[len(a)-1] == '*' {
+		pattern := strings.TrimSuffix(string(a), "*")
+		for act := range supportedActionList {
+			if strings.HasPrefix(string(act), pattern) {
+				return nil
+			}
+		}
+
+		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+	}
+
+	_, found := supportedActionList[a]
+	if !found {
+		return fmt.Errorf("unsupported action: %v", a)
+	}
+	return nil
+}
+
+// Checks if the action is object action
+func (a Action) IsObjectAction() bool {
+	if a[len(a)-1] == '*' {
+		pattern := strings.TrimSuffix(string(a), "*")
+		for act := range supportedObjectActionList {
+			if strings.HasPrefix(string(act), pattern) {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	_, found := supportedObjectActionList[a]
+	return found
+}
+
+func (a Action) WildCardMatch(act Action) bool {
+	if strings.HasSuffix(string(a), "*") {
+		pattern := strings.TrimSuffix(string(a), "*")
+		return strings.HasPrefix(string(act), pattern)
+	}
+	return false
+}
+
+type Actions map[Action]struct{}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (a *Actions) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("actions can't be empty")
+		}
+		*a = make(Actions)
+		for _, s := range ss {
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("actions can't be empty")
+			}
+			*a = make(Actions)
+			err = a.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Validates and adds a new Action to Actions map
+func (a Actions) Add(str string) error {
+	action := Action(str)
+	err := action.IsValid()
+	if err != nil {
+		return err
+	}
+
+	a[action] = struct{}{}
+	return nil
+}
+
+func (a Actions) FindMatch(action Action) bool {
+	_, ok := a[AllActions]
+	if ok {
+		return true
+	}
+	// First O(1) check for non wildcard actions
+	_, found := a[action]
+	if found {
+		return true
+	}
+
+	for act := range a {
+		if strings.HasSuffix(string(act), "*") && act.WildCardMatch(action) {
+			return true
+		}
+	}
+
+	return false
+}

auth/bucket_policy_effect.go

@@ -0,0 +1,34 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import "fmt"
+
+type BucketPolicyAccessType string
+
+const (
+	BucketPolicyAccessTypeDeny  BucketPolicyAccessType = "Deny"
+	BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
+)
+
+// Checks policy statement Effect to be valid ("Deny", "Allow")
+func (bpat BucketPolicyAccessType) Validate() error {
+	switch bpat {
+	case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
+		return nil
+	}
+
+	return fmt.Errorf("invalid effect: %v", bpat)
+}

auth/bucket_policy_principals.go

@@ -0,0 +1,97 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type Principals map[string]struct{}
+
+func (p Principals) Add(key string) {
+	p[key] = struct{}{}
+}
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (p *Principals) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("principals can't be empty")
+		}
+		*p = make(Principals)
+		for _, s := range ss {
+			p.Add(s)
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("principals can't be empty")
+			}
+			*p = make(Principals)
+			p.Add(s)
+		}
+	}
+
+	return err
+}
+
+// Converts Principals map to a slice, by omitting "*"
+func (p Principals) ToSlice() []string {
+	principals := []string{}
+	for p := range p {
+		if p == "*" {
+			continue
+		}
+		principals = append(principals, p)
+	}
+
+	return principals
+}
+
+// Validates Principals by checking user account access keys existence
+func (p Principals) Validate(iam IAMService) error {
+	_, containsWildCard := p["*"]
+	if containsWildCard {
+		if len(p) == 1 {
+			return nil
+		}
+		return fmt.Errorf("principals should either contain * or user access keys")
+	}
+
+	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
+	if err != nil {
+		return err
+	}
+	if len(accs) > 0 {
+		return fmt.Errorf("user accounts don't exist: %v", accs)
+	}
+
+	return nil
+}
+
+func (p Principals) Contains(userAccess string) bool {
+	// "*" means it matches for any user account
+	_, ok := p["*"]
+	if ok {
+		return true
+	}
+
+	_, found := p[userAccess]
+	return found
+}

auth/bucket_policy_resources.go

@@ -0,0 +1,142 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Resources map[string]struct{}
+
+const ResourceArnPrefix = "arn:aws:s3:::"
+
+// Override UnmarshalJSON method to decode both []string and string properties
+func (r *Resources) UnmarshalJSON(data []byte) error {
+	ss := []string{}
+	var err error
+	if err = json.Unmarshal(data, &ss); err == nil {
+		if len(ss) == 0 {
+			return fmt.Errorf("resources can't be empty")
+		}
+		*r = make(Resources)
+		for _, s := range ss {
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		var s string
+		if err = json.Unmarshal(data, &s); err == nil {
+			if s == "" {
+				return fmt.Errorf("resources can't be empty")
+			}
+			*r = make(Resources)
+			err = r.Add(s)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}
+
+// Adds and validates a new resource to Resources map
+func (r Resources) Add(rc string) error {
+	ok, pattern := isValidResource(rc)
+	if !ok {
+		return fmt.Errorf("invalid resource: %v", rc)
+	}
+
+	_, found := r[pattern]
+	if found {
+		return fmt.Errorf("duplicate resource: %v", rc)
+	}
+
+	r[pattern] = struct{}{}
+
+	return nil
+}
+
+// Checks if the resources contain object pattern
+func (r Resources) ContainsObjectPattern() bool {
+	for resource := range r {
+		if resource == "*" || strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Checks if the resources contain bucket pattern
+func (r Resources) ContainsBucketPattern() bool {
+	for resource := range r {
+		if resource == "*" || !strings.Contains(resource, "/") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
+func (r Resources) Validate(bucket string) error {
+	for resource := range r {
+		if !strings.HasPrefix(resource, bucket) {
+			return fmt.Errorf("incorrect bucket name in %v", resource)
+		}
+	}
+
+	return nil
+}
+
+func (r Resources) FindMatch(resource string) bool {
+	for res := range r {
+		if strings.HasSuffix(res, "*") {
+			pattern := strings.TrimSuffix(res, "*")
+			if strings.HasPrefix(resource, pattern) {
+				return true
+			}
+		} else {
+			if res == resource {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// Checks the resource to have arn prefix and not starting with /
+func isValidResource(rc string) (isValid bool, pattern string) {
+	if !strings.HasPrefix(rc, ResourceArnPrefix) {
+		return false, ""
+	}
+
+	res := strings.TrimPrefix(rc, ResourceArnPrefix)
+	if res == "" {
+		return false, ""
+	}
+	// The resource can't start with / (bucket name comes first)
+	if strings.HasPrefix(res, "/") {
+		return false, ""
+	}
+
+	return true, res
+}

aws/NOTICE.txt

@@ -1,3 +1,4 @@
 AWS SDK for Go
 Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 Copyright 2014-2015 Stripe, Inc.
+Copyright 2024 Versity Software

aws/internal/auth/auth.go

@@ -1,45 +0,0 @@
-package auth
-
-import (
-	"github.com/aws/smithy-go/auth"
-	smithyhttp "github.com/aws/smithy-go/transport/http"
-)
-
-// HTTPAuthScheme is the SDK's internal implementation of smithyhttp.AuthScheme
-// for pre-existing implementations where the signer was added to client
-// config. SDK clients will key off of this type and ensure per-operation
-// updates to those signers persist on the scheme itself.
-type HTTPAuthScheme struct {
-	schemeID string
-	signer   smithyhttp.Signer
-}
-
-var _ smithyhttp.AuthScheme = (*HTTPAuthScheme)(nil)
-
-// NewHTTPAuthScheme returns an auth scheme instance with the given config.
-func NewHTTPAuthScheme(schemeID string, signer smithyhttp.Signer) *HTTPAuthScheme {
-	return &HTTPAuthScheme{
-		schemeID: schemeID,
-		signer:   signer,
-	}
-}
-
-// SchemeID identifies the auth scheme.
-func (s *HTTPAuthScheme) SchemeID() string {
-	return s.schemeID
-}
-
-// IdentityResolver gets the identity resolver for the auth scheme.
-func (s *HTTPAuthScheme) IdentityResolver(o auth.IdentityResolverOptions) auth.IdentityResolver {
-	return o.GetIdentityResolver(s.schemeID)
-}
-
-// Signer gets the signer for the auth scheme.
-func (s *HTTPAuthScheme) Signer() smithyhttp.Signer {
-	return s.signer
-}
-
-// WithSigner returns a new instance of the auth scheme with the updated signer.
-func (s *HTTPAuthScheme) WithSigner(signer smithyhttp.Signer) *HTTPAuthScheme {
-	return NewHTTPAuthScheme(s.schemeID, signer)
-}

aws/internal/auth/scheme.go

@@ -1,191 +0,0 @@
-package auth
-
-import (
-	"context"
-	"fmt"
-
-	smithy "github.com/aws/smithy-go"
-	"github.com/aws/smithy-go/middleware"
-)
-
-// SigV4 is a constant representing
-// Authentication Scheme Signature Version 4
-const SigV4 = "sigv4"
-
-// SigV4A is a constant representing
-// Authentication Scheme Signature Version 4A
-const SigV4A = "sigv4a"
-
-// SigV4S3Express identifies the S3 S3Express auth scheme.
-const SigV4S3Express = "sigv4-s3express"
-
-// None is a constant representing the
-// None Authentication Scheme
-const None = "none"
-
-// SupportedSchemes is a data structure
-// that indicates the list of supported AWS
-// authentication schemes
-var SupportedSchemes = map[string]bool{
-	SigV4:          true,
-	SigV4A:         true,
-	SigV4S3Express: true,
-	None:           true,
-}
-
-// AuthenticationScheme is a representation of
-// AWS authentication schemes
-type AuthenticationScheme interface {
-	isAuthenticationScheme()
-}
-
-// AuthenticationSchemeV4 is a AWS SigV4 representation
-type AuthenticationSchemeV4 struct {
-	Name                  string
-	SigningName           *string
-	SigningRegion         *string
-	DisableDoubleEncoding *bool
-}
-
-func (a *AuthenticationSchemeV4) isAuthenticationScheme() {}
-
-// AuthenticationSchemeV4A is a AWS SigV4A representation
-type AuthenticationSchemeV4A struct {
-	Name                  string
-	SigningName           *string
-	SigningRegionSet      []string
-	DisableDoubleEncoding *bool
-}
-
-func (a *AuthenticationSchemeV4A) isAuthenticationScheme() {}
-
-// AuthenticationSchemeNone is a representation for the none auth scheme
-type AuthenticationSchemeNone struct{}
-
-func (a *AuthenticationSchemeNone) isAuthenticationScheme() {}
-
-// NoAuthenticationSchemesFoundError is used in signaling
-// that no authentication schemes have been specified.
-type NoAuthenticationSchemesFoundError struct{}
-
-func (e *NoAuthenticationSchemesFoundError) Error() string {
-	return fmt.Sprint("No authentication schemes specified.")
-}
-
-// UnSupportedAuthenticationSchemeSpecifiedError is used in
-// signaling that only unsupported authentication schemes
-// were specified.
-type UnSupportedAuthenticationSchemeSpecifiedError struct {
-	UnsupportedSchemes []string
-}
-
-func (e *UnSupportedAuthenticationSchemeSpecifiedError) Error() string {
-	return fmt.Sprint("Unsupported authentication scheme specified.")
-}
-
-// GetAuthenticationSchemes extracts the relevant authentication scheme data
-// into a custom strongly typed Go data structure.
-func GetAuthenticationSchemes(p *smithy.Properties) ([]AuthenticationScheme, error) {
-	var result []AuthenticationScheme
-	if !p.Has("authSchemes") {
-		return nil, &NoAuthenticationSchemesFoundError{}
-	}
-
-	authSchemes, _ := p.Get("authSchemes").([]interface{})
-
-	var unsupportedSchemes []string
-	for _, scheme := range authSchemes {
-		authScheme, _ := scheme.(map[string]interface{})
-
-		version := authScheme["name"].(string)
-		switch version {
-		case SigV4, SigV4S3Express:
-			v4Scheme := AuthenticationSchemeV4{
-				Name:                  version,
-				SigningName:           getSigningName(authScheme),
-				SigningRegion:         getSigningRegion(authScheme),
-				DisableDoubleEncoding: getDisableDoubleEncoding(authScheme),
-			}
-			result = append(result, AuthenticationScheme(&v4Scheme))
-		case SigV4A:
-			v4aScheme := AuthenticationSchemeV4A{
-				Name:                  SigV4A,
-				SigningName:           getSigningName(authScheme),
-				SigningRegionSet:      getSigningRegionSet(authScheme),
-				DisableDoubleEncoding: getDisableDoubleEncoding(authScheme),
-			}
-			result = append(result, AuthenticationScheme(&v4aScheme))
-		case None:
-			noneScheme := AuthenticationSchemeNone{}
-			result = append(result, AuthenticationScheme(&noneScheme))
-		default:
-			unsupportedSchemes = append(unsupportedSchemes, authScheme["name"].(string))
-			continue
-		}
-	}
-
-	if len(result) == 0 {
-		return nil, &UnSupportedAuthenticationSchemeSpecifiedError{
-			UnsupportedSchemes: unsupportedSchemes,
-		}
-	}
-
-	return result, nil
-}
-
-type disableDoubleEncoding struct{}
-
-// SetDisableDoubleEncoding sets or modifies the disable double encoding option
-// on the context.
-//
-// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
-// to clear all stack values.
-func SetDisableDoubleEncoding(ctx context.Context, value bool) context.Context {
-	return middleware.WithStackValue(ctx, disableDoubleEncoding{}, value)
-}
-
-// GetDisableDoubleEncoding retrieves the disable double encoding option
-// from the context.
-//
-// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
-// to clear all stack values.
-func GetDisableDoubleEncoding(ctx context.Context) (value bool, ok bool) {
-	value, ok = middleware.GetStackValue(ctx, disableDoubleEncoding{}).(bool)
-	return value, ok
-}
-
-func getSigningName(authScheme map[string]interface{}) *string {
-	signingName, ok := authScheme["signingName"].(string)
-	if !ok || signingName == "" {
-		return nil
-	}
-	return &signingName
-}
-
-func getSigningRegionSet(authScheme map[string]interface{}) []string {
-	untypedSigningRegionSet, ok := authScheme["signingRegionSet"].([]interface{})
-	if !ok {
-		return nil
-	}
-	signingRegionSet := []string{}
-	for _, item := range untypedSigningRegionSet {
-		signingRegionSet = append(signingRegionSet, item.(string))
-	}
-	return signingRegionSet
-}
-
-func getSigningRegion(authScheme map[string]interface{}) *string {
-	signingRegion, ok := authScheme["signingRegion"].(string)
-	if !ok || signingRegion == "" {
-		return nil
-	}
-	return &signingRegion
-}
-
-func getDisableDoubleEncoding(authScheme map[string]interface{}) *bool {
-	disableDoubleEncoding, ok := authScheme["disableDoubleEncoding"].(bool)
-	if !ok {
-		return nil
-	}
-	return &disableDoubleEncoding
-}

aws/internal/auth/scheme_test.go

@@ -1,101 +0,0 @@
-package auth
-
-import (
-	"testing"
-
-	smithy "github.com/aws/smithy-go"
-)
-
-func TestV4(t *testing.T) {
-
-	propsV4 := smithy.Properties{}
-
-	propsV4.Set("authSchemes", interface{}([]interface{}{
-		map[string]interface{}{
-			"disableDoubleEncoding": true,
-			"name":                  "sigv4",
-			"signingName":           "s3",
-			"signingRegion":         "us-west-2",
-		},
-	}))
-
-	result, err := GetAuthenticationSchemes(&propsV4)
-	if err != nil {
-		t.Fatalf("Did not expect error, got %v", err)
-	}
-
-	_, ok := result[0].(AuthenticationScheme)
-	if !ok {
-		t.Fatalf("Did not get expected AuthenticationScheme. %v", result[0])
-	}
-
-	v4Scheme, ok := result[0].(*AuthenticationSchemeV4)
-	if !ok {
-		t.Fatalf("Did not get expected AuthenticationSchemeV4. %v", result[0])
-	}
-
-	if v4Scheme.Name != "sigv4" {
-		t.Fatalf("Did not get expected AuthenticationSchemeV4 signer version name")
-	}
-
-}
-
-func TestV4A(t *testing.T) {
-
-	propsV4A := smithy.Properties{}
-
-	propsV4A.Set("authSchemes", []interface{}{
-		map[string]interface{}{
-			"disableDoubleEncoding": true,
-			"name":                  "sigv4a",
-			"signingName":           "s3",
-			"signingRegionSet":      []string{"*"},
-		},
-	})
-
-	result, err := GetAuthenticationSchemes(&propsV4A)
-	if err != nil {
-		t.Fatalf("Did not expect error, got %v", err)
-	}
-
-	_, ok := result[0].(AuthenticationScheme)
-	if !ok {
-		t.Fatalf("Did not get expected AuthenticationScheme. %v", result[0])
-	}
-
-	v4AScheme, ok := result[0].(*AuthenticationSchemeV4A)
-	if !ok {
-		t.Fatalf("Did not get expected AuthenticationSchemeV4A. %v", result[0])
-	}
-
-	if v4AScheme.Name != "sigv4a" {
-		t.Fatalf("Did not get expected AuthenticationSchemeV4A signer version name")
-	}
-
-}
-
-func TestV4S3Express(t *testing.T) {
-	props := smithy.Properties{}
-	props.Set("authSchemes", []interface{}{
-		map[string]interface{}{
-			"name":                  SigV4S3Express,
-			"signingName":           "s3",
-			"signingRegion":         "us-east-1",
-			"disableDoubleEncoding": true,
-		},
-	})
-
-	result, err := GetAuthenticationSchemes(&props)
-	if err != nil {
-		t.Fatalf("Did not expect error, got %v", err)
-	}
-
-	scheme, ok := result[0].(*AuthenticationSchemeV4)
-	if !ok {
-		t.Fatalf("Did not get expected AuthenticationSchemeV4. %v", result[0])
-	}
-
-	if scheme.Name != SigV4S3Express {
-		t.Fatalf("expected %s, got %s", SigV4S3Express, scheme.Name)
-	}
-}

aws/internal/auth/smithy/bearer_token_adapter.go

@@ -1,43 +0,0 @@
-package smithy
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/aws/smithy-go"
-	"github.com/aws/smithy-go/auth"
-	"github.com/aws/smithy-go/auth/bearer"
-)
-
-// BearerTokenAdapter adapts smithy bearer.Token to smithy auth.Identity.
-type BearerTokenAdapter struct {
-	Token bearer.Token
-}
-
-var _ auth.Identity = (*BearerTokenAdapter)(nil)
-
-// Expiration returns the time of expiration for the token.
-func (v *BearerTokenAdapter) Expiration() time.Time {
-	return v.Token.Expires
-}
-
-// BearerTokenProviderAdapter adapts smithy bearer.TokenProvider to smithy
-// auth.IdentityResolver.
-type BearerTokenProviderAdapter struct {
-	Provider bearer.TokenProvider
-}
-
-var _ (auth.IdentityResolver) = (*BearerTokenProviderAdapter)(nil)
-
-// GetIdentity retrieves a bearer token using the underlying provider.
-func (v *BearerTokenProviderAdapter) GetIdentity(ctx context.Context, _ smithy.Properties) (
-	auth.Identity, error,
-) {
-	token, err := v.Provider.RetrieveBearerToken(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("get token: %w", err)
-	}
-
-	return &BearerTokenAdapter{Token: token}, nil
-}

aws/internal/auth/smithy/bearer_token_signer_adapter.go

@@ -1,35 +0,0 @@
-package smithy
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/aws/smithy-go"
-	"github.com/aws/smithy-go/auth"
-	"github.com/aws/smithy-go/auth/bearer"
-	smithyhttp "github.com/aws/smithy-go/transport/http"
-)
-
-// BearerTokenSignerAdapter adapts smithy bearer.Signer to smithy http
-// auth.Signer.
-type BearerTokenSignerAdapter struct {
-	Signer bearer.Signer
-}
-
-var _ (smithyhttp.Signer) = (*BearerTokenSignerAdapter)(nil)
-
-// SignRequest signs the request with the provided bearer token.
-func (v *BearerTokenSignerAdapter) SignRequest(ctx context.Context, r *smithyhttp.Request, identity auth.Identity, _ smithy.Properties) error {
-	ca, ok := identity.(*BearerTokenAdapter)
-	if !ok {
-		return fmt.Errorf("unexpected identity type: %T", identity)
-	}
-
-	signed, err := v.Signer.SignWithBearerToken(ctx, ca.Token, r)
-	if err != nil {
-		return fmt.Errorf("sign request: %w", err)
-	}
-
-	*r = *signed.(*smithyhttp.Request)
-	return nil
-}

aws/internal/auth/smithy/credentials_adapter.go

@@ -1,46 +0,0 @@
-package smithy
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/smithy-go"
-	"github.com/aws/smithy-go/auth"
-)
-
-// CredentialsAdapter adapts aws.Credentials to auth.Identity.
-type CredentialsAdapter struct {
-	Credentials aws.Credentials
-}
-
-var _ auth.Identity = (*CredentialsAdapter)(nil)
-
-// Expiration returns the time of expiration for the credentials.
-func (v *CredentialsAdapter) Expiration() time.Time {
-	return v.Credentials.Expires
-}
-
-// CredentialsProviderAdapter adapts aws.CredentialsProvider to auth.IdentityResolver.
-type CredentialsProviderAdapter struct {
-	Provider aws.CredentialsProvider
-}
-
-var _ (auth.IdentityResolver) = (*CredentialsProviderAdapter)(nil)
-
-// GetIdentity retrieves AWS credentials using the underlying provider.
-func (v *CredentialsProviderAdapter) GetIdentity(ctx context.Context, _ smithy.Properties) (
-	auth.Identity, error,
-) {
-	if v.Provider == nil {
-		return &CredentialsAdapter{Credentials: aws.Credentials{}}, nil
-	}
-
-	creds, err := v.Provider.Retrieve(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("get credentials: %w", err)
-	}
-
-	return &CredentialsAdapter{Credentials: creds}, nil
-}

aws/internal/auth/smithy/smithy.go

@@ -1,2 +0,0 @@
-// Package smithy adapts concrete AWS auth and signing types to the generic smithy versions.
-package smithy

aws/internal/auth/smithy/v4signer_adapter.go

@@ -1,53 +0,0 @@
-package smithy
-
-import (
-	"context"
-	"fmt"
-
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/aws/smithy-go"
-	"github.com/aws/smithy-go/auth"
-	"github.com/aws/smithy-go/logging"
-	smithyhttp "github.com/aws/smithy-go/transport/http"
-	"github.com/versity/versitygw/aws/internal/sdk"
-)
-
-// V4SignerAdapter adapts v4.HTTPSigner to smithy http.Signer.
-type V4SignerAdapter struct {
-	Signer     v4.HTTPSigner
-	Logger     logging.Logger
-	LogSigning bool
-}
-
-var _ (smithyhttp.Signer) = (*V4SignerAdapter)(nil)
-
-// SignRequest signs the request with the provided identity.
-func (v *V4SignerAdapter) SignRequest(ctx context.Context, r *smithyhttp.Request, identity auth.Identity, props smithy.Properties) error {
-	ca, ok := identity.(*CredentialsAdapter)
-	if !ok {
-		return fmt.Errorf("unexpected identity type: %T", identity)
-	}
-
-	name, ok := smithyhttp.GetSigV4SigningName(&props)
-	if !ok {
-		return fmt.Errorf("sigv4 signing name is required")
-	}
-
-	region, ok := smithyhttp.GetSigV4SigningRegion(&props)
-	if !ok {
-		return fmt.Errorf("sigv4 signing region is required")
-	}
-
-	hash := v4.GetPayloadHash(ctx)
-	err := v.Signer.SignHTTP(ctx, ca.Credentials, r.Request, hash, name, region, sdk.NowTime(), func(o *v4.SignerOptions) {
-		o.DisableURIPathEscaping, _ = smithyhttp.GetDisableDoubleEncoding(&props)
-
-		o.Logger = v.Logger
-		o.LogSigning = v.LogSigning
-	})
-	if err != nil {
-		return fmt.Errorf("sign http: %w", err)
-	}
-
-	return nil
-}

aws/internal/awstesting/assert.go

@@ -1,222 +0,0 @@
-package awstesting
-
-import (
-	"encoding/json"
-	"encoding/xml"
-	"fmt"
-	"net/url"
-	"reflect"
-	"regexp"
-	"sort"
-	"strings"
-	"testing"
-)
-
-// Match is a testing helper to test for testing error by comparing expected
-// with a regular expression.
-func Match(t *testing.T, regex, expected string) {
-	t.Helper()
-
-	if !regexp.MustCompile(regex).Match([]byte(expected)) {
-		t.Errorf("%q\n\tdoes not match /%s/", expected, regex)
-	}
-}
-
-// AssertURL verifies the expected URL is matches the actual.
-func AssertURL(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
-	t.Helper()
-
-	expectURL, err := url.Parse(expect)
-	if err != nil {
-		t.Errorf(errMsg("unable to parse expected URL", err, msgAndArgs))
-		return false
-	}
-	actualURL, err := url.Parse(actual)
-	if err != nil {
-		t.Errorf(errMsg("unable to parse actual URL", err, msgAndArgs))
-		return false
-	}
-
-	equal(t, expectURL.Host, actualURL.Host, msgAndArgs...)
-	equal(t, expectURL.Scheme, actualURL.Scheme, msgAndArgs...)
-	equal(t, expectURL.Path, actualURL.Path, msgAndArgs...)
-
-	return AssertQuery(t, expectURL.Query().Encode(), actualURL.Query().Encode(), msgAndArgs...)
-}
-
-var queryMapKey = regexp.MustCompile(`(.*?)\.[0-9]+\.key`)
-
-// AssertQuery verifies the expect HTTP query string matches the actual.
-func AssertQuery(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
-	t.Helper()
-
-	expectQ, err := url.ParseQuery(expect)
-	if err != nil {
-		t.Errorf(errMsg("unable to parse expected Query", err, msgAndArgs))
-		return false
-	}
-	actualQ, err := url.ParseQuery(actual)
-	if err != nil {
-		t.Errorf(errMsg("unable to parse actual Query", err, msgAndArgs))
-		return false
-	}
-
-	// Make sure the keys are the same
-	if !equal(t, queryValueKeys(expectQ), queryValueKeys(actualQ), msgAndArgs...) {
-		return false
-	}
-
-	keys := map[string][]string{}
-	for key, v := range expectQ {
-		if queryMapKey.Match([]byte(key)) {
-			submatch := queryMapKey.FindStringSubmatch(key)
-			keys[submatch[1]] = append(keys[submatch[1]], v...)
-		}
-	}
-
-	for k, v := range keys {
-		// clear all keys that have prefix
-		for key := range expectQ {
-			if strings.HasPrefix(key, k) {
-				delete(expectQ, key)
-			}
-		}
-
-		sort.Strings(v)
-		for i, value := range v {
-			expectQ[fmt.Sprintf("%s.%d.key", k, i+1)] = []string{value}
-		}
-	}
-
-	for k, expectQVals := range expectQ {
-		sort.Strings(expectQVals)
-		actualQVals := actualQ[k]
-		sort.Strings(actualQVals)
-		if !equal(t, expectQVals, actualQVals, msgAndArgs...) {
-			return false
-		}
-	}
-
-	return true
-}
-
-// AssertJSON verifies that the expect json string matches the actual.
-func AssertJSON(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
-	t.Helper()
-
-	expectVal := map[string]interface{}{}
-	if err := json.Unmarshal([]byte(expect), &expectVal); err != nil {
-		t.Errorf(errMsg("unable to parse expected JSON", err, msgAndArgs...))
-		return false
-	}
-
-	actualVal := map[string]interface{}{}
-	if err := json.Unmarshal([]byte(actual), &actualVal); err != nil {
-		t.Errorf(errMsg("unable to parse actual JSON", err, msgAndArgs...))
-		return false
-	}
-
-	return equal(t, expectVal, actualVal, msgAndArgs...)
-}
-
-// AssertXML verifies that the expect xml string matches the actual.
-func AssertXML(t *testing.T, expect, actual string, container interface{}, msgAndArgs ...interface{}) bool {
-	expectVal := container
-	if err := xml.Unmarshal([]byte(expect), &expectVal); err != nil {
-		t.Errorf(errMsg("unable to parse expected XML", err, msgAndArgs...))
-	}
-
-	actualVal := container
-	if err := xml.Unmarshal([]byte(actual), &actualVal); err != nil {
-		t.Errorf(errMsg("unable to parse actual XML", err, msgAndArgs...))
-	}
-	return equal(t, expectVal, actualVal, msgAndArgs...)
-}
-
-// DidPanic returns if the function paniced and returns true if the function paniced.
-func DidPanic(fn func()) (bool, interface{}) {
-	var paniced bool
-	var msg interface{}
-	func() {
-		defer func() {
-			if msg = recover(); msg != nil {
-				paniced = true
-			}
-		}()
-		fn()
-	}()
-
-	return paniced, msg
-}
-
-// objectsAreEqual determines if two objects are considered equal.
-//
-// This function does no assertion of any kind.
-//
-// Based on github.com/stretchr/testify/assert.ObjectsAreEqual
-// Copied locally to prevent non-test build dependencies on testify
-func objectsAreEqual(expected, actual interface{}) bool {
-	if expected == nil || actual == nil {
-		return expected == actual
-	}
-
-	return reflect.DeepEqual(expected, actual)
-}
-
-// Equal asserts that two objects are equal.
-//
-//	assert.Equal(t, 123, 123, "123 and 123 should be equal")
-//
-// Returns whether the assertion was successful (true) or not (false).
-//
-// Based on github.com/stretchr/testify/assert.Equal
-// Copied locally to prevent non-test build dependencies on testify
-func equal(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) bool {
-	t.Helper()
-
-	if !objectsAreEqual(expected, actual) {
-		t.Errorf("%s\n%s", messageFromMsgAndArgs(msgAndArgs),
-			SprintExpectActual(expected, actual))
-		return false
-	}
-
-	return true
-}
-
-func errMsg(baseMsg string, err error, msgAndArgs ...interface{}) string {
-	message := messageFromMsgAndArgs(msgAndArgs)
-	if message != "" {
-		message += ", "
-	}
-	return fmt.Sprintf("%s%s, %v", message, baseMsg, err)
-}
-
-// Based on github.com/stretchr/testify/assert.messageFromMsgAndArgs
-// Copied locally to prevent non-test build dependencies on testify
-func messageFromMsgAndArgs(msgAndArgs []interface{}) string {
-	if len(msgAndArgs) == 0 || msgAndArgs == nil {
-		return ""
-	}
-	if len(msgAndArgs) == 1 {
-		return msgAndArgs[0].(string)
-	}
-	if len(msgAndArgs) > 1 {
-		return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...)
-	}
-	return ""
-}
-
-func queryValueKeys(v url.Values) []string {
-	keys := make([]string, 0, len(v))
-	for k := range v {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	return keys
-}
-
-// SprintExpectActual returns a string for test failure cases when the actual
-// value is not the same as the expected.
-func SprintExpectActual(expect, actual interface{}) string {
-	return fmt.Sprintf("expect: %+v\nactual: %+v\n", expect, actual)
-}

aws/internal/awstesting/assert_test.go

@@ -1,89 +0,0 @@
-package awstesting_test
-
-import (
-	"encoding/xml"
-	"testing"
-
-	"github.com/versity/versitygw/aws/internal/awstesting"
-)
-
-func TestAssertJSON(t *testing.T) {
-	cases := []struct {
-		e, a    string
-		asserts bool
-	}{
-		{
-			e:       `{"RecursiveStruct":{"RecursiveMap":{"foo":{"NoRecurse":"foo"},"bar":{"NoRecurse":"bar"}}}}`,
-			a:       `{"RecursiveStruct":{"RecursiveMap":{"bar":{"NoRecurse":"bar"},"foo":{"NoRecurse":"foo"}}}}`,
-			asserts: true,
-		},
-	}
-
-	for i, c := range cases {
-		mockT := &testing.T{}
-		if awstesting.AssertJSON(mockT, c.e, c.a) != c.asserts {
-			t.Error("Assert JSON result was not expected.", i)
-		}
-	}
-}
-
-func TestAssertXML(t *testing.T) {
-	cases := []struct {
-		e, a      string
-		asserts   bool
-		container struct {
-			XMLName         xml.Name `xml:"OperationRequest"`
-			NS              string   `xml:"xmlns,attr"`
-			RecursiveStruct struct {
-				RecursiveMap struct {
-					Entries []struct {
-						XMLName xml.Name `xml:"entries"`
-						Key     string   `xml:"key"`
-						Value   struct {
-							XMLName   xml.Name `xml:"value"`
-							NoRecurse string
-						}
-					}
-				}
-			}
-		}
-	}{
-		{
-			e:       `<OperationRequest xmlns="https://foo/"><RecursiveStruct xmlns="https://foo/"><RecursiveMap xmlns="https://foo/"><entry xmlns="https://foo/"><key xmlns="https://foo/">foo</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">foo</NoRecurse></value></entry><entry xmlns="https://foo/"><key xmlns="https://foo/">bar</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">bar</NoRecurse></value></entry></RecursiveMap></RecursiveStruct></OperationRequest>`,
-			a:       `<OperationRequest xmlns="https://foo/"><RecursiveStruct xmlns="https://foo/"><RecursiveMap xmlns="https://foo/"><entry xmlns="https://foo/"><key xmlns="https://foo/">bar</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">bar</NoRecurse></value></entry><entry xmlns="https://foo/"><key xmlns="https://foo/">foo</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">foo</NoRecurse></value></entry></RecursiveMap></RecursiveStruct></OperationRequest>`,
-			asserts: true,
-		},
-	}
-
-	for i, c := range cases {
-		//		mockT := &testing.T{}
-		if awstesting.AssertXML(t, c.e, c.a, c.container) != c.asserts {
-			t.Error("Assert XML result was not expected.", i)
-		}
-	}
-}
-
-func TestAssertQuery(t *testing.T) {
-	cases := []struct {
-		e, a    string
-		asserts bool
-	}{
-		{
-			e:       `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
-			a:       `Action=OperationName&Version=2014-01-01&Foo=val2&Bar=val3`,
-			asserts: false,
-		},
-		{
-			e:       `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
-			a:       `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
-			asserts: true,
-		},
-	}
-
-	for i, c := range cases {
-		mockT := &testing.T{}
-		if awstesting.AssertQuery(mockT, c.e, c.a) != c.asserts {
-			t.Error("Assert Query result was not expected.", i)
-		}
-	}
-}

aws/internal/awstesting/certificate_utils.go

@@ -1,291 +0,0 @@
-package awstesting
-
-import (
-	"bytes"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"fmt"
-	"io/ioutil"
-	"math/big"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"time"
-)
-
-var (
-	// TLSBundleCA is the CA PEM
-	TLSBundleCA []byte
-
-	// TLSBundleCert is the Server PEM
-	TLSBundleCert []byte
-
-	// TLSBundleKey is the Server private key PEM
-	TLSBundleKey []byte
-
-	// ClientTLSCert is the Client PEM
-	ClientTLSCert []byte
-
-	// ClientTLSKey is the Client private key PEM
-	ClientTLSKey []byte
-)
-
-func init() {
-	caPEM, _, caCert, caPrivKey, err := generateRootCA()
-	if err != nil {
-		panic("failed to generate testing root CA, " + err.Error())
-	}
-	TLSBundleCA = caPEM
-
-	serverCertPEM, serverCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey)
-	if err != nil {
-		panic("failed to generate testing server cert, " + err.Error())
-	}
-	TLSBundleCert = serverCertPEM
-	TLSBundleKey = serverCertPrivKeyPEM
-
-	clientCertPEM, clientCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey)
-	if err != nil {
-		panic("failed to generate testing client cert, " + err.Error())
-	}
-	ClientTLSCert = clientCertPEM
-	ClientTLSKey = clientCertPrivKeyPEM
-}
-
-func generateRootCA() (
-	caPEM, caPrivKeyPEM []byte, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, err error,
-) {
-	caCert = &x509.Certificate{
-		SerialNumber: big.NewInt(42),
-		Subject: pkix.Name{
-			Country:      []string{"US"},
-			Organization: []string{"AWS SDK for Go Test Certificate"},
-			CommonName:   "Test Root CA",
-		},
-		NotBefore: time.Now().Add(-time.Minute),
-		NotAfter:  time.Now().AddDate(1, 0, 0),
-		KeyUsage:  x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage: []x509.ExtKeyUsage{
-			x509.ExtKeyUsageClientAuth,
-			x509.ExtKeyUsageServerAuth,
-		},
-		BasicConstraintsValid: true,
-		IsCA:                  true,
-	}
-
-	// Create CA private and public key
-	caPrivKey, err = rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, nil, nil, fmt.Errorf("failed generate CA RSA key, %w", err)
-	}
-
-	// Create CA certificate
-	caBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, &caPrivKey.PublicKey, caPrivKey)
-	if err != nil {
-		return nil, nil, nil, nil, fmt.Errorf("failed generate CA certificate, %w", err)
-	}
-
-	// PEM encode CA certificate and private key
-	var caPEMBuf bytes.Buffer
-	pem.Encode(&caPEMBuf, &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: caBytes,
-	})
-
-	var caPrivKeyPEMBuf bytes.Buffer
-	pem.Encode(&caPrivKeyPEMBuf, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey),
-	})
-
-	return caPEMBuf.Bytes(), caPrivKeyPEMBuf.Bytes(), caCert, caPrivKey, nil
-}
-
-func generateLocalCert(parentCert *x509.Certificate, parentPrivKey *rsa.PrivateKey) (
-	certPEM, certPrivKeyPEM []byte, err error,
-) {
-	cert := &x509.Certificate{
-		SerialNumber: big.NewInt(42),
-		Subject: pkix.Name{
-			Country:      []string{"US"},
-			Organization: []string{"AWS SDK for Go Test Certificate"},
-			CommonName:   "Test Root CA",
-		},
-		IPAddresses: []net.IP{
-			net.IPv4(127, 0, 0, 1),
-			net.IPv6loopback,
-		},
-		NotBefore: time.Now().Add(-time.Minute),
-		NotAfter:  time.Now().AddDate(1, 0, 0),
-		ExtKeyUsage: []x509.ExtKeyUsage{
-			x509.ExtKeyUsageClientAuth,
-			x509.ExtKeyUsageServerAuth,
-		},
-		KeyUsage: x509.KeyUsageDigitalSignature,
-	}
-
-	// Create server private and public key
-	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to generate server RSA private key, %w", err)
-	}
-
-	// Create server certificate
-	certBytes, err := x509.CreateCertificate(rand.Reader, cert, parentCert, &certPrivKey.PublicKey, parentPrivKey)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to generate server certificate, %w", err)
-	}
-
-	// PEM encode certificate and private key
-	var certPEMBuf bytes.Buffer
-	pem.Encode(&certPEMBuf, &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: certBytes,
-	})
-
-	var certPrivKeyPEMBuf bytes.Buffer
-	pem.Encode(&certPrivKeyPEMBuf, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
-	})
-
-	return certPEMBuf.Bytes(), certPrivKeyPEMBuf.Bytes(), nil
-}
-
-// NewTLSClientCertServer creates a new HTTP test server initialize to require
-// HTTP clients authenticate with TLS client certificates.
-func NewTLSClientCertServer(handler http.Handler) (*httptest.Server, error) {
-	server := httptest.NewUnstartedServer(handler)
-
-	if server.TLS == nil {
-		server.TLS = &tls.Config{}
-	}
-	server.TLS.ClientAuth = tls.RequireAndVerifyClientCert
-
-	if server.TLS.ClientCAs == nil {
-		server.TLS.ClientCAs = x509.NewCertPool()
-	}
-	certPem := append(ClientTLSCert, ClientTLSKey...)
-	if ok := server.TLS.ClientCAs.AppendCertsFromPEM(certPem); !ok {
-		return nil, fmt.Errorf("failed to append client certs")
-	}
-
-	return server, nil
-}
-
-// CreateClientTLSCertFiles returns a set of temporary files for the client
-// certificate and key files.
-func CreateClientTLSCertFiles() (cert, key string, err error) {
-	cert, err = createTmpFile(ClientTLSCert)
-	if err != nil {
-		return "", "", err
-	}
-
-	key, err = createTmpFile(ClientTLSKey)
-	if err != nil {
-		return "", "", err
-	}
-
-	return cert, key, nil
-}
-
-func availableLocalAddr(ip string) (v string, err error) {
-	l, err := net.Listen("tcp", ip+":0")
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		closeErr := l.Close()
-		if err == nil {
-			err = closeErr
-		} else if closeErr != nil {
-			err = fmt.Errorf("ip listener close error: %v, original error: %w", closeErr, err)
-		}
-	}()
-
-	return l.Addr().String(), nil
-}
-
-// CreateTLSServer will create the TLS server on an open port using the
-// certificate and key. The address will be returned that the server is running on.
-func CreateTLSServer(cert, key string, mux *http.ServeMux) (string, error) {
-	addr, err := availableLocalAddr("127.0.0.1")
-	if err != nil {
-		return "", err
-	}
-
-	if mux == nil {
-		mux = http.NewServeMux()
-		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {})
-	}
-
-	go func() {
-		if err := http.ListenAndServeTLS(addr, cert, key, mux); err != nil {
-			panic(err)
-		}
-	}()
-
-	for i := 0; i < 60; i++ {
-		if _, err := http.Get("https://" + addr); err != nil && !strings.Contains(err.Error(), "connection refused") {
-			break
-		}
-
-		time.Sleep(1 * time.Second)
-	}
-
-	return "https://" + addr, nil
-}
-
-// CreateTLSBundleFiles returns the temporary filenames for the certificate
-// key, and CA PEM content. These files should be deleted when no longer
-// needed. CleanupTLSBundleFiles can be used for this cleanup.
-func CreateTLSBundleFiles() (cert, key, ca string, err error) {
-	cert, err = createTmpFile(TLSBundleCert)
-	if err != nil {
-		return "", "", "", err
-	}
-
-	key, err = createTmpFile(TLSBundleKey)
-	if err != nil {
-		return "", "", "", err
-	}
-
-	ca, err = createTmpFile(TLSBundleCA)
-	if err != nil {
-		return "", "", "", err
-	}
-
-	return cert, key, ca, nil
-}
-
-// CleanupTLSBundleFiles takes variadic list of files to be deleted.
-func CleanupTLSBundleFiles(files ...string) error {
-	for _, file := range files {
-		if err := os.Remove(file); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func createTmpFile(b []byte) (string, error) {
-	bundleFile, err := ioutil.TempFile(os.TempDir(), "aws-sdk-go-session-test")
-	if err != nil {
-		return "", err
-	}
-
-	_, err = bundleFile.Write(b)
-	if err != nil {
-		return "", err
-	}
-
-	defer bundleFile.Close()
-	return bundleFile.Name(), nil
-}

aws/internal/awstesting/discard.go

@@ -1,11 +0,0 @@
-package awstesting
-
-// DiscardAt is an io.WriteAt that discards
-// the requested bytes to be written
-type DiscardAt struct{}
-
-// WriteAt discards the given []byte slice and returns len(p) bytes
-// as having been written at the given offset. It will never return an error.
-func (d DiscardAt) WriteAt(p []byte, off int64) (n int, err error) {
-	return len(p), nil
-}

aws/internal/awstesting/endless_reader.go

@@ -1,12 +0,0 @@
-package awstesting
-
-// EndlessReader is an io.Reader that will always return
-// that bytes have been read.
-type EndlessReader struct{}
-
-// Read will report that it has read len(p) bytes in p.
-// The content in the []byte will be unmodified.
-// This will never return an error.
-func (e EndlessReader) Read(p []byte) (int, error) {
-	return len(p), nil
-}

aws/internal/awstesting/sandbox/Dockerfile.golang-tip

@@ -1,43 +0,0 @@
-# Based on docker-library's golang 1.6 alpine and wheezy docker files.
-# https://github.com/docker-library/golang/blob/master/1.6/alpine/Dockerfile
-# https://github.com/docker-library/golang/blob/master/1.6/wheezy/Dockerfile
-FROM buildpack-deps:buster-scm
-
-ENV GOLANG_SRC_REPO_URL https://github.com/golang/go
-
-# as of 1.20 Go 1.17 is required to bootstrap
-# see https://github.com/golang/go/issues/44505
-ENV GOLANG_BOOTSTRAP_URL https://go.dev/dl/go1.17.13.linux-amd64.tar.gz
-ENV GOLANG_BOOTSTRAP_SHA256 4cdd2bc664724dc7db94ad51b503512c5ae7220951cac568120f64f8e94399fc
-ENV GOLANG_BOOTSTRAP_PATH /usr/local/bootstrap
-
-# gcc for cgo
-RUN apt-get update && apt-get install -y --no-install-recommends \
-		g++ \
-		gcc \
-		libc6-dev \
-		make \
-		git \
-	&& rm -rf /var/lib/apt/lists/*
-
-# Setup the Bootstrap
-RUN mkdir -p "$GOLANG_BOOTSTRAP_PATH" \
-	&& curl -fsSL "$GOLANG_BOOTSTRAP_URL" -o golang.tar.gz \
-	&& echo "$GOLANG_BOOTSTRAP_SHA256  golang.tar.gz" | sha256sum -c - \
-	&& tar -C "$GOLANG_BOOTSTRAP_PATH" -xzf golang.tar.gz \
-	&& rm golang.tar.gz
-
-# Get and build Go tip
-RUN export GOROOT_BOOTSTRAP=$GOLANG_BOOTSTRAP_PATH/go \
-	&& git clone "$GOLANG_SRC_REPO_URL" /usr/local/go \
-	&& cd /usr/local/go/src \
-	&& ./make.bash \
-	&& rm -rf "$GOLANG_BOOTSTRAP_PATH" /usr/local/go/pkg/bootstrap
-
-# Build Go workspace and environment
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
-RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" \
-	&& chmod -R 777 "$GOPATH"
-
-WORKDIR $GOPATH

aws/internal/awstesting/sandbox/Dockerfile.test.gotip

@@ -1,16 +0,0 @@
-FROM aws-golang:tip
-
-ENV GOPROXY=direct
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        software-properties-common \
-    && wget -O- https://apt.corretto.aws/corretto.key | apt-key add - \
-    && add-apt-repository 'deb https://apt.corretto.aws stable main' \
-    && apt-get update && apt-get install -y --no-install-recommends \
-        vim \
-        java-17-amazon-corretto-jdk \
-    && rm -rf /var/list/apt/lists/*
-
-ADD . /go/src/github.com/aws/aws-sdk-go-v2
-WORKDIR /go/src/github.com/aws/aws-sdk-go-v2
-CMD ["make", "unit"]

aws/internal/awstesting/sandbox/Dockerfile.test.goversion

@@ -1,18 +0,0 @@
-ARG GO_VERSION
-FROM golang:${GO_VERSION}
-
-ENV GOPROXY=direct
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        software-properties-common \
-    && wget -O- https://apt.corretto.aws/corretto.key | apt-key add - \
-    && add-apt-repository 'deb https://apt.corretto.aws stable main' \
-    && apt-get update && apt-get install -y --no-install-recommends \
-        vim \
-        java-17-amazon-corretto-jdk \
-    && rm -rf /var/list/apt/lists/*
-
-ADD . /go/src/github.com/aws/aws-sdk-go-v2
-
-WORKDIR /go/src/github.com/aws/aws-sdk-go-v2
-CMD ["make", "unit"]

aws/internal/awstesting/util.go

@@ -1,201 +0,0 @@
-package awstesting
-
-import (
-	"context"
-	"io"
-	"net/http"
-	"os"
-	"runtime"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-)
-
-// ZeroReader is a io.Reader which will always write zeros to the byte slice provided.
-type ZeroReader struct{}
-
-// Read fills the provided byte slice with zeros returning the number of bytes written.
-func (r *ZeroReader) Read(b []byte) (int, error) {
-	for i := 0; i < len(b); i++ {
-		b[i] = 0
-	}
-	return len(b), nil
-}
-
-// ReadCloser is a io.ReadCloser for unit testing.
-// Designed to test for leaks and whether a handle has
-// been closed
-type ReadCloser struct {
-	Size     int
-	Closed   bool
-	set      bool
-	FillData func(bool, []byte, int, int)
-}
-
-// Read will call FillData and fill it with whatever data needed.
-// Decrements the size until zero, then return io.EOF.
-func (r *ReadCloser) Read(b []byte) (int, error) {
-	if r.Closed {
-		return 0, io.EOF
-	}
-
-	delta := len(b)
-	if delta > r.Size {
-		delta = r.Size
-	}
-	r.Size -= delta
-
-	for i := 0; i < delta; i++ {
-		b[i] = 'a'
-	}
-
-	if r.FillData != nil {
-		r.FillData(r.set, b, r.Size, delta)
-	}
-	r.set = true
-
-	if r.Size > 0 {
-		return delta, nil
-	}
-	return delta, io.EOF
-}
-
-// Close sets Closed to true and returns no error
-func (r *ReadCloser) Close() error {
-	r.Closed = true
-	return nil
-}
-
-// A FakeContext provides a simple stub implementation of a Context
-type FakeContext struct {
-	Error  error
-	DoneCh chan struct{}
-}
-
-// Deadline always will return not set
-func (c *FakeContext) Deadline() (deadline time.Time, ok bool) {
-	return time.Time{}, false
-}
-
-// Done returns a read channel for listening to the Done event
-func (c *FakeContext) Done() <-chan struct{} {
-	return c.DoneCh
-}
-
-// Err returns the error, is nil if not set.
-func (c *FakeContext) Err() error {
-	return c.Error
-}
-
-// Value ignores the Value and always returns nil
-func (c *FakeContext) Value(key interface{}) interface{} {
-	return nil
-}
-
-// StashEnv stashes the current environment variables except variables listed in envToKeepx
-// Returns an function to pop out old environment
-func StashEnv(envToKeep ...string) []string {
-	if runtime.GOOS == "windows" {
-		envToKeep = append(envToKeep, "ComSpec")
-		envToKeep = append(envToKeep, "SYSTEM32")
-		envToKeep = append(envToKeep, "SYSTEMROOT")
-	}
-	envToKeep = append(envToKeep, "PATH", "HOME", "USERPROFILE")
-	extraEnv := getEnvs(envToKeep)
-	originalEnv := os.Environ()
-	os.Clearenv() // clear env
-	for key, val := range extraEnv {
-		os.Setenv(key, val)
-	}
-	return originalEnv
-}
-
-// PopEnv takes the list of the environment values and injects them into the
-// process's environment variable data. Clears any existing environment values
-// that may already exist.
-func PopEnv(env []string) {
-	os.Clearenv()
-
-	for _, e := range env {
-		p := strings.SplitN(e, "=", 2)
-		k, v := p[0], ""
-		if len(p) > 1 {
-			v = p[1]
-		}
-		os.Setenv(k, v)
-	}
-}
-
-// MockCredentialsProvider is a type that can be used to mock out credentials
-// providers
-type MockCredentialsProvider struct {
-	RetrieveFn   func(ctx context.Context) (aws.Credentials, error)
-	InvalidateFn func()
-}
-
-// Retrieve calls the RetrieveFn
-func (p MockCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
-	return p.RetrieveFn(ctx)
-}
-
-// Invalidate calls the InvalidateFn
-func (p MockCredentialsProvider) Invalidate() {
-	p.InvalidateFn()
-}
-
-func getEnvs(envs []string) map[string]string {
-	extraEnvs := make(map[string]string)
-	for _, env := range envs {
-		if val, ok := os.LookupEnv(env); ok && len(val) > 0 {
-			extraEnvs[env] = val
-		}
-	}
-	return extraEnvs
-}
-
-const (
-	signaturePreambleSigV4  = "AWS4-HMAC-SHA256"
-	signaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256"
-)
-
-// SigV4Signature represents a parsed sigv4 or sigv4a signature.
-type SigV4Signature struct {
-	Preamble      string   // e.g. AWS4-HMAC-SHA256, AWS4-ECDSA-P256-SHA256
-	SigningName   string   // generally the service name e.g. "s3"
-	SigningRegion string   // for sigv4a this is the region-set header as-is
-	SignedHeaders []string // list of signed headers
-	Signature     string   // calculated signature
-}
-
-// ParseSigV4Signature deconstructs a sigv4 or sigv4a signature from a set of
-// request headers.
-func ParseSigV4Signature(header http.Header) *SigV4Signature {
-	auth := header.Get("Authorization")
-
-	preamble, after, _ := strings.Cut(auth, " ")
-	credential, after, _ := strings.Cut(after, ", ")
-	signedHeaders, signature, _ := strings.Cut(after, ", ")
-
-	credentialParts := strings.Split(credential, "/")
-
-	// sigv4  : AccessKeyID/DateString/SigningRegion/SigningName/SignatureID
-	// sigv4a : AccessKeyID/DateString/SigningName/SignatureID, region set on
-	//          header
-	var signingName, signingRegion string
-	if preamble == signaturePreambleSigV4 {
-		signingName = credentialParts[3]
-		signingRegion = credentialParts[2]
-	} else if preamble == signaturePreambleSigV4A {
-		signingName = credentialParts[2]
-		signingRegion = header.Get("X-Amz-Region-Set")
-	}
-
-	return &SigV4Signature{
-		Preamble:      preamble,
-		SigningName:   signingName,
-		SigningRegion: signingRegion,
-		SignedHeaders: strings.Split(signedHeaders, ";"),
-		Signature:     signature,
-	}
-}

aws/internal/awstesting/util_test.go

@@ -1,75 +0,0 @@
-package awstesting_test
-
-import (
-	"io"
-	"testing"
-
-	"github.com/versity/versitygw/aws/internal/awstesting"
-)
-
-func TestReadCloserClose(t *testing.T) {
-	rc := awstesting.ReadCloser{Size: 1}
-	err := rc.Close()
-
-	if err != nil {
-		t.Errorf("expect nil, got %v", err)
-	}
-	if !rc.Closed {
-		t.Errorf("expect closed, was not")
-	}
-	if e, a := rc.Size, 1; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-}
-
-func TestReadCloserRead(t *testing.T) {
-	rc := awstesting.ReadCloser{Size: 5}
-	b := make([]byte, 2)
-
-	n, err := rc.Read(b)
-
-	if err != nil {
-		t.Errorf("expect nil, got %v", err)
-	}
-	if e, a := n, 2; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if rc.Closed {
-		t.Errorf("expect not to be closed")
-	}
-	if e, a := rc.Size, 3; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-
-	err = rc.Close()
-	if err != nil {
-		t.Errorf("expect nil, got %v", err)
-	}
-	n, err = rc.Read(b)
-	if e, a := err, io.EOF; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := n, 0; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-}
-
-func TestReadCloserReadAll(t *testing.T) {
-	rc := awstesting.ReadCloser{Size: 5}
-	b := make([]byte, 5)
-
-	n, err := rc.Read(b)
-
-	if e, a := err, io.EOF; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := n, 5; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if rc.Closed {
-		t.Errorf("expect not to be closed")
-	}
-	if e, a := rc.Size, 0; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-}

aws/internal/sdk/interfaces.go

@@ -1,9 +0,0 @@
-package sdk
-
-// Invalidator provides access to a type's invalidate method to make it
-// invalidate it cache.
-//
-// e.g aws.SafeCredentialsProvider's Invalidate method.
-type Invalidator interface {
-	Invalidate()
-}

aws/internal/sdk/time.go

@@ -1,74 +0,0 @@
-package sdk
-
-import (
-	"context"
-	"time"
-)
-
-func init() {
-	NowTime = time.Now
-	Sleep = time.Sleep
-	SleepWithContext = sleepWithContext
-}
-
-// NowTime is a value for getting the current time. This value can be overridden
-// for testing mocking out current time.
-var NowTime func() time.Time
-
-// Sleep is a value for sleeping for a duration. This value can be overridden
-// for testing and mocking out sleep duration.
-var Sleep func(time.Duration)
-
-// SleepWithContext will wait for the timer duration to expire, or the context
-// is canceled. Which ever happens first. If the context is canceled the Context's
-// error will be returned.
-//
-// This value can be overridden for testing and mocking out sleep duration.
-var SleepWithContext func(context.Context, time.Duration) error
-
-// sleepWithContext will wait for the timer duration to expire, or the context
-// is canceled. Which ever happens first. If the context is canceled the
-// Context's error will be returned.
-func sleepWithContext(ctx context.Context, dur time.Duration) error {
-	t := time.NewTimer(dur)
-	defer t.Stop()
-
-	select {
-	case <-t.C:
-		break
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-
-	return nil
-}
-
-// noOpSleepWithContext does nothing, returns immediately.
-func noOpSleepWithContext(context.Context, time.Duration) error {
-	return nil
-}
-
-func noOpSleep(time.Duration) {}
-
-// TestingUseNopSleep is a utility for disabling sleep across the SDK for
-// testing.
-func TestingUseNopSleep() func() {
-	SleepWithContext = noOpSleepWithContext
-	Sleep = noOpSleep
-
-	return func() {
-		SleepWithContext = sleepWithContext
-		Sleep = time.Sleep
-	}
-}
-
-// TestingUseReferenceTime is a utility for swapping the time function across the SDK to return a specific reference time
-// for testing purposes.
-func TestingUseReferenceTime(referenceTime time.Time) func() {
-	NowTime = func() time.Time {
-		return referenceTime
-	}
-	return func() {
-		NowTime = time.Now
-	}
-}

aws/internal/sdk/time_test.go

@@ -1,32 +0,0 @@
-package sdk
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestSleepWithContext(t *testing.T) {
-	ctx, cancelFn := context.WithCancel(context.Background())
-	defer cancelFn()
-
-	err := sleepWithContext(ctx, 1*time.Millisecond)
-	if err != nil {
-		t.Errorf("expect context to not be canceled, got %v", err)
-	}
-}
-
-func TestSleepWithContext_Canceled(t *testing.T) {
-	ctx, cancelFn := context.WithCancel(context.Background())
-	cancelFn()
-
-	err := sleepWithContext(ctx, 10*time.Second)
-	if err == nil {
-		t.Fatalf("expect error, did not get one")
-	}
-
-	if e, a := "context canceled", err.Error(); !strings.Contains(a, e) {
-		t.Errorf("expect %v error, got %v", e, a)
-	}
-}

aws/internal/strings/strings.go

@@ -1,11 +0,0 @@
-package strings
-
-import (
-	"strings"
-)
-
-// HasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
-// under Unicode case-folding.
-func HasPrefixFold(s, prefix string) bool {
-	return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
-}

aws/internal/strings/strings_test.go

@@ -1,81 +0,0 @@
-package strings
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestHasPrefixFold(t *testing.T) {
-	type args struct {
-		s      string
-		prefix string
-	}
-	tests := map[string]struct {
-		args args
-		want bool
-	}{
-		"empty strings and prefix": {
-			args: args{
-				s:      "",
-				prefix: "",
-			},
-			want: true,
-		},
-		"strings starts with prefix": {
-			args: args{
-				s:      "some string",
-				prefix: "some",
-			},
-			want: true,
-		},
-		"prefix longer then string": {
-			args: args{
-				s:      "some",
-				prefix: "some string",
-			},
-		},
-		"equal length string and prefix": {
-			args: args{
-				s:      "short string",
-				prefix: "short string",
-			},
-			want: true,
-		},
-		"different cases": {
-			args: args{
-				s:      "ShOrT StRING",
-				prefix: "short",
-			},
-			want: true,
-		},
-		"empty prefix not empty string": {
-			args: args{
-				s:      "ShOrT StRING",
-				prefix: "",
-			},
-			want: true,
-		},
-		"mixed-case prefixes": {
-			args: args{
-				s:      "SoMe String",
-				prefix: "sOme",
-			},
-			want: true,
-		},
-	}
-	for name, tt := range tests {
-		t.Run(name, func(t *testing.T) {
-			if got := HasPrefixFold(tt.args.s, tt.args.prefix); got != tt.want {
-				t.Errorf("HasPrefixFold() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func BenchmarkHasPrefixFold(b *testing.B) {
-	HasPrefixFold("SoME string", "sOmE")
-}
-
-func BenchmarkHasPrefix(b *testing.B) {
-	strings.HasPrefix(strings.ToLower("SoME string"), strings.ToLower("sOmE"))
-}

aws/signer/internal/v4/header_rules.go

@@ -1,7 +1,7 @@
 package v4
 
 import (
-	sdkstrings "github.com/versity/versitygw/aws/internal/strings"
+	"strings"
 )
 
 // Rules houses a set of Rule needed for validation of a
@@ -61,7 +61,7 @@ type Patterns []string
 // been found
 func (p Patterns) IsValid(value string) bool {
 	for _, pattern := range p {
-		if sdkstrings.HasPrefixFold(value, pattern) {
+		if hasPrefixFold(value, pattern) {
 			return true
 		}
 	}
@@ -80,3 +80,9 @@ func (r InclusiveRules) IsValid(value string) bool {
 	}
 	return true
 }
+
+// hasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
+// under Unicode case-folding.
+func hasPrefixFold(s, prefix string) bool {
+	return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
+}

aws/signer/v4/middleware.go

@@ -1,443 +0,0 @@
-package v4
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
-	"github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics"
-	"github.com/aws/smithy-go/middleware"
-	smithyhttp "github.com/aws/smithy-go/transport/http"
-	internalauth "github.com/versity/versitygw/aws/internal/auth"
-	"github.com/versity/versitygw/aws/internal/sdk"
-	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
-)
-
-const computePayloadHashMiddlewareID = "ComputePayloadHash"
-
-// HashComputationError indicates an error occurred while computing the signing hash
-type HashComputationError struct {
-	Err error
-}
-
-// Error is the error message
-func (e *HashComputationError) Error() string {
-	return fmt.Sprintf("failed to compute payload hash: %v", e.Err)
-}
-
-// Unwrap returns the underlying error if one is set
-func (e *HashComputationError) Unwrap() error {
-	return e.Err
-}
-
-// SigningError indicates an error condition occurred while performing SigV4 signing
-type SigningError struct {
-	Err error
-}
-
-func (e *SigningError) Error() string {
-	return fmt.Sprintf("failed to sign request: %v", e.Err)
-}
-
-// Unwrap returns the underlying error cause
-func (e *SigningError) Unwrap() error {
-	return e.Err
-}
-
-// UseDynamicPayloadSigningMiddleware swaps the compute payload sha256 middleware with a resolver middleware that
-// switches between unsigned and signed payload based on TLS state for request.
-// This middleware should not be used for AWS APIs that do not support unsigned payload signing auth.
-// By default, SDK uses this middleware for known AWS APIs that support such TLS based auth selection .
-//
-// Usage example -
-// S3 PutObject API allows unsigned payload signing auth usage when TLS is enabled, and uses this middleware to
-// dynamically switch between unsigned and signed payload based on TLS state for request.
-func UseDynamicPayloadSigningMiddleware(stack *middleware.Stack) error {
-	_, err := stack.Finalize.Swap(computePayloadHashMiddlewareID, &dynamicPayloadSigningMiddleware{})
-	return err
-}
-
-// dynamicPayloadSigningMiddleware dynamically resolves the middleware that computes and set payload sha256 middleware.
-type dynamicPayloadSigningMiddleware struct {
-}
-
-// ID returns the resolver identifier
-func (m *dynamicPayloadSigningMiddleware) ID() string {
-	return computePayloadHashMiddlewareID
-}
-
-// HandleFinalize delegates SHA256 computation according to whether the request
-// is TLS-enabled.
-func (m *dynamicPayloadSigningMiddleware) HandleFinalize(
-	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
-) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	req, ok := in.Request.(*smithyhttp.Request)
-	if !ok {
-		return out, metadata, fmt.Errorf("unknown transport type %T", in.Request)
-	}
-
-	if req.IsHTTPS() {
-		return (&UnsignedPayload{}).HandleFinalize(ctx, in, next)
-	}
-	return (&ComputePayloadSHA256{}).HandleFinalize(ctx, in, next)
-}
-
-// UnsignedPayload sets the SigV4 request payload hash to unsigned.
-//
-// Will not set the Unsigned Payload magic SHA value, if a SHA has already been
-// stored in the context. (e.g. application pre-computed SHA256 before making
-// API call).
-//
-// This middleware does not check the X-Amz-Content-Sha256 header, if that
-// header is serialized a middleware must translate it into the context.
-type UnsignedPayload struct{}
-
-// AddUnsignedPayloadMiddleware adds unsignedPayload to the operation
-// middleware stack
-func AddUnsignedPayloadMiddleware(stack *middleware.Stack) error {
-	return stack.Finalize.Insert(&UnsignedPayload{}, "ResolveEndpointV2", middleware.After)
-}
-
-// ID returns the unsignedPayload identifier
-func (m *UnsignedPayload) ID() string {
-	return computePayloadHashMiddlewareID
-}
-
-// HandleFinalize sets the payload hash magic value to the unsigned sentinel.
-func (m *UnsignedPayload) HandleFinalize(
-	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
-) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	if GetPayloadHash(ctx) == "" {
-		ctx = SetPayloadHash(ctx, v4Internal.UnsignedPayload)
-	}
-	return next.HandleFinalize(ctx, in)
-}
-
-// ComputePayloadSHA256 computes SHA256 payload hash to sign.
-//
-// Will not set the Unsigned Payload magic SHA value, if a SHA has already been
-// stored in the context. (e.g. application pre-computed SHA256 before making
-// API call).
-//
-// This middleware does not check the X-Amz-Content-Sha256 header, if that
-// header is serialized a middleware must translate it into the context.
-type ComputePayloadSHA256 struct{}
-
-// AddComputePayloadSHA256Middleware adds computePayloadSHA256 to the
-// operation middleware stack
-func AddComputePayloadSHA256Middleware(stack *middleware.Stack) error {
-	return stack.Finalize.Insert(&ComputePayloadSHA256{}, "ResolveEndpointV2", middleware.After)
-}
-
-// RemoveComputePayloadSHA256Middleware removes computePayloadSHA256 from the
-// operation middleware stack
-func RemoveComputePayloadSHA256Middleware(stack *middleware.Stack) error {
-	_, err := stack.Finalize.Remove(computePayloadHashMiddlewareID)
-	return err
-}
-
-// ID is the middleware name
-func (m *ComputePayloadSHA256) ID() string {
-	return computePayloadHashMiddlewareID
-}
-
-// HandleFinalize computes the payload hash for the request, storing it to the
-// context. This is a no-op if a caller has previously set that value.
-func (m *ComputePayloadSHA256) HandleFinalize(
-	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
-) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	if GetPayloadHash(ctx) != "" {
-		return next.HandleFinalize(ctx, in)
-	}
-
-	req, ok := in.Request.(*smithyhttp.Request)
-	if !ok {
-		return out, metadata, &HashComputationError{
-			Err: fmt.Errorf("unexpected request middleware type %T", in.Request),
-		}
-	}
-
-	hash := sha256.New()
-	if stream := req.GetStream(); stream != nil {
-		_, err = io.Copy(hash, stream)
-		if err != nil {
-			return out, metadata, &HashComputationError{
-				Err: fmt.Errorf("failed to compute payload hash, %w", err),
-			}
-		}
-
-		if err := req.RewindStream(); err != nil {
-			return out, metadata, &HashComputationError{
-				Err: fmt.Errorf("failed to seek body to start, %w", err),
-			}
-		}
-	}
-
-	ctx = SetPayloadHash(ctx, hex.EncodeToString(hash.Sum(nil)))
-
-	return next.HandleFinalize(ctx, in)
-}
-
-// SwapComputePayloadSHA256ForUnsignedPayloadMiddleware replaces the
-// ComputePayloadSHA256 middleware with the UnsignedPayload middleware.
-//
-// Use this to disable computing the Payload SHA256 checksum and instead use
-// UNSIGNED-PAYLOAD for the SHA256 value.
-func SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack *middleware.Stack) error {
-	_, err := stack.Finalize.Swap(computePayloadHashMiddlewareID, &UnsignedPayload{})
-	return err
-}
-
-// ContentSHA256Header sets the X-Amz-Content-Sha256 header value to
-// the Payload hash stored in the context.
-type ContentSHA256Header struct{}
-
-// AddContentSHA256HeaderMiddleware adds ContentSHA256Header to the
-// operation middleware stack
-func AddContentSHA256HeaderMiddleware(stack *middleware.Stack) error {
-	return stack.Finalize.Insert(&ContentSHA256Header{}, computePayloadHashMiddlewareID, middleware.After)
-}
-
-// RemoveContentSHA256HeaderMiddleware removes contentSHA256Header middleware
-// from the operation middleware stack
-func RemoveContentSHA256HeaderMiddleware(stack *middleware.Stack) error {
-	_, err := stack.Finalize.Remove((*ContentSHA256Header)(nil).ID())
-	return err
-}
-
-// ID returns the ContentSHA256HeaderMiddleware identifier
-func (m *ContentSHA256Header) ID() string {
-	return "SigV4ContentSHA256Header"
-}
-
-// HandleFinalize sets the X-Amz-Content-Sha256 header value to the Payload hash
-// stored in the context.
-func (m *ContentSHA256Header) HandleFinalize(
-	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
-) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	req, ok := in.Request.(*smithyhttp.Request)
-	if !ok {
-		return out, metadata, &HashComputationError{Err: fmt.Errorf("unexpected request middleware type %T", in.Request)}
-	}
-
-	req.Header.Set(v4Internal.ContentSHAKey, GetPayloadHash(ctx))
-	return next.HandleFinalize(ctx, in)
-}
-
-// SignHTTPRequestMiddlewareOptions is the configuration options for
-// [SignHTTPRequestMiddleware].
-//
-// Deprecated: [SignHTTPRequestMiddleware] is deprecated.
-type SignHTTPRequestMiddlewareOptions struct {
-	CredentialsProvider aws.CredentialsProvider
-	Signer              HTTPSigner
-	LogSigning          bool
-}
-
-// SignHTTPRequestMiddleware is a `FinalizeMiddleware` implementation for SigV4
-// HTTP Signing.
-//
-// Deprecated: AWS service clients no longer use this middleware. Signing as an
-// SDK operation is now performed through an internal per-service middleware
-// which opaquely selects and uses the signer from the resolved auth scheme.
-type SignHTTPRequestMiddleware struct {
-	credentialsProvider aws.CredentialsProvider
-	signer              HTTPSigner
-	logSigning          bool
-}
-
-// NewSignHTTPRequestMiddleware constructs a [SignHTTPRequestMiddleware] using
-// the given [Signer] for signing requests.
-//
-// Deprecated: SignHTTPRequestMiddleware is deprecated.
-func NewSignHTTPRequestMiddleware(options SignHTTPRequestMiddlewareOptions) *SignHTTPRequestMiddleware {
-	return &SignHTTPRequestMiddleware{
-		credentialsProvider: options.CredentialsProvider,
-		signer:              options.Signer,
-		logSigning:          options.LogSigning,
-	}
-}
-
-// ID is the SignHTTPRequestMiddleware identifier.
-//
-// Deprecated: SignHTTPRequestMiddleware is deprecated.
-func (s *SignHTTPRequestMiddleware) ID() string {
-	return "Signing"
-}
-
-// HandleFinalize will take the provided input and sign the request using the
-// SigV4 authentication scheme.
-//
-// Deprecated: SignHTTPRequestMiddleware is deprecated.
-func (s *SignHTTPRequestMiddleware) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	if !haveCredentialProvider(s.credentialsProvider) {
-		return next.HandleFinalize(ctx, in)
-	}
-
-	req, ok := in.Request.(*smithyhttp.Request)
-	if !ok {
-		return out, metadata, &SigningError{Err: fmt.Errorf("unexpected request middleware type %T", in.Request)}
-	}
-
-	signingName, signingRegion := awsmiddleware.GetSigningName(ctx), awsmiddleware.GetSigningRegion(ctx)
-	payloadHash := GetPayloadHash(ctx)
-	if len(payloadHash) == 0 {
-		return out, metadata, &SigningError{Err: fmt.Errorf("computed payload hash missing from context")}
-	}
-
-	mctx := metrics.Context(ctx)
-
-	if mctx != nil {
-		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
-			attempt.CredentialFetchStartTime = sdk.NowTime()
-		}
-	}
-
-	credentials, err := s.credentialsProvider.Retrieve(ctx)
-
-	if mctx != nil {
-		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
-			attempt.CredentialFetchEndTime = sdk.NowTime()
-		}
-	}
-
-	if err != nil {
-		return out, metadata, &SigningError{Err: fmt.Errorf("failed to retrieve credentials: %w", err)}
-	}
-
-	signerOptions := []func(o *SignerOptions){
-		func(o *SignerOptions) {
-			o.Logger = middleware.GetLogger(ctx)
-			o.LogSigning = s.logSigning
-		},
-	}
-
-	// existing DisableURIPathEscaping is equivalent in purpose
-	// to authentication scheme property DisableDoubleEncoding
-	disableDoubleEncoding, overridden := internalauth.GetDisableDoubleEncoding(ctx)
-	if overridden {
-		signerOptions = append(signerOptions, func(o *SignerOptions) {
-			o.DisableURIPathEscaping = disableDoubleEncoding
-		})
-	}
-
-	if mctx != nil {
-		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
-			attempt.SignStartTime = sdk.NowTime()
-		}
-	}
-
-	err = s.signer.SignHTTP(ctx, credentials, req.Request, payloadHash, signingName, signingRegion, sdk.NowTime(), signerOptions...)
-
-	if mctx != nil {
-		if attempt, err := mctx.Data().LatestAttempt(); err == nil {
-			attempt.SignEndTime = sdk.NowTime()
-		}
-	}
-
-	if err != nil {
-		return out, metadata, &SigningError{Err: fmt.Errorf("failed to sign http request, %w", err)}
-	}
-
-	ctx = awsmiddleware.SetSigningCredentials(ctx, credentials)
-
-	return next.HandleFinalize(ctx, in)
-}
-
-// StreamingEventsPayload signs input event stream messages.
-type StreamingEventsPayload struct{}
-
-// AddStreamingEventsPayload adds the streamingEventsPayload middleware to the stack.
-func AddStreamingEventsPayload(stack *middleware.Stack) error {
-	return stack.Finalize.Add(&StreamingEventsPayload{}, middleware.Before)
-}
-
-// ID identifies the middleware.
-func (s *StreamingEventsPayload) ID() string {
-	return computePayloadHashMiddlewareID
-}
-
-// HandleFinalize marks the input stream to be signed with SigV4.
-func (s *StreamingEventsPayload) HandleFinalize(
-	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
-) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	contentSHA := GetPayloadHash(ctx)
-	if len(contentSHA) == 0 {
-		contentSHA = v4Internal.StreamingEventsPayload
-	}
-
-	ctx = SetPayloadHash(ctx, contentSHA)
-
-	return next.HandleFinalize(ctx, in)
-}
-
-// GetSignedRequestSignature attempts to extract the signature of the request.
-// Returning an error if the request is unsigned, or unable to extract the
-// signature.
-func GetSignedRequestSignature(r *http.Request) ([]byte, error) {
-	const authHeaderSignatureElem = "Signature="
-
-	if auth := r.Header.Get(authorizationHeader); len(auth) != 0 {
-		ps := strings.Split(auth, ", ")
-		for _, p := range ps {
-			if idx := strings.Index(p, authHeaderSignatureElem); idx >= 0 {
-				sig := p[len(authHeaderSignatureElem):]
-				if len(sig) == 0 {
-					return nil, fmt.Errorf("invalid request signature authorization header")
-				}
-				return hex.DecodeString(sig)
-			}
-		}
-	}
-
-	if sig := r.URL.Query().Get("X-Amz-Signature"); len(sig) != 0 {
-		return hex.DecodeString(sig)
-	}
-
-	return nil, fmt.Errorf("request not signed")
-}
-
-func haveCredentialProvider(p aws.CredentialsProvider) bool {
-	if p == nil {
-		return false
-	}
-
-	return !aws.IsCredentialsProvider(p, (*aws.AnonymousCredentials)(nil))
-}
-
-type payloadHashKey struct{}
-
-// GetPayloadHash retrieves the payload hash to use for signing
-//
-// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
-// to clear all stack values.
-func GetPayloadHash(ctx context.Context) (v string) {
-	v, _ = middleware.GetStackValue(ctx, payloadHashKey{}).(string)
-	return v
-}
-
-// SetPayloadHash sets the payload hash to be used for signing the request
-//
-// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
-// to clear all stack values.
-func SetPayloadHash(ctx context.Context, hash string) context.Context {
-	return middleware.WithStackValue(ctx, payloadHashKey{}, hash)
-}

aws/signer/v4/middleware_test.go

@@ -1,415 +0,0 @@
-package v4
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
-	"github.com/aws/smithy-go/logging"
-	"github.com/aws/smithy-go/middleware"
-	smithyhttp "github.com/aws/smithy-go/transport/http"
-	"github.com/google/go-cmp/cmp"
-	"github.com/versity/versitygw/aws/internal/awstesting/unit"
-)
-
-func TestComputePayloadHashMiddleware(t *testing.T) {
-	cases := []struct {
-		content      io.Reader
-		expectedHash string
-		expectedErr  interface{}
-	}{
-		0: {
-			content: func() io.Reader {
-				br := bytes.NewReader([]byte("some content"))
-				return br
-			}(),
-			expectedHash: "290f493c44f5d63d06b374d0a5abd292fae38b92cab2fae5efefe1b0e9347f56",
-		},
-		1: {
-			content: func() io.Reader {
-				return &nonSeeker{}
-			}(),
-			expectedErr: &HashComputationError{},
-		},
-		2: {
-			content: func() io.Reader {
-				return &semiSeekable{}
-			}(),
-			expectedErr: &HashComputationError{},
-		},
-	}
-
-	for i, tt := range cases {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			c := &ComputePayloadSHA256{}
-
-			next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
-				value := GetPayloadHash(ctx)
-				if len(value) == 0 {
-					t.Fatalf("expected payload hash value to be on context")
-				}
-				if e, a := tt.expectedHash, value; e != a {
-					t.Errorf("expected %v, got %v", e, a)
-				}
-
-				return out, metadata, err
-			})
-
-			stream, err := smithyhttp.NewStackRequest().(*smithyhttp.Request).SetStream(tt.content)
-			if err != nil {
-				t.Fatalf("expected no error, got %v", err)
-			}
-
-			_, _, err = c.HandleFinalize(context.Background(), middleware.FinalizeInput{Request: stream}, next)
-			if err != nil && tt.expectedErr == nil {
-				t.Errorf("expected no error, got %v", err)
-			} else if err != nil && tt.expectedErr != nil {
-				e, a := tt.expectedErr, err
-				if !errors.As(a, &e) {
-					t.Errorf("expected error type %T, got %T", e, a)
-				}
-			} else if err == nil && tt.expectedErr != nil {
-				t.Errorf("expected error, got nil")
-			}
-		})
-	}
-}
-
-type httpSignerFunc func(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
-
-func (f httpSignerFunc) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error {
-	return f(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...)
-}
-
-func TestSignHTTPRequestMiddleware(t *testing.T) {
-	cases := map[string]struct {
-		creds       aws.CredentialsProvider
-		hash        string
-		logSigning  bool
-		expectedErr interface{}
-	}{
-		"success": {
-			creds: unit.StubCredentialsProvider{},
-			hash:  "0123456789abcdef",
-		},
-		"error": {
-			creds:       unit.StubCredentialsProvider{},
-			hash:        "",
-			expectedErr: &SigningError{},
-		},
-		"anonymous creds": {
-			creds: aws.AnonymousCredentials{},
-		},
-		"nil creds": {
-			creds: nil,
-		},
-		"with log signing": {
-			creds:      unit.StubCredentialsProvider{},
-			hash:       "0123456789abcdef",
-			logSigning: true,
-		},
-	}
-
-	const (
-		signingName   = "serviceId"
-		signingRegion = "regionName"
-	)
-
-	for name, tt := range cases {
-		t.Run(name, func(t *testing.T) {
-			c := &SignHTTPRequestMiddleware{
-				credentialsProvider: tt.creds,
-				signer: httpSignerFunc(
-					func(ctx context.Context,
-						credentials aws.Credentials, r *http.Request, payloadHash string,
-						service string, region string, signingTime time.Time,
-						optFns ...func(*SignerOptions),
-					) error {
-						var options SignerOptions
-						for _, fn := range optFns {
-							fn(&options)
-						}
-						if options.Logger == nil {
-							t.Errorf("expect logger, got none")
-						}
-						if options.LogSigning {
-							options.Logger.Logf(logging.Debug, t.Name())
-						}
-
-						expectCreds, _ := unit.StubCredentialsProvider{}.Retrieve(context.Background())
-						if e, a := expectCreds, credentials; e != a {
-							t.Errorf("expected %v, got %v", e, a)
-						}
-						if e, a := tt.hash, payloadHash; e != a {
-							t.Errorf("expected %v, got %v", e, a)
-						}
-						if e, a := signingName, service; e != a {
-							t.Errorf("expected %v, got %v", e, a)
-						}
-						if e, a := signingRegion, region; e != a {
-							t.Errorf("expected %v, got %v", e, a)
-						}
-						return nil
-					}),
-				logSigning: tt.logSigning,
-			}
-
-			next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
-				return out, metadata, err
-			})
-
-			ctx := awsmiddleware.SetSigningRegion(
-				awsmiddleware.SetSigningName(context.Background(), signingName),
-				signingRegion)
-
-			var loggerBuf bytes.Buffer
-			logger := logging.NewStandardLogger(&loggerBuf)
-			ctx = middleware.SetLogger(ctx, logger)
-
-			if len(tt.hash) != 0 {
-				ctx = SetPayloadHash(ctx, tt.hash)
-			}
-
-			_, _, err := c.HandleFinalize(ctx, middleware.FinalizeInput{
-				Request: &smithyhttp.Request{Request: &http.Request{}},
-			}, next)
-			if err != nil && tt.expectedErr == nil {
-				t.Errorf("expected no error, got %v", err)
-			} else if err != nil && tt.expectedErr != nil {
-				e, a := tt.expectedErr, err
-				if !errors.As(a, &e) {
-					t.Errorf("expected error type %T, got %T", e, a)
-				}
-			} else if err == nil && tt.expectedErr != nil {
-				t.Errorf("expected error, got nil")
-			}
-
-			if tt.logSigning {
-				if e, a := t.Name(), loggerBuf.String(); !strings.Contains(a, e) {
-					t.Errorf("expect %v logged in %v", e, a)
-				}
-			} else {
-				if loggerBuf.Len() != 0 {
-					t.Errorf("expect no log, got %v", loggerBuf.String())
-				}
-			}
-		})
-	}
-}
-
-func TestSwapComputePayloadSHA256ForUnsignedPayloadMiddleware(t *testing.T) {
-	cases := map[string]struct {
-		InitStep  func(*middleware.Stack) error
-		Mutator   func(*middleware.Stack) error
-		ExpectErr string
-		ExpectIDs []string
-	}{
-		"swap in place": {
-			InitStep: func(s *middleware.Stack) (err error) {
-				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
-				if err != nil {
-					return err
-				}
-				err = AddComputePayloadSHA256Middleware(s)
-				if err != nil {
-					return err
-				}
-				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
-				if err != nil {
-					return err
-				}
-				return nil
-			},
-			Mutator: SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
-			ExpectIDs: []string{
-				"ResolveEndpointV2",
-				computePayloadHashMiddlewareID, // should snap to after resolve endpoint
-				"before",
-				"after",
-			},
-		},
-
-		"already unsigned payload exists": {
-			InitStep: func(s *middleware.Stack) (err error) {
-				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
-				if err != nil {
-					return err
-				}
-				err = AddUnsignedPayloadMiddleware(s)
-				if err != nil {
-					return err
-				}
-				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
-				if err != nil {
-					return err
-				}
-				return nil
-			},
-			Mutator: SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
-			ExpectIDs: []string{
-				"ResolveEndpointV2",
-				computePayloadHashMiddlewareID,
-				"before",
-				"after",
-			},
-		},
-
-		"no compute payload": {
-			InitStep: func(s *middleware.Stack) (err error) {
-				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
-				if err != nil {
-					return err
-				}
-				err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
-				if err != nil {
-					return err
-				}
-				return nil
-			},
-			Mutator:   SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
-			ExpectErr: "not found, " + computePayloadHashMiddlewareID,
-		},
-	}
-
-	for name, c := range cases {
-		t.Run(name, func(t *testing.T) {
-			stack := middleware.NewStack(t.Name(), smithyhttp.NewStackRequest)
-			stack.Finalize.Add(&nopResolveEndpoint{}, middleware.After)
-			if err := c.InitStep(stack); err != nil {
-				t.Fatalf("expect no error, got %v", err)
-			}
-
-			err := c.Mutator(stack)
-			if len(c.ExpectErr) != 0 {
-				if err == nil {
-					t.Fatalf("expect error, got none")
-				}
-				if e, a := c.ExpectErr, err.Error(); !strings.Contains(a, e) {
-					t.Fatalf("expect error to contain %v, got %v", e, a)
-				}
-				return
-			}
-			if err != nil {
-				t.Fatalf("expect no error, got %v", err)
-			}
-
-			if diff := cmp.Diff(c.ExpectIDs, stack.Finalize.List()); len(diff) != 0 {
-				t.Errorf("expect match\n%v", diff)
-			}
-		})
-	}
-}
-
-func TestUseDynamicPayloadSigningMiddleware(t *testing.T) {
-	cases := map[string]struct {
-		content      io.Reader
-		url          string
-		expectedHash string
-		expectedErr  interface{}
-	}{
-		"TLS disabled": {
-			content: func() io.Reader {
-				br := bytes.NewReader([]byte("some content"))
-				return br
-			}(),
-			url:          "http://localhost.com/",
-			expectedHash: "290f493c44f5d63d06b374d0a5abd292fae38b92cab2fae5efefe1b0e9347f56",
-		},
-		"TLS enabled": {
-			content: func() io.Reader {
-				br := bytes.NewReader([]byte("some content"))
-				return br
-			}(),
-			url:          "https://localhost.com/",
-			expectedHash: "UNSIGNED-PAYLOAD",
-		},
-	}
-
-	for name, tt := range cases {
-		t.Run(name, func(t *testing.T) {
-			c := &dynamicPayloadSigningMiddleware{}
-
-			next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
-				value := GetPayloadHash(ctx)
-				if len(value) == 0 {
-					t.Fatalf("expected payload hash value to be on context")
-				}
-				if e, a := tt.expectedHash, value; e != a {
-					t.Errorf("expected %v, got %v", e, a)
-				}
-
-				return out, metadata, err
-			})
-
-			req := smithyhttp.NewStackRequest().(*smithyhttp.Request)
-			req.URL, _ = url.Parse(tt.url)
-			stream, err := req.SetStream(tt.content)
-			if err != nil {
-				t.Fatalf("expected no error, got %v", err)
-			}
-
-			_, _, err = c.HandleFinalize(context.Background(), middleware.FinalizeInput{Request: stream}, next)
-			if err != nil && tt.expectedErr == nil {
-				t.Errorf("expected no error, got %v", err)
-			} else if err != nil && tt.expectedErr != nil {
-				e, a := tt.expectedErr, err
-				if !errors.As(a, &e) {
-					t.Errorf("expected error type %T, got %T", e, a)
-				}
-			} else if err == nil && tt.expectedErr != nil {
-				t.Errorf("expected error, got nil")
-			}
-		})
-	}
-}
-
-type nonSeeker struct{}
-
-func (nonSeeker) Read(p []byte) (n int, err error) {
-	return 0, io.EOF
-}
-
-type semiSeekable struct {
-	hasSeeked bool
-}
-
-func (s *semiSeekable) Seek(offset int64, whence int) (int64, error) {
-	if !s.hasSeeked {
-		s.hasSeeked = true
-		return 0, nil
-	}
-	return 0, fmt.Errorf("io seek error")
-}
-
-func (*semiSeekable) Read(p []byte) (n int, err error) {
-	return 0, io.EOF
-}
-
-type nopResolveEndpoint struct{}
-
-func (*nopResolveEndpoint) ID() string { return "ResolveEndpointV2" }
-
-func (*nopResolveEndpoint) HandleFinalize(
-	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
-) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	return out, metadata, err
-}
-
-var (
-	_ middleware.FinalizeMiddleware = &UnsignedPayload{}
-	_ middleware.FinalizeMiddleware = &ComputePayloadSHA256{}
-	_ middleware.FinalizeMiddleware = &ContentSHA256Header{}
-	_ middleware.FinalizeMiddleware = &SignHTTPRequestMiddleware{}
-)

aws/signer/v4/presign_middleware.go

@@ -1,127 +0,0 @@
-package v4
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
-	"github.com/aws/smithy-go/middleware"
-	smithyHTTP "github.com/aws/smithy-go/transport/http"
-	"github.com/versity/versitygw/aws/internal/sdk"
-)
-
-// HTTPPresigner is an interface to a SigV4 signer that can sign create a
-// presigned URL for a HTTP requests.
-type HTTPPresigner interface {
-	PresignHTTP(
-		ctx context.Context, credentials aws.Credentials, r *http.Request,
-		payloadHash string, service string, region string, signingTime time.Time,
-		optFns ...func(*SignerOptions),
-	) (url string, signedHeader http.Header, err error)
-}
-
-// PresignedHTTPRequest provides the URL and signed headers that are included
-// in the presigned URL.
-type PresignedHTTPRequest struct {
-	URL          string
-	Method       string
-	SignedHeader http.Header
-}
-
-// PresignHTTPRequestMiddlewareOptions is the options for the PresignHTTPRequestMiddleware middleware.
-type PresignHTTPRequestMiddlewareOptions struct {
-	CredentialsProvider aws.CredentialsProvider
-	Presigner           HTTPPresigner
-	LogSigning          bool
-}
-
-// PresignHTTPRequestMiddleware provides the Finalize middleware for creating a
-// presigned URL for an HTTP request.
-//
-// Will short circuit the middleware stack and not forward onto the next
-// Finalize handler.
-type PresignHTTPRequestMiddleware struct {
-	credentialsProvider aws.CredentialsProvider
-	presigner           HTTPPresigner
-	logSigning          bool
-}
-
-// NewPresignHTTPRequestMiddleware returns a new PresignHTTPRequestMiddleware
-// initialized with the presigner.
-func NewPresignHTTPRequestMiddleware(options PresignHTTPRequestMiddlewareOptions) *PresignHTTPRequestMiddleware {
-	return &PresignHTTPRequestMiddleware{
-		credentialsProvider: options.CredentialsProvider,
-		presigner:           options.Presigner,
-		logSigning:          options.LogSigning,
-	}
-}
-
-// ID provides the middleware ID.
-func (*PresignHTTPRequestMiddleware) ID() string { return "PresignHTTPRequest" }
-
-// HandleFinalize will take the provided input and create a presigned url for
-// the http request using the SigV4 presign authentication scheme.
-//
-// Since the signed request is not a valid HTTP request
-func (s *PresignHTTPRequestMiddleware) HandleFinalize(
-	ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
-) (
-	out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-) {
-	req, ok := in.Request.(*smithyHTTP.Request)
-	if !ok {
-		return out, metadata, &SigningError{
-			Err: fmt.Errorf("unexpected request middleware type %T", in.Request),
-		}
-	}
-
-	httpReq := req.Build(ctx)
-	if !haveCredentialProvider(s.credentialsProvider) {
-		out.Result = &PresignedHTTPRequest{
-			URL:          httpReq.URL.String(),
-			Method:       httpReq.Method,
-			SignedHeader: http.Header{},
-		}
-
-		return out, metadata, nil
-	}
-
-	signingName := awsmiddleware.GetSigningName(ctx)
-	signingRegion := awsmiddleware.GetSigningRegion(ctx)
-	payloadHash := GetPayloadHash(ctx)
-	if len(payloadHash) == 0 {
-		return out, metadata, &SigningError{
-			Err: fmt.Errorf("computed payload hash missing from context"),
-		}
-	}
-
-	credentials, err := s.credentialsProvider.Retrieve(ctx)
-	if err != nil {
-		return out, metadata, &SigningError{
-			Err: fmt.Errorf("failed to retrieve credentials: %w", err),
-		}
-	}
-
-	u, h, err := s.presigner.PresignHTTP(ctx, credentials,
-		httpReq, payloadHash, signingName, signingRegion, sdk.NowTime(),
-		func(o *SignerOptions) {
-			o.Logger = middleware.GetLogger(ctx)
-			o.LogSigning = s.logSigning
-		})
-	if err != nil {
-		return out, metadata, &SigningError{
-			Err: fmt.Errorf("failed to sign http request, %w", err),
-		}
-	}
-
-	out.Result = &PresignedHTTPRequest{
-		URL:          u,
-		Method:       httpReq.Method,
-		SignedHeader: h,
-	}
-
-	return out, metadata, nil
-}

aws/signer/v4/presign_middleware_test.go

@@ -1,224 +0,0 @@
-package v4
-
-import (
-	"bytes"
-	"context"
-	"net/http"
-	"net/url"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
-	"github.com/aws/smithy-go/logging"
-	"github.com/aws/smithy-go/middleware"
-	smithyhttp "github.com/aws/smithy-go/transport/http"
-	"github.com/google/go-cmp/cmp"
-	"github.com/versity/versitygw/aws/internal/awstesting/unit"
-)
-
-type httpPresignerFunc func(
-	ctx context.Context, credentials aws.Credentials, r *http.Request,
-	payloadHash string, service string, region string, signingTime time.Time,
-	optFns ...func(*SignerOptions),
-) (url string, signedHeader http.Header, err error)
-
-func (f httpPresignerFunc) PresignHTTP(
-	ctx context.Context, credentials aws.Credentials, r *http.Request,
-	payloadHash string, service string, region string, signingTime time.Time,
-	optFns ...func(*SignerOptions),
-) (
-	url string, signedHeader http.Header, err error,
-) {
-	return f(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...)
-}
-
-func TestPresignHTTPRequestMiddleware(t *testing.T) {
-	cases := map[string]struct {
-		Request      *http.Request
-		Creds        aws.CredentialsProvider
-		PayloadHash  string
-		LogSigning   bool
-		ExpectResult *PresignedHTTPRequest
-		ExpectErr    string
-	}{
-		"success": {
-			Request: &http.Request{
-				URL: func() *url.URL {
-					u, _ := url.Parse("https://example.aws/path?query=foo")
-					return u
-				}(),
-				Header: http.Header{},
-			},
-			Creds:       unit.StubCredentialsProvider{},
-			PayloadHash: "0123456789abcdef",
-			ExpectResult: &PresignedHTTPRequest{
-				URL:          "https://example.aws/path?query=foo",
-				SignedHeader: http.Header{},
-			},
-		},
-		"error": {
-			Request: func() *http.Request {
-				return &http.Request{}
-			}(),
-			Creds:       unit.StubCredentialsProvider{},
-			PayloadHash: "",
-			ExpectErr:   "failed to sign request",
-		},
-		"anonymous creds": {
-			Request: &http.Request{
-				URL: func() *url.URL {
-					u, _ := url.Parse("https://example.aws/path?query=foo")
-					return u
-				}(),
-				Header: http.Header{},
-			},
-			Creds:       unit.StubCredentialsProvider{},
-			PayloadHash: "",
-			ExpectErr:   "failed to sign request",
-			ExpectResult: &PresignedHTTPRequest{
-				URL:          "https://example.aws/path?query=foo",
-				SignedHeader: http.Header{},
-			},
-		},
-		"nil creds": {
-			Request: &http.Request{
-				URL: func() *url.URL {
-					u, _ := url.Parse("https://example.aws/path?query=foo")
-					return u
-				}(),
-				Header: http.Header{},
-			},
-			Creds: nil,
-			ExpectResult: &PresignedHTTPRequest{
-				URL:          "https://example.aws/path?query=foo",
-				SignedHeader: http.Header{},
-			},
-		},
-		"with log signing": {
-			Request: &http.Request{
-				URL: func() *url.URL {
-					u, _ := url.Parse("https://example.aws/path?query=foo")
-					return u
-				}(),
-				Header: http.Header{},
-			},
-			Creds:       unit.StubCredentialsProvider{},
-			PayloadHash: "0123456789abcdef",
-			ExpectResult: &PresignedHTTPRequest{
-				URL:          "https://example.aws/path?query=foo",
-				SignedHeader: http.Header{},
-			},
-
-			LogSigning: true,
-		},
-	}
-
-	const (
-		signingName   = "serviceId"
-		signingRegion = "regionName"
-	)
-
-	for name, c := range cases {
-		t.Run(name, func(t *testing.T) {
-			m := &PresignHTTPRequestMiddleware{
-				credentialsProvider: c.Creds,
-
-				presigner: httpPresignerFunc(func(
-					ctx context.Context, credentials aws.Credentials, r *http.Request,
-					payloadHash string, service string, region string, signingTime time.Time,
-					optFns ...func(*SignerOptions),
-				) (url string, signedHeader http.Header, err error) {
-					var options SignerOptions
-					for _, fn := range optFns {
-						fn(&options)
-					}
-					if options.Logger == nil {
-						t.Errorf("expect logger, got none")
-					}
-					if options.LogSigning {
-						options.Logger.Logf(logging.Debug, t.Name())
-					}
-
-					if !haveCredentialProvider(c.Creds) {
-						t.Errorf("expect presigner not to be called for not credentials provider")
-					}
-
-					expectCreds, _ := unit.StubCredentialsProvider{}.Retrieve(context.Background())
-					if e, a := expectCreds, credentials; e != a {
-						t.Errorf("expected %v, got %v", e, a)
-					}
-					if e, a := c.PayloadHash, payloadHash; e != a {
-						t.Errorf("expected %v, got %v", e, a)
-					}
-					if e, a := signingName, service; e != a {
-						t.Errorf("expected %v, got %v", e, a)
-					}
-					if e, a := signingRegion, region; e != a {
-						t.Errorf("expected %v, got %v", e, a)
-					}
-
-					return c.ExpectResult.URL, c.ExpectResult.SignedHeader, nil
-				}),
-				logSigning: c.LogSigning,
-			}
-
-			next := middleware.FinalizeHandlerFunc(
-				func(ctx context.Context, in middleware.FinalizeInput) (
-					out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
-				) {
-					t.Errorf("expect next handler not to be called")
-					return out, metadata, err
-				})
-
-			ctx := awsmiddleware.SetSigningRegion(
-				awsmiddleware.SetSigningName(context.Background(), signingName),
-				signingRegion)
-
-			var loggerBuf bytes.Buffer
-			logger := logging.NewStandardLogger(&loggerBuf)
-			ctx = middleware.SetLogger(ctx, logger)
-
-			if len(c.PayloadHash) != 0 {
-				ctx = SetPayloadHash(ctx, c.PayloadHash)
-			}
-
-			result, _, err := m.HandleFinalize(ctx, middleware.FinalizeInput{
-				Request: &smithyhttp.Request{
-					Request: c.Request,
-				},
-			}, next)
-			if len(c.ExpectErr) != 0 {
-				if err == nil {
-					t.Fatalf("expect error, got none")
-				}
-				if e, a := c.ExpectErr, err.Error(); !strings.Contains(a, e) {
-					t.Fatalf("expect error to contain %v, got %v", e, a)
-				}
-				return
-			}
-			if err != nil {
-				t.Fatalf("expect no error, got %v", err)
-			}
-
-			if diff := cmp.Diff(c.ExpectResult, result.Result); len(diff) != 0 {
-				t.Errorf("expect result match\n%v", diff)
-			}
-
-			if c.LogSigning {
-				if e, a := t.Name(), loggerBuf.String(); !strings.Contains(a, e) {
-					t.Errorf("expect %v logged in %v", e, a)
-				}
-			} else {
-				if loggerBuf.Len() != 0 {
-					t.Errorf("expect no log, got %v", loggerBuf.String())
-				}
-			}
-		})
-	}
-}
-
-var (
-	_ middleware.FinalizeMiddleware = &PresignHTTPRequestMiddleware{}
-)

aws/signer/v4/stream.go

@@ -1,87 +0,0 @@
-package v4
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
-)
-
-// EventStreamSigner is an AWS EventStream protocol signer.
-type EventStreamSigner interface {
-	GetSignature(ctx context.Context, headers, payload []byte, signingTime time.Time, optFns ...func(*StreamSignerOptions)) ([]byte, error)
-}
-
-// StreamSignerOptions is the configuration options for StreamSigner.
-type StreamSignerOptions struct{}
-
-// StreamSigner implements Signature Version 4 (SigV4) signing of event stream encoded payloads.
-type StreamSigner struct {
-	options StreamSignerOptions
-
-	credentials aws.Credentials
-	service     string
-	region      string
-
-	prevSignature []byte
-
-	signingKeyDeriver *v4Internal.SigningKeyDeriver
-}
-
-// NewStreamSigner returns a new AWS EventStream protocol signer.
-func NewStreamSigner(credentials aws.Credentials, service, region string, seedSignature []byte, optFns ...func(*StreamSignerOptions)) *StreamSigner {
-	o := StreamSignerOptions{}
-
-	for _, fn := range optFns {
-		fn(&o)
-	}
-
-	return &StreamSigner{
-		options:           o,
-		credentials:       credentials,
-		service:           service,
-		region:            region,
-		signingKeyDeriver: v4Internal.NewSigningKeyDeriver(),
-		prevSignature:     seedSignature,
-	}
-}
-
-// GetSignature signs the provided header and payload bytes.
-func (s *StreamSigner) GetSignature(ctx context.Context, headers, payload []byte, signingTime time.Time, optFns ...func(*StreamSignerOptions)) ([]byte, error) {
-	options := s.options
-
-	for _, fn := range optFns {
-		fn(&options)
-	}
-
-	prevSignature := s.prevSignature
-
-	st := v4Internal.NewSigningTime(signingTime)
-
-	sigKey := s.signingKeyDeriver.DeriveKey(s.credentials, s.service, s.region, st)
-
-	scope := v4Internal.BuildCredentialScope(st, s.region, s.service)
-
-	stringToSign := s.buildEventStreamStringToSign(headers, payload, prevSignature, scope, &st)
-
-	signature := v4Internal.HMACSHA256(sigKey, []byte(stringToSign))
-	s.prevSignature = signature
-
-	return signature, nil
-}
-
-func (s *StreamSigner) buildEventStreamStringToSign(headers, payload, previousSignature []byte, credentialScope string, signingTime *v4Internal.SigningTime) string {
-	hash := sha256.New()
-	return strings.Join([]string{
-		"AWS4-HMAC-SHA256-PAYLOAD",
-		signingTime.TimeFormat(),
-		credentialScope,
-		hex.EncodeToString(previousSignature),
-		hex.EncodeToString(makeHash(hash, headers)),
-		hex.EncodeToString(makeHash(hash, payload)),
-	}, "\n")
-}

aws/signer/v4/v4_test.go

@@ -7,7 +7,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -208,7 +207,7 @@ func TestBuildCanonicalRequest(t *testing.T) {
 
 func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
 	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
-	req.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
+	req.Body = io.NopCloser(bytes.NewReader([]byte{}))
 
 	s := NewSigner()
 

aws/staticcheck.conf

@@ -1 +0,0 @@
-checks = []

backend/azure/azure.go

@@ -422,7 +422,7 @@ func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput)
 	return azureErrToS3Err(err)
 }
 
-func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	delResult, errs := []types.DeletedObject{}, []types.Error{}
 	for _, obj := range input.Delete.Objects {
 		err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
@@ -449,7 +449,7 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
 		}
 	}
 
-	return s3response.DeleteObjectsResult{
+	return s3response.DeleteResult{
 		Deleted: delResult,
 		Error:   errs,
 	}, nil
@@ -466,16 +466,6 @@ func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3
 		return nil, azureErrToS3Err(err)
 	}
 
-	dstContainerAcl, err := getAclFromMetadata(res.Metadata, aclKeyCapital)
-	if err != nil {
-		return nil, err
-	}
-
-	err = auth.VerifyACL(*dstContainerAcl, *input.ExpectedBucketOwner, types.PermissionWrite, false)
-	if err != nil {
-		return nil, err
-	}
-
 	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource && isMetaSame(res.Metadata, input.Metadata) {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
 	}

backend/backend.go

@@ -40,6 +40,9 @@ type Backend interface {
 	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
 	PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
 	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
+	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
+	DeleteBucketPolicy(_ context.Context, bucket string) error
 
 	// multipart operations
 	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
@@ -60,7 +63,7 @@ type Backend interface {
 	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
 	DeleteObject(context.Context, *s3.DeleteObjectInput) error
-	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
 	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
 
@@ -118,6 +121,15 @@ func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVers
 func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
 func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -168,8 +180,8 @@ func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input)
 func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
-	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)

backend/posix/posix.go

@@ -61,6 +61,7 @@ const (
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
 	aclkey              = "user.acl"
 	etagkey             = "user.etag"
+	policykey           = "user.policy"
 )
 
 func New(rootdir string) (*Posix, error) {
@@ -74,6 +75,12 @@ func New(rootdir string) (*Posix, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
+	_, err = xattr.FGet(f, "user.test")
+	if errors.Is(err, syscall.ENOTSUP) {
+		f.Close()
+		return nil, fmt.Errorf("xattr not supported on %v", rootdir)
+	}
+
 	return &Posix{rootfd: f, rootdir: rootdir}, nil
 }
 
@@ -369,10 +376,9 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
-			if err != nil {
-				return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
-			}
+		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 		}
 	}
 	err = f.link()
@@ -436,7 +442,7 @@ func loadUserMetaData(path string, m map[string]string) (contentType, contentEnc
 			continue
 		}
 		b, err := xattr.Get(path, e)
-		if err == syscall.ENODATA {
+		if err == errNoData {
 			m[strings.TrimPrefix(e, fmt.Sprintf("user.%v.", metaHdr))] = ""
 			continue
 		}
@@ -1189,7 +1195,7 @@ func (p *Posix) removeParents(bucket, object string) error {
 	return nil
 }
 
-func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	// delete object already checks bucket
 	delResult, errs := []types.DeletedObject{}, []types.Error{}
 	for _, obj := range input.Delete.Objects {
@@ -1218,7 +1224,7 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 		}
 	}
 
-	return s3response.DeleteObjectsResult{
+	return s3response.DeleteResult{
 		Deleted: delResult,
 		Error:   errs,
 	}, nil
@@ -1423,7 +1429,6 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 	}
 	dstBucket := *input.Bucket
 	dstObject := *input.Key
-	owner := *input.ExpectedBucketOwner
 
 	_, err := os.Stat(srcBucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1441,22 +1446,6 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
-	dstBucketACLBytes, err := xattr.Get(dstBucket, aclkey)
-	if err != nil {
-		return nil, fmt.Errorf("get dst bucket acl tag: %w", err)
-	}
-
-	var dstBucketACL auth.ACL
-	err = json.Unmarshal(dstBucketACLBytes, &dstBucketACL)
-	if err != nil {
-		return nil, fmt.Errorf("parse dst bucket acl: %w", err)
-	}
-
-	err = auth.VerifyACL(dstBucketACL, owner, types.PermissionWrite, false)
-	if err != nil {
-		return nil, err
-	}
-
 	objPath := filepath.Join(srcBucket, srcObject)
 	f, err := os.Open(objPath)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1852,6 +1841,58 @@ func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object string)
 	return p.PutObjectTagging(ctx, bucket, object, nil)
 }
 
+func (p *Posix) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if policy == nil {
+		if err := xattr.Remove(bucket, policykey); err != nil {
+			if isNoAttr(err) {
+				return nil
+			}
+
+			return fmt.Errorf("remove policy: %w", err)
+		}
+
+		return nil
+	}
+
+	if err := xattr.Set(bucket, policykey, policy); err != nil {
+		return fmt.Errorf("set policy: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Posix) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	policy, err := xattr.Get(bucket, policykey)
+	if isNoAttr(err) {
+		return []byte{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get bucket policy: %w", err)
+	}
+
+	return policy, nil
+}
+
+func (p *Posix) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	return p.PutBucketPolicy(ctx, bucket, nil)
+}
+
 func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1935,7 +1976,7 @@ func isNoAttr(err error) bool {
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == syscall.ENODATA {
+	if err == errNoData {
 		return true
 	}
 	return false

backend/posix/posix_windows.go

@@ -1,89 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package posix
-
-import (
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-)
-
-type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	size    int64
-}
-
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
-	// Create a temp file for upload while in progress (see link comments below).
-	err := os.MkdirAll(dir, 0700)
-	if err != nil {
-		return nil, fmt.Errorf("make temp dir: %w", err)
-	}
-	f, err := os.CreateTemp(dir,
-		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-	if err != nil {
-		return nil, err
-	}
-	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
-}
-
-func (tmp *tmpfile) link() error {
-	tempname := tmp.f.Name()
-	// cleanup in case anything goes wrong, if rename succeeds then
-	// this will no longer exist
-	defer os.Remove(tempname)
-
-	// We use Rename as the atomic operation for object puts. The upload is
-	// written to a temp file to not conflict with any other simultaneous
-	// uploads. The final operation is to move the temp file into place for
-	// the object. This ensures the object semantics of last upload completed
-	// wins and is not some combination of writes from simultaneous uploads.
-	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err := os.Remove(objPath)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove stale path: %w", err)
-	}
-
-	err = tmp.f.Close()
-	if err != nil {
-		return fmt.Errorf("close tmpfile: %w", err)
-	}
-
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
-}
-
-func (tmp *tmpfile) Write(b []byte) (int, error) {
-	if int64(len(b)) > tmp.size {
-		return 0, fmt.Errorf("write exceeds content length")
-	}
-
-	n, err := tmp.f.Write(b)
-	tmp.size -= int64(n)
-	return n, err
-}
-
-func (tmp *tmpfile) cleanup() {
-	tmp.f.Close()
-}

backend/posix/with_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !freebsd && !openbsd && !netbsd
+// +build !freebsd,!openbsd,!netbsd
+
+package posix
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENODATA
+)

backend/posix/with_otmpfile.go

@@ -12,6 +12,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build linux
+// +build linux
+
 package posix
 
 import (

backend/posix/without_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build freebsd || openbsd || netbsd
+// +build freebsd openbsd netbsd
+
+package posix
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENOATTR
+)

backend/posix/without_otmpfile.go

@@ -12,6 +12,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build !linux
+// +build !linux
+
 package posix
 
 import (

backend/s3proxy/s3.go

@@ -320,17 +320,17 @@ func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput)
 	return handleError(err)
 }
 
-func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	if len(input.Delete.Objects) == 0 {
 		input.Delete.Objects = []types.ObjectIdentifier{}
 	}
 
 	output, err := s.client.DeleteObjects(ctx, input)
 	if err != nil {
-		return s3response.DeleteObjectsResult{}, handleError(err)
+		return s3response.DeleteResult{}, handleError(err)
 	}
 
-	return s3response.DeleteObjectsResult{
+	return s3response.DeleteResult{
 		Deleted: output.Deleted,
 		Error:   output.Errors,
 	}, nil

backend/scoutfs/scoutfs.go

@@ -25,7 +25,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -140,7 +139,10 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	partsize := int64(0)
 	var totalsize int64
 	for i, p := range parts {
-		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
+		if p.PartNumber == nil {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
+		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
 		fi, err := os.Lstat(partPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
@@ -179,9 +181,9 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	defer f.cleanup()
 
 	for _, p := range parts {
-		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
+		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
+			return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
@@ -190,7 +192,7 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
+			return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
 		}
 	}
 
@@ -201,10 +203,9 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
-			if err != nil {
-				return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
-			}
+		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 		}
 	}
 	err = f.link()
@@ -268,7 +269,7 @@ func loadUserMetaData(path string, m map[string]string) (contentType, contentEnc
 			continue
 		}
 		b, err := xattr.Get(path, e)
-		if err == syscall.ENODATA {
+		if err == errNoData {
 			m[strings.TrimPrefix(e, "user.")] = ""
 			continue
 		}
@@ -456,6 +457,13 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, err
 	}
 
+	objSize := fi.Size()
+	if fi.IsDir() {
+		// directory objects are always 0 len
+		objSize = 0
+		length = 0
+	}
+
 	if length == -1 {
 		length = fi.Size() - startOffset + 1
 	}
@@ -464,6 +472,11 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
+	var contentRange string
+	if acceptRange != "" {
+		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
+	}
+
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
@@ -521,6 +534,7 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		Metadata:        userMetaData,
 		TagCount:        &tagCount,
 		StorageClass:    types.StorageClassStandard,
+		ContentRange:    &contentRange,
 	}, nil
 }
 
@@ -806,7 +820,7 @@ func isNoAttr(err error) bool {
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == syscall.ENODATA {
+	if err == errNoData {
 		return true
 	}
 	return false

backend/scoutfs/scoutfs_incompat.go

@@ -34,7 +34,7 @@ var (
 	errNotSupported = errors.New("not supported")
 )
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+func openTmpFile(_, _, _ string, _ int64) (*tmpfile, error) {
 	return nil, errNotSupported
 }
 
@@ -49,10 +49,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
-func moveData(from *os.File, to *os.File) error {
+func moveData(_, _ *os.File) error {
 	return errNotSupported
 }
 
-func statMore(path string) (stat, error) {
+func statMore(_ string) (stat, error) {
 	return stat{}, errNotSupported
 }

backend/scoutfs/with_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !freebsd && !openbsd && !netbsd
+// +build !freebsd,!openbsd,!netbsd
+
+package scoutfs
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENODATA
+)

backend/scoutfs/without_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build freebsd || openbsd || netbsd
+// +build freebsd openbsd netbsd
+
+package scoutfs
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENOATTR
+)

cmd/versitygw/gateway_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/versity/versitygw/backend/posix"
-	"github.com/versity/versitygw/integration"
+	"github.com/versity/versitygw/tests/integration"
 )
 
 const (

cmd/versitygw/main.go

@@ -197,13 +197,13 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
-			EnvVars:     []string{"LOGFILE"},
+			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
 			Destination: &accessLog,
 		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
-			EnvVars:     []string{"WEBHOOK"},
+			EnvVars:     []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"},
 			Destination: &logWebhookURL,
 		},
 		&cli.StringFlag{
@@ -369,6 +369,10 @@ func initFlags() []cli.Flag {
 }
 
 func runGateway(ctx context.Context, be backend.Backend) error {
+	if rootUserAccess == "" || rootUserSecret == "" {
+		return fmt.Errorf("root user access and secret key must be provided")
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:           "versitygw",
 		ServerHeader:      "VERSITYGW",
@@ -492,7 +496,6 @@ Loop:
 	for {
 		select {
 		case <-ctx.Done():
-			err = ctx.Err()
 			break Loop
 		case err = <-c:
 			break Loop
@@ -512,12 +515,18 @@ Loop:
 
 	err = iam.Shutdown()
 	if err != nil {
+		if saveErr == nil {
+			saveErr = err
+		}
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
 	if logger != nil {
 		err := logger.Shutdown()
 		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
 			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
 		}
 	}

cmd/versitygw/s3.go

@@ -44,6 +44,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server access key id",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_ACCESS_KEY"},
 				Destination: &s3proxyAccess,
 				Aliases:     []string{"a"},
 			},
@@ -52,6 +53,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server secret access key",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_SECRET_KEY"},
 				Destination: &s3proxySecret,
 				Aliases:     []string{"s"},
 			},
@@ -59,23 +61,27 @@ to an s3 storage backend service.`,
 				Name:        "endpoint",
 				Usage:       "s3 service endpoint, default AWS if not specified",
 				Value:       "",
+				EnvVars:     []string{"VGW_S3_ENDPOINT"},
 				Destination: &s3proxyEndpoint,
 			},
 			&cli.StringFlag{
 				Name:        "region",
 				Usage:       "s3 service region, default 'us-east-1' if not specified",
 				Value:       "us-east-1",
+				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
+				EnvVars:     []string{"VGW_S3_SSL_SKIP_VERIFY"},
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
@@ -83,6 +89,7 @@ to an s3 storage backend service.`,
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DEBUG"},
 				Destination: &s3proxyDebug,
 			},
 		},

cmd/versitygw/scoutfs.go

@@ -48,6 +48,7 @@ move interfaces as well as support for tiered filesystems.`,
 				Name:        "glacier",
 				Usage:       "enable glacier emulation mode",
 				Aliases:     []string{"g"},
+				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
 		},

cmd/versitygw/test.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/integration"
+	"github.com/versity/versitygw/tests/integration"
 )
 
 var (

docker-compose.yml

@@ -29,7 +29,7 @@ services:
       - "10002:10002"
     restart: always
     hostname: azurite
-    command: "azurite --oauth basic --cert /certs/azurite.pem --key /certs/azurite-key.pem --blobHost 0.0.0.0"
+    command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
     volumes:
       - ./certs:/certs
   azuritegw:

extra/example.conf

@@ -0,0 +1,272 @@
+###################################
+# VersityGW systemd configuration #
+###################################
+
+# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
+# For example, if the service name is "mygateway", then the file should be
+# named /etc/versitygw.d/mygateway.conf.
+# The systemd template file /lib/systemd/system/versitygw@.service will
+# automatically load the configuration file for the service name.
+# To start the gateway, use the following command:
+# systemctl start versitygw@mygateway
+# To enable the gateway to start on boot, use the following command:
+# systemctl enable versitygw@mygateway
+# To stop the gateway, use the following command:
+# systemctl stop versitygw@mygateway
+
+# There can be multiple gateway services running on the same host. Each
+# gateway service must have a unique service name with a unique configuration
+# file in /etc/versitygw.d/. They must also listen on different ports and/or
+# interfaces using the VGW_PORT option.
+
+##############################
+# VersityGW Required Options #
+##############################
+
+# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3
+# This defines the backend that the VGW will use for data access.
+VGW_BACKEND=posix
+
+# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
+# as the the top level directory for the gateway.
+# All sub directories of the top level directory are treated as buckets,
+# and all files/directories below the "bucket directory" are treated as
+# the objects. The object name is split on "/" separator to translate
+# to posix storage.
+# For example:
+# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+VGW_BACKEND_ARG=
+
+############################
+# VersityGW Global Options #
+############################
+
+# commented options are the default values
+
+# The following must be set, and do not have default values
+# The access and secret options will specify the root account credentials.
+# The root account is granted full authorization to all API requests after
+# authentication.
+ROOT_ACCESS_KEY_ID=
+ROOT_SECRET_ACCESS_KEY=
+
+# The following are optional, and have the default values as listed
+
+# The VGW_PORT option will specify the listening port for the S3 server.
+# This option can use either the form <ip>:<port> which will listen only
+# on the network interface that matches the IP on the specified port, or
+# :<port> which will listen on all network interfaces on the specified port.
+# The <ip> spec can either be IP dotted notation or a resolvable hostname.
+# The <port> spec can either be a numeric port or the service name typically
+# in /etc/services.
+#VGW_PORT=:7070
+
+# The VGW_REGION option will specify the region that the S3 server will
+# report to clients. This option is optional, and defaults to "us-east-1".
+#VGW_REGION=us-east-1
+
+# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
+# private key that the S3 server will use for SSL connections. This option
+# is optional, and defaults to not using SSL.
+#VGW_CERT=
+#VGW_KEY=
+
+# The VGW_ADMIN_PORT option will specify the listening port for the admin
+# server. The admin server endpoint can optionally be set to listen on a
+# different interface or port than the S3 service. This allows for better
+# control of firewall restrictions to the admin endpoint. The certs for this
+# can be different certs than specified for the S3 service. The default when
+# these are not specified is to have the admin server listen on the same
+# endpoint as the S3 service.
+# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
+# server will use SSL.
+#VGW_ADMIN_PORT=
+#VGW_ADMIN_CERT=
+#VGW_ADMIN_CERT_KEY=
+
+# The VGW_QUIET option when set will supress the S3 server request summary
+# logging to stdout.
+#VGW_QUIET=false
+
+# The VGW_HEALTH option when set will specify the URL to accept health checks
+# on. The health check endpoint is often used for load balancers to verify
+# gateway is alive. The health endpoint masks any bucket with this setting.
+# For example, if the health endpoint is set to /health, the gateway will not
+# allow creating or listing contents of a bucket called "health". The health
+# endpoint is unauthenticated, and returns a 200 status for GET.
+#VGW_HEALTH=
+
+###############
+# Access Logs #
+###############
+
+# The VGW_ACCESS_LOG option when set will specify the file to log all S3
+# server requests to. This option is optional, and defaults to not logging.
+# It is suggested to use absolute paths for the server log file because the
+# server may chdir into the backend root directory and change locations for
+# relative paths.
+# The log file format follows the AWS S3 access log format documented in
+# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
+#VGW_ACCESS_LOG=
+
+# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
+# S3 server request access logs to. The access logs are JSON encoded when
+# sent to the webhook.
+#VGW_LOG_WEBHOOK_URL=
+
+##############
+# Event Logs #
+##############
+
+# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
+# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
+# bucket events will be sent to the kafka service. The gateway events are
+# similar to AWS S3 events, and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_KAFKA_URL=
+#VGW_EVENT_KAFKA_TOPIC=
+#VGW_EVENT_KAFKA_KEY=
+
+# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
+# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the
+# the NATS messaging service. The gateway events are  similar to AWS S3 events,
+# and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_NATS_URL=
+#VGW_EVENT_NATS_TOPIC=
+
+# The VGW_DEBUG option enables verbose debug log output to stdout. This output
+# includes details for signature verification steps. This is generally only
+# useful for debugging the S3 server, and should not be used in production.
+#VGW_DEBUG=false
+
+################
+# IAM services #
+################
+
+# The VGW_IAM_DIR option will enable the internal IAM service with accounts
+# stored in a file under the specified directory. This is provided to minimize
+# dependencies on outside services for basic functionality. The local account
+# files are plain text and only protected with file permissions. This IAM
+# service is added for convenience, but is not considered as secure or scalable
+# as a dedicated IAM service.
+#VGW_IAM_DIR=
+
+# The ldap options will enable the LDAP IAM service with accounts stored in an
+# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
+# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
+# secret credentials and role respectively. The other options are used to
+# connect to the LDAP service.
+#VGW_IAM_LDAP_URL=
+#VGW_IAM_LDAP_BASE_DN=
+#VGW_IAM_LDAP_BIND_DN=
+#VGW_IAM_LDAP_BIND_PASS=
+#VGW_IAM_LDAP_QUERY_BASE=
+#VGW_IAM_LDAP_OBJECT_CLASSES=
+#VGW_IAM_LDAP_ACCESS_ATR=
+#VGW_IAM_LDAP_SECRET_ATR=
+#VGW_IAM_LDAP_ROLE_ATR=
+
+# The VGW_S3 IAM service is similar to the internal IAM service, but instead
+# stores the account information JSON encoded in an S3 object. This should use
+# a bucket that is not accessible to general users when using s3 backend to
+# prevent access to account credentials. This IAM service is added for
+# convenience, but is not considered as secure or scalable as a dedicated IAM
+# service.
+#VGW_S3_IAM_ACCESS_KEY=
+#VGW_S3_IAM_SECRET_KEY=
+#VGW_S3_IAM_REGION=
+#VGW_S3_IAM_ENDPOINT=
+#VGW_S3_IAM_BUCKET=
+#VGW_S3_IAM_NO_VERIFY=
+
+###############
+# IAM caching #
+###############
+
+# The IAM cache is intended to ease the load on the IAM service and increase
+# the Gateway performance by caching accounts and credentials for the TTL time
+# interval. Disabling this will cause a request to the configured IAM service
+# for each incoming request to retrieve the corresponding account credentials.
+# The cache is enabled by default. The TTL specifies how long to cache
+# credentials, and the prune value determines the interval for expired entries
+# to be removed from the cache. Increasing the TTL may lessen the load on the
+# IAM service backend, but may have out of date account info until the next
+# interval. Increasing the prune value may reduce memory use at the cost of
+# added CPU to check cache expirations.
+#VGW_IAM_CACHE_DISABLE=false
+#VGW_IAM_CACHE_TTL=120
+#VGW_IAM_CACHE_PRUNE=3600
+
+######################################
+# VersityGW Backend Specific Options #
+######################################
+
+#########
+# posix #
+#########
+
+# The posix backend translates S3 requests to file access in a local filesystem.
+# The posix backend requires a filesystem that supports extended attributes.
+# The top level directory for the gateway must be provided. All sub directories
+# of the top level directory are treated as buckets, and all files/directories
+# below the "bucket directory" are treated as the objects. The object
+# name is split on "/" separator to translate to posix storage.
+# For example:
+# top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+
+# There are currently no further options other than VGW_BACKEND_ARG for the
+# posix backend.
+
+###########
+# scoutfs #
+###########
+
+# The scoutfs backend requires a ScoutFS filesystem type for the backend
+# path. The glacier mode functionality requires ScoutAM to be configured
+# for tiering data from the ScoutFS filesystem to a mass stroage system.
+# The mass storage system is often one or more tape libraries. Due to the
+# high latency of tape, the glacier mode functionality is designed to
+# give feedback to clients about object state and offer ability to request
+# data to be staged back to disk without the client dealing with long
+# request timeout settings.
+
+# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
+# GET object:  if file offline, return invalid object state
+# HEAD object: if file offline, set obj storage class to GLACIER
+#              if file offline and staging, x-amz-restore: ongoing-request="true"
+#              if file offline and not staging, x-amz-restore: ongoing-request="false"
+#              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
+#              note: this expiry-date is not used but provided for client glacier compatibility
+# ListObjects: if file offline, set obj storage class to GLACIER
+# RestoreObject: add batch stage request to file
+#VGW_SCOUTFS_GLACIER=false
+
+######
+# s3 #
+######
+
+# The s3 backend allows the gateway to forward requests to an S3 compatible
+# service. This allows the gateway to act as a proxy for another S3 service.
+# The backend S3 access is all done with a single configured account. The
+# gateway will manage incoming multi-tenant access with the gateway configured
+# IAM service. This gives stroage admins the ability to manage local gateway
+# accounts while maintaining full control and a single account for the backend
+# S3 service.
+
+# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
+# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
+# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
+#VGW_S3_ACCESS_KEY=
+#VGW_S3_SECRET_KEY=
+#VGW_S3_REGION=
+#VGW_S3_ENDPOINT=
+#VGW_S3_DISABLE_CHECKSUM=false
+#VGW_S3_SSL_SKIP_VERIFY=false
+#VGW_S3_DEBUG=false

extra/posttrans.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+systemctl daemon-reload

extra/versitygw@.service

@@ -0,0 +1,33 @@
+[Unit]
+Description=VersityGW
+Documentation=https://github.com/versity/versitygw/wiki
+Wants=network-online.target
+After=network-online.target remote-fs.target
+AssertFileIsExecutable=/usr/bin/versitygw
+AssertPathExists=/etc/versitygw.d/%i.conf
+
+[Service]
+WorkingDirectory=/root
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=versitygw-%i
+
+User=root
+Group=root
+
+EnvironmentFile=/etc/versitygw.d/%i.conf
+
+ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"'
+
+# Let systemd restart this service always
+Restart=always
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65536
+
+# Specifies the maximum number of threads this process can create
+TasksMax=infinity
+
+[Install]
+WantedBy=multi-user.target
+

go.mod

@@ -6,14 +6,14 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
-	github.com/aws/aws-sdk-go-v2 v1.25.2
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2
+	github.com/aws/aws-sdk-go-v2 v1.26.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
 	github.com/aws/smithy-go v1.20.1
 	github.com/go-ldap/ldap/v3 v3.4.6
-	github.com/gofiber/fiber/v2 v2.52.2
+	github.com/gofiber/fiber/v2 v2.52.3
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.33.1
+	github.com/nats-io/nats.go v1.34.0
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/urfave/cli/v2 v2.27.1
@@ -26,11 +26,11 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -47,16 +47,16 @@ require (
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.5
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.5
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.9
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.9
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -16,42 +16,42 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w=
-github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
+github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
+github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
-github.com/aws/aws-sdk-go-v2/config v1.27.5 h1:brBPsyRFQn97M1ZhQ9tLXkO7Zytiar0NS06FGmEJBdg=
-github.com/aws/aws-sdk-go-v2/config v1.27.5/go.mod h1:I53uvsfddRRTG5YcC4n5Z3aOD1BU8hYCoIG7iEJG4wM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.5 h1:yn3zSvIKC2NZIs40cY3kckcy9Zma96PrRR07N54PCvY=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.5/go.mod h1:8JcKPAGZVnDWuR5lusAwmrSDtZnDIAnpQWaDC9RFt2g=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7 h1:/r2O0R/JAD1Y1iCxxz7nClKntXqB9CLTrxu7csrAsSA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7/go.mod h1:TbQoOduGh1PZbTNRqaEemgj/e+mmFC3hScHEQDTcUoQ=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 h1:bNo4LagzUKbjdxE0tIcR9pMzLR2U/Tgie1Hq1HQ3iH8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 h1:EtOU5jsPdIQNP+6Q2C5e3d65NKT1PeCiQk+9OdzO12Q=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
+github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg=
+github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 h1:F+PUZee9mlfpEJVZdgyewRumKekS9O3fftj8fEMt0rQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13/go.mod h1:Rl7i2dEWGHGsBIJCpUxlRt7VwK/HyXxICxdvIRssQHE=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 h1:en92G0Z7xlksoOylkUhuBSfJgijC7rHVLRdnIlHEs0E=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2/go.mod h1:HgtQ/wN5G+8QSlK62lbOtNwQ3wTSByJ4wH2rCkPt+AE=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3 h1:fpFzBoro/MetYBk+8kxoQGMeKSkXbymnbUh2gy6nVgk=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3/go.mod h1:qmQPbMe5NQk/nEmpkl8iHyCSREJjEbRUrnqHpHabLlM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3 h1:x0N5ftQzgcfRpCpTiyZC40pvNUJYhzf4UgCsAyO6/P8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 h1:1oY1AVEisRI4HNuFoLdRUB0hC63ylDAN6Me3MrfclEg=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2/go.mod h1:KZ03VgvZwSjkT7fOetQ/wF3MZUvYFirlI1H5NklUNsY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2 h1:ukAaTX8n/pX0Essg9CxW8VCjACv75vnNo2GRONR1w1Q=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2/go.mod h1:wt4wZz/CBlJJwY0L7X6vPQ9njh2aHi59knqpJ6B/2cM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 h1:utEGkfdQ4L6YW/ietH7111ZYglLJvS+sLriHJ1NBJEQ=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 h1:9/GylMS45hGGFCcMrUZDVayQE1jYSIN6da9jo7RAYIw=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.2 h1:0YjXuWdYHvsm0HnT4vO8XpwG1D+i2roxSCBoN6deJ7M=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.2/go.mod h1:jI+FWmYkSMn+4APWmZiZTgt0oM0TrvymD51FMqCnWgA=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
 github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
@@ -65,8 +65,8 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
 github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
-github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo=
-github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/fiber/v2 v2.52.3 h1:bgAZwPv0aHIfRwIUdkWhg6U8D3MEYnoJjT+HfW/dDTo=
+github.com/gofiber/fiber/v2 v2.52.3/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
 github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -90,8 +90,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
-github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk=
+github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=

s3api/controllers/backend_moq_test.go

@@ -44,6 +44,9 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketPolicy method")
+//			},
 //			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
 //				panic("mock out the DeleteBucketTagging method")
 //			},
@@ -53,12 +56,15 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) error {
 //				panic("mock out the DeleteObjectTagging method")
 //			},
-//			DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+//			DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 //				panic("mock out the DeleteObjects method")
 //			},
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+//				panic("mock out the GetBucketPolicy method")
+//			},
 //			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 //				panic("mock out the GetBucketTagging method")
 //			},
@@ -107,6 +113,9 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
+//				panic("mock out the PutBucketPolicy method")
+//			},
 //			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
 //				panic("mock out the PutBucketTagging method")
 //			},
@@ -168,6 +177,9 @@ type BackendMock struct {
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
+	// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
+	DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteBucketTaggingFunc mocks the DeleteBucketTagging method.
 	DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
 
@@ -178,11 +190,14 @@ type BackendMock struct {
 	DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error
 
 	// DeleteObjectsFunc mocks the DeleteObjects method.
-	DeleteObjectsFunc func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	DeleteObjectsFunc func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketPolicyFunc mocks the GetBucketPolicy method.
+	GetBucketPolicyFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
+
 	// GetBucketTaggingFunc mocks the GetBucketTagging method.
 	GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
 
@@ -231,6 +246,9 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketPolicyFunc mocks the PutBucketPolicy method.
+	PutBucketPolicyFunc func(contextMoqParam context.Context, bucket string, policy []byte) error
+
 	// PutBucketTaggingFunc mocks the PutBucketTagging method.
 	PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error
 
@@ -319,6 +337,13 @@ type BackendMock struct {
 			// DeleteBucketInput is the deleteBucketInput argument value.
 			DeleteBucketInput *s3.DeleteBucketInput
 		}
+		// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
+		DeleteBucketPolicy []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteBucketTagging holds details about calls to the DeleteBucketTagging method.
 		DeleteBucketTagging []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -356,6 +381,13 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketPolicy holds details about calls to the GetBucketPolicy method.
+		GetBucketPolicy []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetBucketTagging holds details about calls to the GetBucketTagging method.
 		GetBucketTagging []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -474,6 +506,15 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketPolicy holds details about calls to the PutBucketPolicy method.
+		PutBucketPolicy []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Policy is the policy argument value.
+			Policy []byte
+		}
 		// PutBucketTagging holds details about calls to the PutBucketTagging method.
 		PutBucketTagging []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -557,11 +598,13 @@ type BackendMock struct {
 	lockCreateBucket            sync.RWMutex
 	lockCreateMultipartUpload   sync.RWMutex
 	lockDeleteBucket            sync.RWMutex
+	lockDeleteBucketPolicy      sync.RWMutex
 	lockDeleteBucketTagging     sync.RWMutex
 	lockDeleteObject            sync.RWMutex
 	lockDeleteObjectTagging     sync.RWMutex
 	lockDeleteObjects           sync.RWMutex
 	lockGetBucketAcl            sync.RWMutex
+	lockGetBucketPolicy         sync.RWMutex
 	lockGetBucketTagging        sync.RWMutex
 	lockGetBucketVersioning     sync.RWMutex
 	lockGetObject               sync.RWMutex
@@ -578,6 +621,7 @@ type BackendMock struct {
 	lockListObjectsV2           sync.RWMutex
 	lockListParts               sync.RWMutex
 	lockPutBucketAcl            sync.RWMutex
+	lockPutBucketPolicy         sync.RWMutex
 	lockPutBucketTagging        sync.RWMutex
 	lockPutBucketVersioning     sync.RWMutex
 	lockPutObject               sync.RWMutex
@@ -851,6 +895,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketPolicy calls DeleteBucketPolicyFunc.
+func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketPolicyFunc == nil {
+		panic("BackendMock.DeleteBucketPolicyFunc: method is nil but Backend.DeleteBucketPolicy was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketPolicy.Lock()
+	mock.calls.DeleteBucketPolicy = append(mock.calls.DeleteBucketPolicy, callInfo)
+	mock.lockDeleteBucketPolicy.Unlock()
+	return mock.DeleteBucketPolicyFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketPolicyCalls gets all the calls that were made to DeleteBucketPolicy.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketPolicyCalls())
+func (mock *BackendMock) DeleteBucketPolicyCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketPolicy.RLock()
+	calls = mock.calls.DeleteBucketPolicy
+	mock.lockDeleteBucketPolicy.RUnlock()
+	return calls
+}
+
 // DeleteBucketTagging calls DeleteBucketTaggingFunc.
 func (mock *BackendMock) DeleteBucketTagging(contextMoqParam context.Context, bucket string) error {
 	if mock.DeleteBucketTaggingFunc == nil {
@@ -964,7 +1044,7 @@ func (mock *BackendMock) DeleteObjectTaggingCalls() []struct {
 }
 
 // DeleteObjects calls DeleteObjectsFunc.
-func (mock *BackendMock) DeleteObjects(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (mock *BackendMock) DeleteObjects(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	if mock.DeleteObjectsFunc == nil {
 		panic("BackendMock.DeleteObjectsFunc: method is nil but Backend.DeleteObjects was just called")
 	}
@@ -1035,6 +1115,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketPolicy calls GetBucketPolicyFunc.
+func (mock *BackendMock) GetBucketPolicy(contextMoqParam context.Context, bucket string) ([]byte, error) {
+	if mock.GetBucketPolicyFunc == nil {
+		panic("BackendMock.GetBucketPolicyFunc: method is nil but Backend.GetBucketPolicy was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketPolicy.Lock()
+	mock.calls.GetBucketPolicy = append(mock.calls.GetBucketPolicy, callInfo)
+	mock.lockGetBucketPolicy.Unlock()
+	return mock.GetBucketPolicyFunc(contextMoqParam, bucket)
+}
+
+// GetBucketPolicyCalls gets all the calls that were made to GetBucketPolicy.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketPolicyCalls())
+func (mock *BackendMock) GetBucketPolicyCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketPolicy.RLock()
+	calls = mock.calls.GetBucketPolicy
+	mock.lockGetBucketPolicy.RUnlock()
+	return calls
+}
+
 // GetBucketTagging calls GetBucketTaggingFunc.
 func (mock *BackendMock) GetBucketTagging(contextMoqParam context.Context, bucket string) (map[string]string, error) {
 	if mock.GetBucketTaggingFunc == nil {
@@ -1623,6 +1739,46 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketPolicy calls PutBucketPolicyFunc.
+func (mock *BackendMock) PutBucketPolicy(contextMoqParam context.Context, bucket string, policy []byte) error {
+	if mock.PutBucketPolicyFunc == nil {
+		panic("BackendMock.PutBucketPolicyFunc: method is nil but Backend.PutBucketPolicy was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Policy          []byte
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Policy:          policy,
+	}
+	mock.lockPutBucketPolicy.Lock()
+	mock.calls.PutBucketPolicy = append(mock.calls.PutBucketPolicy, callInfo)
+	mock.lockPutBucketPolicy.Unlock()
+	return mock.PutBucketPolicyFunc(contextMoqParam, bucket, policy)
+}
+
+// PutBucketPolicyCalls gets all the calls that were made to PutBucketPolicy.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketPolicyCalls())
+func (mock *BackendMock) PutBucketPolicyCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Policy          []byte
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Policy          []byte
+	}
+	mock.lockPutBucketPolicy.RLock()
+	calls = mock.calls.PutBucketPolicy
+	mock.lockPutBucketPolicy.RUnlock()
+	return calls
+}
+
 // PutBucketTagging calls PutBucketTaggingFunc.
 func (mock *BackendMock) PutBucketTagging(contextMoqParam context.Context, bucket string, tags map[string]string) error {
 	if mock.PutBucketTaggingFunc == nil {

s3api/controllers/base.go

@@ -80,7 +80,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectTaggingAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -139,7 +147,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			}
 		}
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.ListMultipartUploadPartsAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -169,7 +185,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionReadAcp,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectAclAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -191,7 +215,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectAttributesAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -218,7 +250,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.GetObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -341,7 +381,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketTaggingAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -378,7 +425,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketVersioningAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -406,8 +460,42 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("policy") {
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketPolicyAction,
+		})
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "GetBucketPolicy",
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		data, err := c.be.GetBucketPolicy(ctx.Context(), bucket)
+		return SendXMLResponse(ctx, data, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				Action:      "GetBucketPolicy",
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	if ctx.Request().URI().QueryArgs().Has("versions") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.ListBucketVersionsAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -444,7 +532,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ_ACP", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionReadAcp,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.GetBucketAclAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -470,7 +565,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("uploads") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.ListBucketMultipartUploadsAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -505,7 +607,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.QueryInt("list-type") == 2 {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.ListBucketAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -540,7 +649,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.ListBucketAction,
+	})
 	if err != nil {
 		return SendXMLResponse(ctx, nil, err,
 			&MetaOpts{
@@ -595,7 +711,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
-		var bucketTagging s3response.Tagging
+		var bucketTagging s3response.TaggingInput
 		err := xml.Unmarshal(ctx.Body(), &bucketTagging)
 		if err != nil {
 			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest),
@@ -620,7 +736,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			tags[tag.Key] = tag.Value
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -641,7 +764,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 
 	if ctx.Request().URI().QueryArgs().Has("versioning") {
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketVersioningAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -676,6 +806,45 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 			})
 	}
 
+	if ctx.Request().URI().QueryArgs().Has("policy") {
+		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketPolicyAction,
+		})
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "PutBucketPolicy",
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = auth.ValidatePolicyDocument(ctx.Body(), bucket, c.iam)
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "PutBucketPolicy",
+					BucketOwner: parsedAcl.Owner,
+				},
+			)
+		}
+
+		err = c.be.PutBucketPolicy(ctx.Context(), bucket, ctx.Body())
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				Action:      "PutBucketPolicy",
+				BucketOwner: parsedAcl.Owner,
+			})
+	}
+
 	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
 
 	if ctx.Request().URI().QueryArgs().Has("acl") {
@@ -683,7 +852,14 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 		var accessControlPolicy auth.AccessControlPolicy
 
 		parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE_ACP", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWriteAcp,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketAclAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -803,9 +979,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error {
 	}
 
 	defACL := auth.ACL{
-		ACL:      "private",
-		Owner:    acct.Access,
-		Grantees: []auth.Grantee{},
+		Owner: acct.Access,
 	}
 
 	updAcl, err := auth.UpdateACL(&s3.PutBucketAclInput{
@@ -868,6 +1042,9 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 
 	// Other headers
 	contentLengthStr := ctx.Get("Content-Length")
+	if contentLengthStr == "" {
+		contentLengthStr = "0"
+	}
 	bucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner")
 
 	grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
@@ -881,7 +1058,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		var objTagging s3response.Tagging
+		var objTagging s3response.TaggingInput
 		err := xml.Unmarshal(ctx.Body(), &objTagging)
 		if err != nil {
 			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest),
@@ -906,7 +1083,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			tags[tag.Key] = tag.Value
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutBucketTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -940,6 +1125,24 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				})
 		}
 
+		err := auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutObjectAction,
+		})
+		if err != nil {
+			return SendXMLResponse(ctx, nil, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "UploadPartCopy",
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
 		resp, err := c.be.UploadPartCopy(ctx.Context(), &s3.UploadPartCopyInput{
 			Bucket:              &bucket,
 			Key:                 &keyStart,
@@ -969,7 +1172,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutObjectAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1108,7 +1319,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if copySource != "" {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        keyStart,
+			Action:        auth.PutObjectAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -1181,7 +1400,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 
 	metadata := utils.GetUserMetaData(&ctx.Request().Header)
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        keyStart,
+		Action:        auth.PutObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1237,7 +1464,14 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.PutBucketTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1257,7 +1491,42 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	if ctx.Request().URI().QueryArgs().Has("policy") {
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Action:        auth.DeleteBucketPolicyAction,
+		})
+		if err != nil {
+			return SendResponse(ctx, err,
+				&MetaOpts{
+					Logger:      c.logger,
+					Action:      "DeleteBucketPolicy",
+					BucketOwner: parsedAcl.Owner,
+				})
+		}
+
+		err = c.be.DeleteBucketPolicy(ctx.Context(), bucket)
+		return SendResponse(ctx, err,
+			&MetaOpts{
+				Logger:      c.logger,
+				Action:      "DeleteBucketPolicy",
+				BucketOwner: parsedAcl.Owner,
+				Status:      http.StatusNoContent,
+			})
+	}
+
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.DeleteBucketAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1296,7 +1565,14 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.DeleteObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1336,12 +1612,20 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("tagging") {
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.DeleteObjectTaggingAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
 					Logger:      c.logger,
-					Action:      "RemoveObjectTagging",
+					Action:      "DeleteObjectTagging",
 					BucketOwner: parsedAcl.Owner,
 				})
 		}
@@ -1361,7 +1645,15 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 		expectedBucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner")
 		requestPayer := ctx.Get("X-Amz-Request-Payer")
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.AbortMultipartUploadAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1388,7 +1680,15 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.DeleteObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1421,7 +1721,14 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error {
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Action:        auth.ListBucketAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1458,7 +1765,15 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 		key = strings.Join([]string{key, keyEnd}, "/")
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionRead,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.GetObjectAction,
+	})
 	if err != nil {
 		return SendResponse(ctx, err,
 			&MetaOpts{
@@ -1564,7 +1879,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.RestoreObjectAction,
+		})
 		if err != nil {
 			return SendResponse(ctx, err,
 				&MetaOpts{
@@ -1602,7 +1925,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err = auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot)
+		err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionRead,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.GetObjectAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -1644,7 +1975,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 				})
 		}
 
-		err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+		err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+			Acl:           parsedAcl,
+			AclPermission: types.PermissionWrite,
+			IsRoot:        isRoot,
+			Acc:           acct,
+			Bucket:        bucket,
+			Object:        key,
+			Action:        auth.PutObjectAction,
+		})
 		if err != nil {
 			return SendXMLResponse(ctx, nil, err,
 				&MetaOpts{
@@ -1683,7 +2022,15 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			})
 	}
 
-	err := auth.VerifyACL(parsedAcl, acct.Access, "WRITE", isRoot)
+	err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
+		Acl:           parsedAcl,
+		AclPermission: types.PermissionWrite,
+		IsRoot:        isRoot,
+		Acc:           acct,
+		Bucket:        bucket,
+		Object:        key,
+		Action:        auth.PutObjectAction,
+	})
 	if err != nil {
 		return SendXMLResponse(ctx, nil, err,
 			&MetaOpts{
@@ -1788,7 +2135,13 @@ func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *MetaOpts) error {
 
 	var b []byte
 
-	if resp != nil {
+	// Handle already encoded responses(text, json...)
+	encodedResp, ok := resp.([]byte)
+	if ok {
+		b = encodedResp
+	}
+
+	if resp != nil && !ok {
 		if b, err = xml.Marshal(resp); err != nil {
 			return err
 		}
@@ -1818,6 +2171,14 @@ func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *MetaOpts) error {
 		})
 	}
 
+	if ok {
+		if len(b) > 0 {
+			ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(b)))
+		}
+
+		return ctx.Send(b)
+	}
+
 	msglen := len(xmlhdr) + len(b)
 	if msglen > maxXMLBodyLen {
 		log.Printf("XML encoded body len %v exceeds max len %v",

s3api/controllers/base_test.go

@@ -352,6 +352,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
 				return &s3.ListObjectVersionsOutput{}, nil
 			},
+			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+				return []byte{}, nil
+			},
 		},
 	}
 
@@ -468,6 +471,15 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "List-actions-get-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "List-actions-list-object-versions-success",
 			app:  app,
@@ -558,6 +570,19 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</VersioningConfiguration>
 	`
 
+	policyBody := `
+	{
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:GetObject",
+				"Resource": "arn:aws:s3:::my-bucket/*"
+			}
+		]
+	}
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -575,6 +600,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
 				return nil
 			},
+			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
+				return nil
+			},
 		},
 	}
 	// Mock ctx.Locals
@@ -651,6 +679,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 200,
 		},
+		{
+			name: "Put-bucket-policy-invalid-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", strings.NewReader(policyBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Put-bucket-acl-invalid-acl",
 			app:  app,
@@ -1046,8 +1092,8 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
-				return s3response.DeleteObjectsResult{}, nil
+			DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+				return s3response.DeleteResult{}, nil
 			},
 		},
 	}

s3api/middlewares/acl-parser.go

@@ -47,7 +47,8 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 			ctx.Method() == http.MethodPut &&
 			!ctx.Request().URI().QueryArgs().Has("acl") &&
 			!ctx.Request().URI().QueryArgs().Has("tagging") &&
-			!ctx.Request().URI().QueryArgs().Has("versioning") {
+			!ctx.Request().URI().QueryArgs().Has("versioning") &&
+			!ctx.Request().URI().QueryArgs().Has("policy") {
 			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}

s3api/utils/auth_test.go

@@ -84,7 +84,7 @@ func Test_Client_UserAgent(t *testing.T) {
 		}
 
 		req.Host = host
-		req.Header.Add("X-Amz-Content-Sha256", zeroLenSig)
+		req.Header.Set("X-Amz-Content-Sha256", zeroLenSig)
 
 		signer := v4.NewSigner()
 

s3api/utils/utils.go

@@ -20,11 +20,13 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/aws/smithy-go/encoding/httpbinding"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/s3err"
@@ -73,6 +75,13 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		}
 	})
 
+	// make sure all headers in the signed headers are present
+	for _, header := range signedHdrs {
+		if httpReq.Header.Get(header) == "" {
+			httpReq.Header.Set(header, "")
+		}
+	}
+
 	// Check if Content-Length in signed headers
 	// If content length is non 0, then the header will be included
 	if !includeHeader("Content-Length", signedHdrs) {
@@ -107,16 +116,18 @@ func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, cont
 	}
 
 	uri := string(ctx.Request().URI().Path())
+	uri = httpbinding.EscapePath(uri, false)
 	isFirst := true
 
 	ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
 		_, ok := signedQueryArgs[string(key)]
 		if !ok {
+			escapeValue := url.QueryEscape(string(value))
 			if isFirst {
-				uri += fmt.Sprintf("?%s=%s", key, value)
+				uri += fmt.Sprintf("?%s=%s", key, escapeValue)
 				isFirst = false
 			} else {
-				uri += fmt.Sprintf("&%s=%s", key, value)
+				uri += fmt.Sprintf("&%s=%s", key, escapeValue)
 			}
 		}
 	})

s3api/utils/utils_test.go

@@ -35,6 +35,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 		args    args
 		want    *http.Request
 		wantErr bool
+		hdrs    []string
 	}{
 		{
 			name: "Success-response",
@@ -43,6 +44,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request,
 			wantErr: false,
+			hdrs:    []string{},
 		},
 		{
 			name: "Success-response-With-Headers",
@@ -51,11 +53,12 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request2,
 			wantErr: false,
+			hdrs:    []string{"X-Amz-Mfa"},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
+			got, err := createHttpRequestFromCtx(tt.args.ctx, tt.hdrs, 0)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
 				return

s3response/s3response.go

@@ -112,11 +112,15 @@ type Tagging struct {
 	TagSet  TagSet   `xml:"TagSet"`
 }
 
+type TaggingInput struct {
+	TagSet TagSet `xml:"TagSet"`
+}
+
 type DeleteObjects struct {
 	Objects []types.ObjectIdentifier `xml:"Object"`
 }
 
-type DeleteObjectsResult struct {
+type DeleteResult struct {
 	Deleted []types.DeletedObject
 	Error   []types.Error
 }

s3select/arch32.go

@@ -0,0 +1,24 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !amd64 && !arm64 && !ppc64le && !riscv64
+// +build !amd64,!arm64,!ppc64le,!riscv64
+
+package s3select
+
+import "math"
+
+const (
+	maxMessageSize = math.MaxInt
+)

s3select/arch64.go

@@ -0,0 +1,22 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build amd64 || arm64 || ppc64le || riscv64
+// +build amd64 arm64 ppc64le riscv64
+
+package s3select
+
+const (
+	maxMessageSize = 5 * 1024 * 1024 * 1024
+)

s3select/message-handler.go

@@ -123,8 +123,7 @@ func generatePrelude(msgLen int, headerLen int) []byte {
 }
 
 const (
-	maxHeaderSize  = 1024 * 1024
-	maxMessageSize = 5 * 1024 * 1024 * 1024
+	maxHeaderSize = 1024 * 1024
 )
 
 func genMessage(header, payload []byte) []byte {

tests/.env.default

@@ -1,6 +1,7 @@
 AWS_PROFILE=versity
 AWS_ENDPOINT_URL=https://127.0.0.1:7070
 VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
 BACKEND=posix
 LOCAL_FOLDER=/tmp/gw
 BUCKET_ONE_NAME=versity-gwtest-bucket-one
@@ -8,4 +9,6 @@ BUCKET_TWO_NAME=versity-gwtest-bucket-two
 #RECREATE_BUCKETS=true
 CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
-S3CMD_CONFIG=./tests/s3cfg.local.default
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets
+MC_ALIAS=versity

tests/.env.s3.default

@@ -0,0 +1,14 @@
+AWS_PROFILE=versity_s3
+AWS_ENDPOINT_URL=https://127.0.0.1:7070
+VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
+BACKEND=s3
+LOCAL_FOLDER=/tmp/gw
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+#RECREATE_BUCKETS=true
+CERT=$PWD/cert.pem
+KEY=$PWD/versitygw.pem
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets.s3
+MC_ALIAS=versity

tests/.env.static

@@ -1,8 +0,0 @@
-AWS_PROFILE=versity
-AWS_ENDPOINT_URL=http://127.0.0.1:7070
-VERSITY_EXE=./versitygw
-BACKEND=posix
-LOCAL_FOLDER=/tmp/gw
-BUCKET_ONE_NAME=versity-gwtest-bucket-one-static
-BUCKET_TWO_NAME=versity-gwtest-bucket-two-static
-RECREATE_BUCKETS=false

tests/README.md

@@ -3,7 +3,10 @@
 ## Instructions - Running Locally
 
 1. Build the `versitygw` binary.
-2. Install the aws command-line interface if unavailable on your machine.  Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
+2. Install the command-line interface(s) you want to test if unavailable on your machine.  
+   * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
+   * **s3cmd**:  Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
+   * **mc**:  Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
 3. Install BATS.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
 4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
 5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
@@ -22,8 +25,8 @@
     openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
     openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
 ```
-8.  Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
-9. In the root repo folder, run with `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
+9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
 ## Instructions - Running With Docker
 

tests/integration/group-tests.go

@@ -43,6 +43,7 @@ func TestPresignedAuthentication(s *S3Conf) {
 	PresignedAuth_incorrect_secret_key(s)
 	PresignedAuth_PutObject_success(s)
 	PresignedAuth_Put_GetObject_with_data(s)
+	PresignedAuth_Put_GetObject_with_UTF8_chars(s)
 	PresignedAuth_UploadPart(s)
 }
 
@@ -241,6 +242,41 @@ func TestGetBucketAcl(s *S3Conf) {
 	GetBucketAcl_success(s)
 }
 
+func TestPutBucketPolicy(s *S3Conf) {
+	PutBucketPolicy_non_existing_bucket(s)
+	PutBucketPolicy_invalid_effect(s)
+	PutBucketPolicy_empty_actions_string(s)
+	PutBucketPolicy_empty_actions_array(s)
+	PutBucketPolicy_invalid_action(s)
+	PutBucketPolicy_unsupported_action(s)
+	PutBucketPolicy_incorrect_action_wildcard_usage(s)
+	PutBucketPolicy_empty_principals_string(s)
+	PutBucketPolicy_empty_principals_array(s)
+	PutBucketPolicy_principals_incorrect_wildcard_usage(s)
+	PutBucketPolicy_non_existing_principals(s)
+	PutBucketPolicy_empty_resources_string(s)
+	PutBucketPolicy_empty_resources_array(s)
+	PutBucketPolicy_invalid_resource_prefix(s)
+	PutBucketPolicy_invalid_resource_with_starting_slash(s)
+	PutBucketPolicy_duplicate_resource(s)
+	PutBucketPolicy_incorrect_bucket_name(s)
+	PutBucketPolicy_object_action_on_bucket_resource(s)
+	PutBucketPolicy_bucket_action_on_object_resource(s)
+	PutBucketPolicy_success(s)
+}
+
+func TestGetBucketPolicy(s *S3Conf) {
+	GetBucketPolicy_non_existing_bucket(s)
+	GetBucketPolicy_default_empty_policy(s)
+	GetBucketPolicy_success(s)
+}
+
+func TestDeleteBucketPolicy(s *S3Conf) {
+	DeleteBucketPolicy_non_existing_bucket(s)
+	DeleteBucketPolicy_remove_before_setting(s)
+	DeleteBucketPolicy_success(s)
+}
+
 func TestFullFlow(s *S3Conf) {
 	TestAuthentication(s)
 	TestPresignedAuthentication(s)
@@ -270,6 +306,9 @@ func TestFullFlow(s *S3Conf) {
 	TestCompleteMultipartUpload(s)
 	TestPutBucketAcl(s)
 	TestGetBucketAcl(s)
+	TestPutBucketPolicy(s)
+	TestGetBucketPolicy(s)
+	TestDeleteBucketPolicy(s)
 }
 
 func TestPosix(s *S3Conf) {
@@ -329,6 +368,7 @@ func GetIntTests() IntTests {
 		"PresignedAuth_incorrect_secret_key":                    PresignedAuth_incorrect_secret_key,
 		"PresignedAuth_PutObject_success":                       PresignedAuth_PutObject_success,
 		"PresignedAuth_Put_GetObject_with_data":                 PresignedAuth_Put_GetObject_with_data,
+		"PresignedAuth_Put_GetObject_with_UTF8_chars":           PresignedAuth_Put_GetObject_with_UTF8_chars,
 		"PresignedAuth_UploadPart":                              PresignedAuth_UploadPart,
 		"CreateBucket_invalid_bucket_name":                      CreateBucket_invalid_bucket_name,
 		"CreateBucket_existing_bucket":                          CreateBucket_existing_bucket,
@@ -443,6 +483,32 @@ func GetIntTests() IntTests {
 		"GetBucketAcl_non_existing_bucket":                      GetBucketAcl_non_existing_bucket,
 		"GetBucketAcl_access_denied":                            GetBucketAcl_access_denied,
 		"GetBucketAcl_success":                                  GetBucketAcl_success,
+		"PutBucketPolicy_non_existing_bucket":                   PutBucketPolicy_non_existing_bucket,
+		"PutBucketPolicy_invalid_effect":                        PutBucketPolicy_invalid_effect,
+		"PutBucketPolicy_empty_actions_string":                  PutBucketPolicy_empty_actions_string,
+		"PutBucketPolicy_empty_actions_array":                   PutBucketPolicy_empty_actions_array,
+		"PutBucketPolicy_invalid_action":                        PutBucketPolicy_invalid_action,
+		"PutBucketPolicy_unsupported_action":                    PutBucketPolicy_unsupported_action,
+		"PutBucketPolicy_incorrect_action_wildcard_usage":       PutBucketPolicy_incorrect_action_wildcard_usage,
+		"PutBucketPolicy_empty_principals_string":               PutBucketPolicy_empty_principals_string,
+		"PutBucketPolicy_empty_principals_array":                PutBucketPolicy_empty_principals_array,
+		"PutBucketPolicy_principals_incorrect_wildcard_usage":   PutBucketPolicy_principals_incorrect_wildcard_usage,
+		"PutBucketPolicy_non_existing_principals":               PutBucketPolicy_non_existing_principals,
+		"PutBucketPolicy_empty_resources_string":                PutBucketPolicy_empty_resources_string,
+		"PutBucketPolicy_empty_resources_array":                 PutBucketPolicy_empty_resources_array,
+		"PutBucketPolicy_invalid_resource_prefix":               PutBucketPolicy_invalid_resource_prefix,
+		"PutBucketPolicy_invalid_resource_with_starting_slash":  PutBucketPolicy_invalid_resource_with_starting_slash,
+		"PutBucketPolicy_duplicate_resource":                    PutBucketPolicy_duplicate_resource,
+		"PutBucketPolicy_incorrect_bucket_name":                 PutBucketPolicy_incorrect_bucket_name,
+		"PutBucketPolicy_object_action_on_bucket_resource":      PutBucketPolicy_object_action_on_bucket_resource,
+		"PutBucketPolicy_bucket_action_on_object_resource":      PutBucketPolicy_bucket_action_on_object_resource,
+		"PutBucketPolicy_success":                               PutBucketPolicy_success,
+		"GetBucketPolicy_non_existing_bucket":                   GetBucketPolicy_non_existing_bucket,
+		"GetBucketPolicy_default_empty_policy":                  GetBucketPolicy_default_empty_policy,
+		"GetBucketPolicy_success":                               GetBucketPolicy_success,
+		"DeleteBucketPolicy_non_existing_bucket":                DeleteBucketPolicy_non_existing_bucket,
+		"DeleteBucketPolicy_remove_before_setting":              DeleteBucketPolicy_remove_before_setting,
+		"DeleteBucketPolicy_success":                            DeleteBucketPolicy_success,
 		"PutObject_overwrite_dir_obj":                           PutObject_overwrite_dir_obj,
 		"PutObject_overwrite_file_obj":                          PutObject_overwrite_file_obj,
 		"PutObject_dir_obj_with_data":                           PutObject_dir_obj_with_data,

tests/integration/tests.go

@@ -1448,8 +1448,6 @@ func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
 			return fmt.Errorf("read get object response body %w", err)
 		}
 
-		fmt.Println(resp.Request.Method, resp.ContentLength, string(respBody))
-
 		if string(respBody) != data {
 			return fmt.Errorf("expected get object response body to be %v, instead got %s", data, respBody)
 		}
@@ -1463,6 +1461,72 @@ func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
 	})
 }
 
+func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error {
+	testName := "PresignedAuth_Put_GetObject_with_data"
+	return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
+		bucket, obj := getBucketName(), "my-$%^&*;"
+		err := setup(s, bucket)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		httpClient := http.Client{
+			Timeout: shortTimeout,
+		}
+
+		req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
+		if err != nil {
+			return err
+		}
+
+		req.Header = v4req.SignedHeader
+
+		resp, err := httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected my-obj to be successfully uploaded and get %v response status, instead got %v", http.StatusOK, resp.StatusCode)
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		v4GetReq, err := client.PresignGetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &obj})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		req, err = http.NewRequest(v4GetReq.Method, v4GetReq.URL, nil)
+		if err != nil {
+			return err
+		}
+
+		resp, err = httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected get object response status to be %v, instead got %v", http.StatusOK, resp.StatusCode)
+		}
+
+		err = teardown(s, bucket)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func PresignedAuth_UploadPart(s *S3Conf) error {
 	testName := "PresignedAuth_UploadPart"
 	return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
@@ -3201,28 +3265,35 @@ func CopyObject_not_owned_source_bucket(s *S3Conf) error {
 		}
 
 		usr := user{
-			access: "admin1",
-			secret: "admin1secret",
-			role:   "admin",
+			access: "grt1",
+			secret: "grt1secret",
+			role:   "user",
 		}
 
 		cfg := *s
 		cfg.awsID = usr.access
 		cfg.awsSecret = usr.secret
 
+		userS3Client := s3.NewFromConfig(cfg.Config())
+
 		err = createUsers(s, []user{usr})
 		if err != nil {
 			return err
 		}
 
 		dstBucket := getBucketName()
-		err = setup(&cfg, dstBucket)
+		err = setup(s, dstBucket)
+		if err != nil {
+			return err
+		}
+
+		err = changeBucketsOwner(s, []string{bucket}, usr.access)
 		if err != nil {
 			return err
 		}
 
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-		_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
+		_, err = userS3Client.CopyObject(ctx, &s3.CopyObjectInput{
 			Bucket:     &dstBucket,
 			Key:        getPtr("obj-1"),
 			CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
@@ -5177,6 +5248,555 @@ func GetBucketAcl_success(s *S3Conf) error {
 	})
 }
 
+func PutBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "PutBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_effect(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_effect"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("invalid_effect", `"*"`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid effect: invalid_effect")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_actions_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_actions_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `""`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_actions_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_actions_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `[]`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_action(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"ListObjects"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid action: ListObjects")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_unsupported_action(s *S3Conf) error {
+	testName := "PutBucketPolicy_unsupported_action"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:PutLifecycleConfiguration"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported action: s3:PutLifecycleConfiguration")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_incorrect_action_wildcard_usage(s *S3Conf) error {
+	testName := "PutBucketPolicy_incorrect_action_wildcard_usage"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `"s3:hello*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("invalid wildcard usage: s3:hello prefix is not in the supported actions list")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_principals_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_principals_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `""`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_principals_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_principals_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `[]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_principals_incorrect_wildcard_usage(s *S3Conf) error {
+	testName := "PutBucketPolicy_principals_incorrect_wildcard_usage"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*", "grt1"]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("principals should either contain * or user access keys")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_non_existing_principals(s *S3Conf) error {
+	testName := "PutBucketPolicy_non_existing_principals"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"]`, `"s3:*"`, `"arn:aws:s3:::*"`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		apiErr1 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"}))
+		apiErr2 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_2", "a_rarely_existing_user_account_1"}))
+
+		err1 := checkApiErr(err, apiErr1)
+		err2 := checkApiErr(err, apiErr2)
+
+		if err1 != nil && err2 != nil {
+			return err1
+		}
+
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_resources_string(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_resources_string"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, `""`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_empty_resources_array(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_resources_array"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, `[]`)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_resource_prefix(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_resource_prefix"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:iam:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_invalid_resource_with_starting_slash(s *S3Conf) error {
+	testName := "PutBucketPolicy_invalid_resource_with_starting_slash"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::/%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_duplicate_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_duplicate_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, fmt.Sprintf("[%v, %v]", resource, resource))
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("duplicate resource: %v", resource[1:len(resource)-1]))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_incorrect_bucket_name(s *S3Conf) error {
+	testName := "PutBucketPolicy_incorrect_bucket_name"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::prefix-%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:*"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("incorrect bucket name in prefix-%v", bucket))); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_object_action_on_bucket_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_object_action_on_bucket_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:PutObjectTagging"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported object action 's3:PutObjectTagging' on the specified resources")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_bucket_action_on_object_resource(s *S3Conf) error {
+	testName := "PutBucketPolicy_object_action_on_bucket_resource"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		resource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+		doc := genPolicyDoc("Allow", `["*"]`, `"s3:DeleteBucket"`, resource)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+
+		if err := checkApiErr(err, getMalformedPolicyError("unsupported bucket action 's3:DeleteBucket' on the specified resources")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func PutBucketPolicy_success(s *S3Conf) error {
+	testName := "PutBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		err := createUsers(s, []user{
+			{"grt1", "grt1secret", "user"},
+			{"grt2", "grt2secret", "user"},
+		})
+		if err != nil {
+			return err
+		}
+
+		bucketResource := fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)
+		objectResource := fmt.Sprintf(`"arn:aws:s3:::%v/*"`, bucket)
+
+		for _, doc := range []string{
+			genPolicyDoc("Allow", `["grt1", "grt2"]`, `["s3:DeleteBucket", "s3:GetBucketAcl"]`, bucketResource),
+			genPolicyDoc("Deny", `"*"`, `"s3:DeleteBucket"`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket)),
+			genPolicyDoc("Allow", `"grt1"`, `["s3:PutBucketVersioning", "s3:ListMultipartUploadParts", "s3:ListBucket"]`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+			genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+			genPolicyDoc("Allow", `"*"`, `"s3:Get*"`, objectResource),
+			genPolicyDoc("Deny", `"*"`, `"s3:Create*"`, fmt.Sprintf(`[%v, %v]`, bucketResource, objectResource)),
+		} {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			_, err = s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+				Bucket: &bucket,
+				Policy: &doc,
+			})
+			cancel()
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func GetBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "GetBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func GetBucketPolicy_default_empty_policy(s *S3Conf) error {
+	testName := "GetBucketPolicy_default_empty_policy"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy != nil {
+			return fmt.Errorf("expected policy to be nil, instead got %s", *out.Policy)
+		}
+
+		return nil
+	})
+}
+
+func GetBucketPolicy_success(s *S3Conf) error {
+	testName := "GetBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `["s3:DeleteBucket", "s3:GetBucketTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy == nil {
+			return fmt.Errorf("expected non nil policy result")
+		}
+
+		if *out.Policy != doc {
+			return fmt.Errorf("expected the bucket policy to be %v, instead got %v", doc, *out.Policy)
+		}
+
+		return nil
+	})
+}
+
+func DeleteBucketPolicy_non_existing_bucket(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_non_existing_bucket"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: getPtr("non_existing_bucket"),
+		})
+		cancel()
+
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func DeleteBucketPolicy_remove_before_setting(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_remove_before_setting"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+
+		return err
+	})
+}
+
+func DeleteBucketPolicy_success(s *S3Conf) error {
+	testName := "DeleteBucketPolicy_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := genPolicyDoc("Allow", `"*"`, `["s3:DeleteBucket", "s3:GetBucketTagging"]`, fmt.Sprintf(`"arn:aws:s3:::%v"`, bucket))
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if out.Policy != nil {
+			return fmt.Errorf("expected policy to be nil, instead got %s", *out.Policy)
+		}
+
+		return err
+	})
+}
+
 // IAM related tests
 // multi-user iam tests
 func IAM_user_access_denied(s *S3Conf) error {

tests/integration/utils.go

@@ -219,12 +219,17 @@ func checkApiErr(err error, apiErr s3err.APIError) error {
 	}
 	var ae smithy.APIError
 	if errors.As(err, &ae) {
-		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
-			return nil
+		if ae.ErrorCode() != apiErr.Code {
+			return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, ae.ErrorCode())
 		}
 
-		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+		if ae.ErrorMessage() != apiErr.Description {
+			return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, ae.ErrorMessage())
+		}
+
+		return nil
 	}
+
 	return fmt.Errorf("expected aws api error, instead got: %w", err)
 }
 
@@ -603,3 +608,28 @@ func changeAuthCred(uri, newVal string, index int) (string, error) {
 
 	return urlParsed.String(), nil
 }
+
+func genPolicyDoc(effect, principal, action, resource string) string {
+	jsonTemplate := `
+	{
+		"Statement": [
+			{
+				"Effect":  "%s",
+				"Principal": %s,
+				"Action":  %s,
+				"Resource":  %s
+			}
+		]
+	}
+	`
+
+	return fmt.Sprintf(jsonTemplate, effect, principal, action, resource)
+}
+
+func getMalformedPolicyError(msg string) s3err.APIError {
+	return s3err.APIError{
+		Code:           "MalformedPolicy",
+		Description:    msg,
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}

tests/run.sh

@@ -1,16 +1,75 @@
 #!/bin/bash
 
+# Function to display help information
+show_help() {
+    echo "Usage: $0 [option...]"
+    echo "   -h, --help          Display this help message and exit"
+    echo "   -s, --static        Don't remove buckets between tests"
+    echo "   aws                 Run tests with aws cli"
+    echo "   aws-posix           Run posix tests with aws cli"
+    echo "   s3cmd               Run tests with s3cmd utility"
+    echo "   mc                  Run tests with mc utility"
+}
+
+handle_param() {
+  case $1 in
+      -h|--help)
+          show_help
+          exit 0
+          ;;
+      -s|--static)
+          export RECREATE_BUCKETS=false
+          ;;
+      aws|aws-posix|s3cmd|mc)
+          set_command_type "$1"
+          ;;
+      *) # Handle unrecognized options or positional arguments
+          echo "Unrecognized option: $1" >&2
+          exit 1
+          ;;
+  esac
+}
+
+set_command_type() {
+  if [[ -n $command_type ]]; then
+    echo "Error:  command type already set"
+    exit 1
+  fi
+  command_type=$1
+  export command_type
+}
+
+export RECREATE_BUCKETS=true
+while [[ "$#" -gt 0 ]]; do
+  handle_param "$1"
+  shift # past argument or value
+done
+
 if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
   echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
   exit 1
 fi
-export RECREATE_BUCKETS=true
-if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then
-  exit 1
+
+if [[ $RECREATE_BUCKETS == false ]]; then
+  ./tests/setup_static.sh || exit_code=$?
+  if [[ exit_code -ne 0 ]]; then
+    exit 1
+  fi
 fi
-if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then
-  exit 1
-fi
-if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then
-  exit 1
-fi
+
+case $command_type in
+  aws)
+    "$HOME"/bin/bats ./tests/test_aws.sh || exit_code=$?
+    ;;
+  aws-posix)
+    "$HOME"/bin/bats ./tests/test_aws_posix.sh || exit_code=$?
+    ;;
+  s3cmd)
+    "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
+    ;;
+  mc)
+    "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
+    ;;
+esac
+
+exit $exit_code

tests/run_all.sh

@@ -4,9 +4,28 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
   echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
   exit 1
 fi
-if ! ./tests/run.sh; then
+if ! ./tests/run.sh aws; then
   exit 1
 fi
-if ! ./tests/run_static.sh; then
+if ! ./tests/run.sh aws-posix; then
   exit 1
-fi
+fi
+if ! ./tests/run.sh s3cmd; then
+  exit 1
+fi
+if ! ./tests/run.sh mc; then
+  exit 1
+fi
+if ! ./tests/run.sh -s aws; then
+  exit 1
+fi
+if ! ./tests/run.sh -s aws-posix; then
+  exit 1
+fi
+if ! ./tests/run.sh -s s3cmd; then
+  exit 1
+fi
+if ! ./tests/run.sh -s mc; then
+  exit 1
+fi
+exit 0

tests/run_static.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
-  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
-  exit 1
-fi
-result=0
-export RECREATE_BUCKETS=false
-./tests/setup_static.sh
-if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then
-  result=1
-fi
-if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then
-  result=1
-fi
-if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then
-  result=1
-fi
-./tests/teardown_static.sh
-exit $result

tests/s3cfg.local.default

@@ -3,6 +3,7 @@ host_base = 127.0.0.1:7070
 host_bucket = 127.0.0.1:7070
 bucket_location = us-east-1
 use_https = True
+signurl_use_https = True
 
 # Enable S3 v4 signature APIs
 signature_v2 = False

tests/s3cmd_tests.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bats
-
-source ./tests/setup.sh
-source ./tests/test_common.sh
-source ./tests/util.sh
-
-# test s3cmd bucket creation/deletion
-@test "test_create_delete_bucket_s3cmd" {
-  test_common_create_delete_bucket "s3cmd"
-}
-
-# test s3cmd put object
-@test "test_put_object_s3cmd" {
-  test_common_put_object "s3cmd"
-}
-
-# test listing buckets on versitygw
-@test "test_list_buckets_s3cmd" {
-  test_common_list_buckets "s3cmd"
-}
-
-@test "test_list_objects_s3cmd" {
-  test_common_list_objects "s3cmd"
-}
-
-@test "test_multipart_upload_s3cmd" {
-
-  bucket_file="largefile"
-
-  create_large_file "$bucket_file" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload"
-
-  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-
-  put_object "s3cmd" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$?
-  [[ $put_result -eq 0 ]] || fail "failed to copy file"
-
-  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
-  delete_test_files $bucket_file
-}

tests/setup.sh

@@ -1,73 +1,47 @@
 #!/usr/bin/env bash
 
+source ./tests/setup_mc.sh
+source ./tests/versity.sh
+
 # bats setup function
 setup() {
-  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then
-    source tests/.secrets
-  else
-    echo "Warning: no secrets file found"
+  start_versity || start_result=$?
+  if [[ $start_result -ne 0 ]]; then
+    echo "error starting versity executable"
+    return 1
   fi
-  if [ -z "$VERSITYGW_TEST_ENV" ]; then
-    if [ -r tests/.env ]; then
-      source tests/.env
-    else
-      echo "Warning: no .env file found in tests folder"
+
+  check_params || check_result=$?
+  if [[ $check_result -ne 0 ]]; then
+    echo "parameter check failed"
+    return 1
+  fi
+
+  if [[ $RUN_S3CMD == true ]]; then
+    S3CMD_OPTS=()
+    S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
+    S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
+    S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
+    export S3CMD_CONFIG S3CMD_OPTS
+  fi
+
+  if [[ $RUN_MC == true ]]; then
+    check_add_mc_alias || check_result=$?
+    if [[ $check_result -ne 0 ]]; then
+      echo "mc alias check/add failed"
+      return 1
     fi
-  else
-    # shellcheck source=./.env.default
-    source "$VERSITYGW_TEST_ENV"
   fi
 
-  check_params
-
-  base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE"
-  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
-    base_command+=" --cert $CERT --key $KEY"
-  fi
-  base_command+=" $BACKEND $LOCAL_FOLDER &"
-  eval "$base_command"
-
-  versitygw_pid=$!
-  S3CMD_OPTS=()
-  S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
-  S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
-  S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
-  export versitygw_pid \
-    AWS_PROFILE \
-    AWS_ENDPOINT_URL \
-    LOCAL_FOLDER \
+  export AWS_PROFILE \
     BUCKET_ONE_NAME \
-    BUCKET_TWO_NAME \
-    S3CMD_CONFIG \
-    S3CMD_OPTS \
-    RECREATE_BUCKETS
+    BUCKET_TWO_NAME
 }
 
-# make sure required environment variables are defined properly
+# make sure required environment variables for tests are defined properly
 # return 0 for yes, 1 for no
 check_params() {
-  if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-    echo "No AWS access key set"
-    return 1
-  elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
-    echo "No AWS secret access key set"
-    return 1
-  elif [ -z "$VERSITY_EXE" ]; then
-    echo "No versity executable location set"
-    return 1
-  elif [ -z "$BACKEND" ]; then
-    echo "No backend parameter set (options: 'posix')"
-    return 1
-  elif [ -z "$AWS_PROFILE" ]; then
-    echo "No AWS profile set"
-    return 1
-  elif [ -z "$LOCAL_FOLDER" ]; then
-    echo "No local storage folder set"
-    return 1
-  elif [ -z "$AWS_ENDPOINT_URL" ]; then
-    echo "No AWS endpoint URL set"
-    return 1
-  elif [ -z "$BUCKET_ONE_NAME" ]; then
+  if [ -z "$BUCKET_ONE_NAME" ]; then
     echo "No bucket one name set"
     return 1
   elif [ -z "$BUCKET_TWO_NAME" ]; then
@@ -80,6 +54,7 @@ check_params() {
     echo "RECREATE_BUCKETS must be 'true' or 'false'"
     return 1
   fi
+  return 0
 }
 
 # fail a test
@@ -91,14 +66,5 @@ fail() {
 
 # bats teardown function
 teardown() {
-  if [ -n "$versitygw_pid" ]; then
-    if ps -p "$versitygw_pid" > /dev/null; then
-      kill "$versitygw_pid"
-      wait "$versitygw_pid" || true
-    else
-      echo "Process with PID $versitygw_pid does not exist."
-    fi
-  else
-    echo "versitygw_pid is not set or empty."
-  fi
+  stop_versity
 }

tests/setup_mc.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+check_for_alias() {
+  local alias_result
+  aliases=$(mc alias list)
+  if [[ $alias_result -ne 0 ]]; then
+    echo "error checking for aliases: $aliases"
+    return 2
+  fi
+  while IFS= read -r line; do
+    error=$(echo "$line" | grep -w "$MC_ALIAS ")
+    if [[ $? -eq 0 ]]; then
+      return 0
+    fi
+  done <<< "$aliases"
+  return 1
+}
+
+check_add_mc_alias() {
+  check_for_alias || alias_result=$?
+  if [[ $alias_result -eq 2 ]]; then
+    echo "error checking for aliases"
+    return 1
+  fi
+  if [[ $alias_result -eq 0 ]]; then
+    return 0
+  fi
+  local set_result
+  error=$(mc alias set --insecure "$MC_ALIAS" "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$?
+  if [[ $set_result -ne 0 ]]; then
+    echo "error setting alias: $error"
+    return 1
+  fi
+  return 0
+}

tests/test_aws.sh

@@ -11,8 +11,12 @@ source ./tests/test_common.sh
 }
 
 # test adding and removing an object on versitygw
-@test "test_put_object" {
-  test_common_put_object "aws"
+@test "test_put_object-with-data" {
+  test_common_put_object_with_data "aws"
+}
+
+@test "test_put_object-no-data" {
+  test_common_put_object_no_data "aws"
 }
 
 # test listing buckets on versitygw
@@ -97,27 +101,7 @@ source ./tests/test_common.sh
 
 # test abilty to set and retrieve bucket tags
 @test "test-set-get-bucket-tags" {
-
-  local key="test_key"
-  local value="test_value"
-
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-
-  get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
-  tag_set=$(echo "$tags" | jq '.TagSet')
-  [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
-
-  put_bucket_tag "$BUCKET_ONE_NAME" $key $value
-  get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$?
-  [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
-  tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
-  tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
-  [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
-  [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  test_common_set_get_bucket_tags "aws"
 }
 
 # test v1 s3api list objects command
@@ -184,34 +168,7 @@ source ./tests/test_common.sh
 
 # test abilty to set and retrieve object tags
 @test "test-set-get-object-tags" {
-
-  local bucket_file="bucket-file"
-  local key="test_key"
-  local value="test_value"
-
-  create_test_files "$bucket_file" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating test files"
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-  local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
-  put_object "aws" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
-  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
-
-  get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting object tags"
-  tag_set=$(echo "$tags" | jq '.TagSet')
-  [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
-
-  put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value
-  get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
-  [[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
-  tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
-  tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
-  [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
-  [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
-  delete_test_files $bucket_file
+  test_common_set_get_object_tags "aws"
 }
 
 # test multi-part upload
@@ -358,3 +315,7 @@ source ./tests/test_common.sh
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
   delete_test_files $bucket_file
 }
+
+@test "test-presigned-url-utf8-chars" {
+  test_common_presigned_url_utf8_chars "aws"
+}