Merge pull request #464 from versity/ben/presign_escape

fix: escape path and query for presign signature validation

.github/workflows/docker.yaml

@@ -0,0 +1,45 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registries:
+    name: Push Docker image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            versity/versitygw
+            ghcr.io/${{ github.repository }}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/system.yml

@@ -1,19 +1,19 @@
 name: system tests
 on: pull_request
-#on:
-#  workflow_dispatch:
-#    inputs:
-#      run_workflow:
-#        description: 'Run command-line tests'
 jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
     steps:
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
 
+      - name: Install ShellCheck
+        run: sudo apt-get install shellcheck
+
+      - name: Run ShellCheck
+        run: shellcheck -S warning ./tests/*.sh
+
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
@@ -29,14 +29,31 @@ jobs:
           git clone https://github.com/bats-core/bats-core.git
           cd bats-core && ./install.sh $HOME
 
-      - name: Build and Run
+      - name: Install s3cmd
+        run: |
+          sudo apt-get install s3cmd
+
+      - name: Install mc
+        run: |
+          curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
+          chmod 755 /usr/local/bin/mc
+
+      - name: Build and run
         run: |
           make testbin
-          export AWS_ACCESS_KEY_ID=user
-          export AWS_SECRET_ACCESS_KEY=pass
-          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity 
+          export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+          export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+          export AWS_REGION=us-east-1
+          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
-          export VERSITY_EXE=./versitygw
+          aws configure set aws_region $AWS_REGION --profile versity
           mkdir /tmp/gw
-          VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/s3_bucket_tests.sh
-          VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/posix_tests.sh
+          export WORKSPACE=$GITHUB_WORKSPACE
+          openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          mkdir /tmp/cover
+          VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+
+      - name: Coverage report
+        run: |
+          go tool covdata percent -i=/tmp/cover

.gitignore

@@ -41,7 +41,10 @@ VERSION
 dist/
 
 # secrets file for local github-actions testing
-.secrets
+tests/.secrets*
+
+# IAM users files often created in testing
+users.json
 
 # env files for testing
 .env*

.goreleaser.yaml

@@ -6,6 +6,7 @@ builds:
   - goos:
       - linux
       - darwin
+      - freebsd
       # windows is untested, we can start doing windows releases
       # if someone is interested in taking on testing
       # - windows
@@ -14,8 +15,7 @@ builds:
       # once we need to enable this, we will need to do per distro releases
       - CGO_ENABLED=0
     main: ./cmd/versitygw
-    binary: ./cmd/versitygw
-    id: versitygw
+    binary: versitygw
     goarch:
       - amd64
       - arm64
@@ -26,17 +26,34 @@ archives:
   - format: tar.gz
     # this name template makes the OS and Arch compatible with the results of uname.
     name_template: >-
-      {{ .ProjectName }}_
+      {{ .ProjectName }}_v{{ .Version }}_
       {{- title .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
+
+    # Set this to true if you want all files in the archive to be in a single directory.
+    # If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
+    # you'll get a folder 'goreleaser_Linux_arm64'.
+    # If set to false, all files are extracted separately.
+    # You can also set it to a custom folder name (templating is supported).
+    wrap_in_directory: true
+
     # use zip for windows archives
     format_overrides:
     - goos: windows
       format: zip
 
+    # Additional files/globs you want to add to the archive.
+    #
+    # Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
+    # Templates: allowed
+    files:
+      - README.md
+      - LICENSE
+      - NOTICE
+
 checksum:
   name_template: 'checksums.txt'
 
@@ -51,5 +68,59 @@ changelog:
       - '^test:'
       - '^Merge '
 
+nfpms:
+  - id: packages
+    package_name: versitygw
+    vendor: Versity Software
+    homepage: https://github.com/versity/versitygw
+    maintainer: Ben McClelland <ben.mcclelland@versity.com>
+
+    description: |-
+      The Versity S3 Gateway.
+      A high-performance tool facilitating translation between AWS S3 API
+      requests and various backend storage systems, including POSIX file
+      backend storage. Its stateless architecture enables deployment in
+      clusters for increased throughput, distributing requests across gateways
+      for optimal performance. With a focus on modularity, it supports future
+      extensions for additional backend systems.
+
+    license: Apache 2.0
+
+    builds:
+      - versitygw
+
+    formats:
+      - deb
+      - rpm
+
+    umask: 0o002
+    bindir: /usr/bin
+    epoch: "1"
+    release: "1"
+
+    rpm:
+      group: "System Environment/Daemons"
+      # RPM specific scripts.
+      scripts:
+        # The pretrans script runs before all RPM package transactions / stages.
+        #pretrans: ./extra/pretrans.sh
+        # The posttrans script runs after all RPM package transactions / stages.
+        posttrans: ./extra/posttrans.sh
+
+    contents:
+      - src: extra/versitygw@.service
+        dst: /lib/systemd/system/versitygw@.service
+
+      - src: extra/example.conf
+        dst: /etc/versitygw.d/example.conf
+        type: config
+  
+      - dst: /etc/versitygw.d
+        type: dir
+        file_info:
+          mode: 0700
+
+
+
 # yaml-language-server: $schema=https://goreleaser.com/static/schema.json
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine
+FROM golang:latest
 
 WORKDIR /app
 
@@ -8,6 +8,7 @@ RUN go mod download
 COPY ./ ./
 
 WORKDIR /app/cmd/versitygw
+ENV CGO_ENABLED=0
 RUN go build -o versitygw
 
 FROM alpine:latest

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.20
+FROM golang:latest
 
 WORKDIR /app
 
@@ -6,6 +6,7 @@ COPY go.mod ./
 RUN go mod download
 
 COPY ./ ./
+COPY certs/* /etc/pki/tls/certs/
 
 ARG IAM_DIR=/tmp/vgw
 ARG SETUP_DIR=/tmp/vgw
@@ -14,4 +15,4 @@ RUN mkdir -p $IAM_DIR
 RUN mkdir -p $SETUP_DIR
 
 RUN go get github.com/githubnemo/CompileDaemon
-RUN go install github.com/githubnemo/CompileDaemon
+RUN go install github.com/githubnemo/CompileDaemon

Dockerfile_test_bats

@@ -0,0 +1,79 @@
+FROM --platform=linux/arm64 ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    git \
+    make \
+    wget \
+    curl \
+    unzip \
+    tzdata \
+    s3cmd \
+    jq \
+    ca-certificates && \
+    update-ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /tmp
+
+# Install AWS cli
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+
+# Install mc
+RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
+  --create-dirs \
+  -o /usr/local/minio-binaries/mc && \
+chmod -R 755 /usr/local/minio-binaries
+ENV PATH="/usr/local/minio-binaries":${PATH}
+
+# Download Go 1.21 (adjust the version and platform as needed)
+RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
+
+# Extract the downloaded archive
+RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
+
+# Set Go environment variables
+ENV PATH="/usr/local/go/bin:${PATH}"
+ENV GOPATH="/go"
+ENV GOBIN="$GOPATH/bin"
+
+# Make the directory for Go packages
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+# Create tester user
+RUN groupadd -r tester && useradd -r -g tester tester
+RUN mkdir /home/tester && chown tester:tester /home/tester
+ENV HOME=/home/tester
+
+# install bats
+RUN git clone https://github.com/bats-core/bats-core.git && \
+    cd bats-core && \
+    ./install.sh /home/tester
+
+USER tester
+COPY --chown=tester:tester . /home/tester
+
+WORKDIR /home/tester
+RUN cp tests/.env.docker.default tests/.env.docker
+RUN cp tests/s3cfg.local.default tests/s3cfg.local
+RUN make
+
+RUN . tests/.secrets && \
+    export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
+    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
+    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
+    aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
+
+RUN mkdir /tmp/gw
+
+RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen_bits:2048 && \
+    openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
+        -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+
+ENV WORKSPACE=.
+ENV VERSITYGW_TEST_ENV=tests/.env.docker
+
+CMD ["tests/run_all.sh"]

Makefile

@@ -59,20 +59,13 @@ cleanall: clean
 	rm -f $(BIN)
 	rm -f versitygw-*.tar
 	rm -f versitygw-*.tar.gz
-	rm -f versitygw.spec
-
-%.spec: %.spec.in
-	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
-	mv $@+ $@
 
 TARFILE = $(BIN)-$(VERSION).tar
 
-dist: $(BIN).spec
+dist:
 	echo $(VERSION) >VERSION
 	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
-	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
 	rm -f VERSION
-	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
 
 # Creates and runs S3 gateway instance in a docker container

README.md

@@ -8,8 +8,6 @@
 
  [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-**Current status:** Ready for general testing, Issue reports welcome.
-
 **News:**<br>
 * New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
 

auth/acl.go

@@ -99,34 +99,34 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		grantees := []Grantee{}
 		accs := []string{}
 
-		if input.GrantRead != nil {
+		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
 			fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
 
-			if *input.GrantFullControl != "" {
+			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
 					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
 				}
 			}
-			if *input.GrantRead != "" {
+			if input.GrantRead != nil && *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
 					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
 				}
 			}
-			if *input.GrantReadACP != "" {
+			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
 					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
 				}
 			}
-			if *input.GrantWrite != "" {
+			if input.GrantWrite != nil && *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
 					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
 				}
 			}
-			if *input.GrantWriteACP != "" {
+			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
 					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
@@ -137,6 +137,9 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
 		} else {
 			cache := make(map[string]bool)
 			for _, grt := range input.AccessControlPolicy.Grants {
+				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
+					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
+				}
 				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
 				if _, ok := cache[*grt.Grantee.ID]; !ok {
 					cache[*grt.Grantee.ID] = true
@@ -239,14 +242,34 @@ func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool)
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
-func IsAdmin(acct Account, isRoot bool) error {
+func MayCreateBucket(acct Account, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
 
-	if acct.Role == "admin" {
+	if acct.Role == RoleUser {
+		return s3err.GetAPIError(s3err.ErrAccessDenied)
+	}
+
+	return nil
+}
+
+func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
+	// Owner check
+	if acct.Access == acl.Owner {
 		return nil
 	}
 
+	// Root user has access over almost everything
+	if isRoot {
+		return nil
+	}
+
+	// Admin user case
+	if acct.Role == RoleAdmin {
+		return nil
+	}
+
+	// Return access denied in all other cases
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }

auth/iam.go

@@ -20,11 +20,19 @@ import (
 	"time"
 )
 
+type Role string
+
+const (
+	RoleUser     Role = "user"
+	RoleAdmin    Role = "admin"
+	RoleUserPlus Role = "userplus"
+)
+
 // Account is a gateway IAM account
 type Account struct {
 	Access    string `json:"access"`
 	Secret    string `json:"secret"`
-	Role      string `json:"role"`
+	Role      Role   `json:"role"`
 	UserID    int    `json:"userID"`
 	GroupID   int    `json:"groupID"`
 	ProjectID int    `json:"projectID"`

auth/iam_cache.go

@@ -130,7 +130,7 @@ func (c *IAMCache) CreateAccount(account Account) error {
 	acct := Account{
 		Access: strings.Clone(account.Access),
 		Secret: strings.Clone(account.Secret),
-		Role:   strings.Clone(account.Role),
+		Role:   Role(strings.Clone(string(account.Role))),
 	}
 
 	c.iamcache.set(acct.Access, acct)

auth/iam_ldap.go

@@ -46,7 +46,7 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
-	userEntry.Attribute(ld.roleAtr, []string{account.Role})
+	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -78,7 +78,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 	return Account{
 		Access: entry.GetAttributeValue(ld.accessAtr),
 		Secret: entry.GetAttributeValue(ld.secretAtr),
-		Role:   entry.GetAttributeValue(ld.roleAtr),
+		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
 	}, nil
 }
 
@@ -120,7 +120,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		result = append(result, Account{
 			Access: el.GetAttributeValue(ld.accessAtr),
 			Secret: el.GetAttributeValue(ld.secretAtr),
-			Role:   el.GetAttributeValue(ld.roleAtr),
+			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
 		})
 	}
 

aws/LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

aws/NOTICE.txt

@@ -0,0 +1,4 @@
+AWS SDK for Go
+Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2014-2015 Stripe, Inc.
+Copyright 2024 Versity Software

aws/README.md

@@ -0,0 +1,11 @@
+# AWS SDK Go v2
+
+This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License.
+
+## Description
+
+The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements.
+
+## License
+
+The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information.

aws/internal/awstesting/unit/unit.go

@@ -0,0 +1,61 @@
+// Package unit performs initialization and validation for unit tests
+package unit
+
+import (
+	"context"
+	"crypto/rsa"
+	"math/big"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func init() {
+	config = aws.Config{}
+	config.Region = "mock-region"
+	config.Credentials = StubCredentialsProvider{}
+}
+
+// StubCredentialsProvider provides a stub credential provider that returns
+// static credentials that never expire.
+type StubCredentialsProvider struct{}
+
+// Retrieve satisfies the CredentialsProvider interface. Returns stub
+// credential value, and never error.
+func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) {
+	return aws.Credentials{
+		AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION",
+		Source: "unit test credentials",
+	}, nil
+}
+
+var config aws.Config
+
+// Config returns a copy of the mock configuration for unit tests.
+func Config() aws.Config { return config.Copy() }
+
+// RSAPrivateKey is used for testing functionality that requires some
+// sort of private key. Taken from crypto/rsa/rsa_test.go
+//
+// Credit to golang 1.11
+var RSAPrivateKey = &rsa.PrivateKey{
+	PublicKey: rsa.PublicKey{
+		N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"),
+		E: 3,
+	},
+	D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"),
+	Primes: []*big.Int{
+		fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"),
+		fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"),
+	},
+}
+
+// Taken from crypto/rsa/rsa_test.go
+//
+// Credit to golang 1.11
+func fromBase10(base10 string) *big.Int {
+	i, ok := new(big.Int).SetString(base10, 10)
+	if !ok {
+		panic("bad number: " + base10)
+	}
+	return i
+}

aws/signer/internal/v4/cache.go

@@ -0,0 +1,115 @@
+package v4
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func lookupKey(service, region string) string {
+	var s strings.Builder
+	s.Grow(len(region) + len(service) + 3)
+	s.WriteString(region)
+	s.WriteRune('/')
+	s.WriteString(service)
+	return s.String()
+}
+
+type derivedKey struct {
+	AccessKey  string
+	Date       time.Time
+	Credential []byte
+}
+
+type derivedKeyCache struct {
+	values map[string]derivedKey
+	mutex  sync.RWMutex
+}
+
+func newDerivedKeyCache() derivedKeyCache {
+	return derivedKeyCache{
+		values: make(map[string]derivedKey),
+	}
+}
+
+func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte {
+	key := lookupKey(service, region)
+	s.mutex.RLock()
+	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
+		s.mutex.RUnlock()
+		return cred
+	}
+	s.mutex.RUnlock()
+
+	s.mutex.Lock()
+	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
+		s.mutex.Unlock()
+		return cred
+	}
+	cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime)
+	entry := derivedKey{
+		AccessKey:  credentials.AccessKeyID,
+		Date:       signingTime.Time,
+		Credential: cred,
+	}
+	s.values[key] = entry
+	s.mutex.Unlock()
+
+	return cred
+}
+
+func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) {
+	cacheEntry, ok := s.retrieveFromCache(key)
+	if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) {
+		return cacheEntry.Credential, true
+	}
+	return nil, false
+}
+
+func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) {
+	if v, ok := s.values[key]; ok {
+		return v, true
+	}
+	return derivedKey{}, false
+}
+
+// SigningKeyDeriver derives a signing key from a set of credentials
+type SigningKeyDeriver struct {
+	cache derivedKeyCache
+}
+
+// NewSigningKeyDeriver returns a new SigningKeyDeriver
+func NewSigningKeyDeriver() *SigningKeyDeriver {
+	return &SigningKeyDeriver{
+		cache: newDerivedKeyCache(),
+	}
+}
+
+// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing.
+func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte {
+	return k.cache.Get(credential, service, region, signingTime)
+}
+
+func deriveKey(secret, service, region string, t SigningTime) []byte {
+	hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat()))
+	hmacRegion := HMACSHA256(hmacDate, []byte(region))
+	hmacService := HMACSHA256(hmacRegion, []byte(service))
+	return HMACSHA256(hmacService, []byte("aws4_request"))
+}
+
+func isSameDay(x, y time.Time) bool {
+	xYear, xMonth, xDay := x.Date()
+	yYear, yMonth, yDay := y.Date()
+
+	if xYear != yYear {
+		return false
+	}
+
+	if xMonth != yMonth {
+		return false
+	}
+
+	return xDay == yDay
+}

aws/signer/internal/v4/const.go

@@ -0,0 +1,40 @@
+package v4
+
+// Signature Version 4 (SigV4) Constants
+const (
+	// EmptyStringSHA256 is the hex encoded sha256 value of an empty string
+	EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
+
+	// UnsignedPayload indicates that the request payload body is unsigned
+	UnsignedPayload = "UNSIGNED-PAYLOAD"
+
+	// AmzAlgorithmKey indicates the signing algorithm
+	AmzAlgorithmKey = "X-Amz-Algorithm"
+
+	// AmzSecurityTokenKey indicates the security token to be used with temporary credentials
+	AmzSecurityTokenKey = "X-Amz-Security-Token"
+
+	// AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z'
+	AmzDateKey = "X-Amz-Date"
+
+	// AmzCredentialKey is the access key ID and credential scope
+	AmzCredentialKey = "X-Amz-Credential"
+
+	// AmzSignedHeadersKey is the set of headers signed for the request
+	AmzSignedHeadersKey = "X-Amz-SignedHeaders"
+
+	// AmzSignatureKey is the query parameter to store the SigV4 signature
+	AmzSignatureKey = "X-Amz-Signature"
+
+	// TimeFormat is the time format to be used in the X-Amz-Date header or query parameter
+	TimeFormat = "20060102T150405Z"
+
+	// ShortTimeFormat is the shorten time format used in the credential scope
+	ShortTimeFormat = "20060102"
+
+	// ContentSHAKey is the SHA256 of request body
+	ContentSHAKey = "X-Amz-Content-Sha256"
+
+	// StreamingEventsPayload indicates that the request payload body is a signed event stream.
+	StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS"
+)

aws/signer/internal/v4/header_rules.go

@@ -0,0 +1,88 @@
+package v4
+
+import (
+	"strings"
+)
+
+// Rules houses a set of Rule needed for validation of a
+// string value
+type Rules []Rule
+
+// Rule interface allows for more flexible rules and just simply
+// checks whether or not a value adheres to that Rule
+type Rule interface {
+	IsValid(value string) bool
+}
+
+// IsValid will iterate through all rules and see if any rules
+// apply to the value and supports nested rules
+func (r Rules) IsValid(value string) bool {
+	for _, rule := range r {
+		if rule.IsValid(value) {
+			return true
+		}
+	}
+	return false
+}
+
+// MapRule generic Rule for maps
+type MapRule map[string]struct{}
+
+// IsValid for the map Rule satisfies whether it exists in the map
+func (m MapRule) IsValid(value string) bool {
+	_, ok := m[value]
+	return ok
+}
+
+// AllowList is a generic Rule for include listing
+type AllowList struct {
+	Rule
+}
+
+// IsValid for AllowList checks if the value is within the AllowList
+func (w AllowList) IsValid(value string) bool {
+	return w.Rule.IsValid(value)
+}
+
+// ExcludeList is a generic Rule for exclude listing
+type ExcludeList struct {
+	Rule
+}
+
+// IsValid for AllowList checks if the value is within the AllowList
+func (b ExcludeList) IsValid(value string) bool {
+	return !b.Rule.IsValid(value)
+}
+
+// Patterns is a list of strings to match against
+type Patterns []string
+
+// IsValid for Patterns checks each pattern and returns if a match has
+// been found
+func (p Patterns) IsValid(value string) bool {
+	for _, pattern := range p {
+		if hasPrefixFold(value, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// InclusiveRules rules allow for rules to depend on one another
+type InclusiveRules []Rule
+
+// IsValid will return true if all rules are true
+func (r InclusiveRules) IsValid(value string) bool {
+	for _, rule := range r {
+		if !rule.IsValid(value) {
+			return false
+		}
+	}
+	return true
+}
+
+// hasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
+// under Unicode case-folding.
+func hasPrefixFold(s, prefix string) bool {
+	return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
+}

aws/signer/internal/v4/headers.go

@@ -0,0 +1,72 @@
+package v4
+
+// IgnoredHeaders is a list of headers that are ignored during signing
+var IgnoredHeaders = Rules{
+	ExcludeList{
+		MapRule{
+			"Authorization": struct{}{},
+			// some clients use user-agent in signed headers
+			// "User-Agent":      struct{}{},
+			"X-Amzn-Trace-Id": struct{}{},
+			"Expect":          struct{}{},
+		},
+	},
+}
+
+// RequiredSignedHeaders is a allow list for Build canonical headers.
+var RequiredSignedHeaders = Rules{
+	AllowList{
+		MapRule{
+			"Cache-Control":                         struct{}{},
+			"Content-Disposition":                   struct{}{},
+			"Content-Encoding":                      struct{}{},
+			"Content-Language":                      struct{}{},
+			"Content-Md5":                           struct{}{},
+			"Content-Type":                          struct{}{},
+			"Expires":                               struct{}{},
+			"If-Match":                              struct{}{},
+			"If-Modified-Since":                     struct{}{},
+			"If-None-Match":                         struct{}{},
+			"If-Unmodified-Since":                   struct{}{},
+			"Range":                                 struct{}{},
+			"X-Amz-Acl":                             struct{}{},
+			"X-Amz-Copy-Source":                     struct{}{},
+			"X-Amz-Copy-Source-If-Match":            struct{}{},
+			"X-Amz-Copy-Source-If-Modified-Since":   struct{}{},
+			"X-Amz-Copy-Source-If-None-Match":       struct{}{},
+			"X-Amz-Copy-Source-If-Unmodified-Since": struct{}{},
+			"X-Amz-Copy-Source-Range":               struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       struct{}{},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   struct{}{},
+			"X-Amz-Expected-Bucket-Owner":                                 struct{}{},
+			"X-Amz-Grant-Full-control":                                    struct{}{},
+			"X-Amz-Grant-Read":                                            struct{}{},
+			"X-Amz-Grant-Read-Acp":                                        struct{}{},
+			"X-Amz-Grant-Write":                                           struct{}{},
+			"X-Amz-Grant-Write-Acp":                                       struct{}{},
+			"X-Amz-Metadata-Directive":                                    struct{}{},
+			"X-Amz-Mfa":                                                   struct{}{},
+			"X-Amz-Request-Payer":                                         struct{}{},
+			"X-Amz-Server-Side-Encryption":                                struct{}{},
+			"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id":                 struct{}{},
+			"X-Amz-Server-Side-Encryption-Context":                        struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm":             struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Key":                   struct{}{},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":               struct{}{},
+			"X-Amz-Storage-Class":                                         struct{}{},
+			"X-Amz-Website-Redirect-Location":                             struct{}{},
+			"X-Amz-Content-Sha256":                                        struct{}{},
+			"X-Amz-Tagging":                                               struct{}{},
+		},
+	},
+	Patterns{"X-Amz-Object-Lock-"},
+	Patterns{"X-Amz-Meta-"},
+}
+
+// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value
+// represents whether or not it is a pattern.
+var AllowedQueryHoisting = InclusiveRules{
+	ExcludeList{RequiredSignedHeaders},
+	Patterns{"X-Amz-"},
+}

aws/signer/internal/v4/headers_test.go

@@ -0,0 +1,63 @@
+package v4
+
+import "testing"
+
+func TestAllowedQueryHoisting(t *testing.T) {
+	cases := map[string]struct {
+		Header      string
+		ExpectHoist bool
+	}{
+		"object-lock": {
+			Header:      "X-Amz-Object-Lock-Mode",
+			ExpectHoist: false,
+		},
+		"s3 metadata": {
+			Header:      "X-Amz-Meta-SomeName",
+			ExpectHoist: false,
+		},
+		"another header": {
+			Header:      "X-Amz-SomeOtherHeader",
+			ExpectHoist: true,
+		},
+		"non X-AMZ header": {
+			Header:      "X-SomeOtherHeader",
+			ExpectHoist: false,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a {
+				t.Errorf("expect hoist %v, was %v", e, a)
+			}
+		})
+	}
+}
+
+func TestIgnoredHeaders(t *testing.T) {
+	cases := map[string]struct {
+		Header        string
+		ExpectIgnored bool
+	}{
+		"expect": {
+			Header:        "Expect",
+			ExpectIgnored: true,
+		},
+		"authorization": {
+			Header:        "Authorization",
+			ExpectIgnored: true,
+		},
+		"X-AMZ header": {
+			Header:        "X-Amz-Content-Sha256",
+			ExpectIgnored: false,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a {
+				t.Errorf("expect ignored %v, was %v", e, a)
+			}
+		})
+	}
+}

aws/signer/internal/v4/hmac.go

@@ -0,0 +1,13 @@
+package v4
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+)
+
+// HMACSHA256 computes a HMAC-SHA256 of data given the provided key.
+func HMACSHA256(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}

aws/signer/internal/v4/host.go

@@ -0,0 +1,75 @@
+package v4
+
+import (
+	"net/http"
+	"strings"
+)
+
+// SanitizeHostForHeader removes default port from host and updates request.Host
+func SanitizeHostForHeader(r *http.Request) {
+	host := getHost(r)
+	port := portOnly(host)
+	if port != "" && isDefaultPort(r.URL.Scheme, port) {
+		r.Host = stripPort(host)
+	}
+}
+
+// Returns host from request
+func getHost(r *http.Request) string {
+	if r.Host != "" {
+		return r.Host
+	}
+
+	return r.URL.Host
+}
+
+// Hostname returns u.Host, without any port number.
+//
+// If Host is an IPv6 literal with a port number, Hostname returns the
+// IPv6 literal without the square brackets. IPv6 literals may include
+// a zone identifier.
+//
+// Copied from the Go 1.8 standard library (net/url)
+func stripPort(hostport string) string {
+	colon := strings.IndexByte(hostport, ':')
+	if colon == -1 {
+		return hostport
+	}
+	if i := strings.IndexByte(hostport, ']'); i != -1 {
+		return strings.TrimPrefix(hostport[:i], "[")
+	}
+	return hostport[:colon]
+}
+
+// Port returns the port part of u.Host, without the leading colon.
+// If u.Host doesn't contain a port, Port returns an empty string.
+//
+// Copied from the Go 1.8 standard library (net/url)
+func portOnly(hostport string) string {
+	colon := strings.IndexByte(hostport, ':')
+	if colon == -1 {
+		return ""
+	}
+	if i := strings.Index(hostport, "]:"); i != -1 {
+		return hostport[i+len("]:"):]
+	}
+	if strings.Contains(hostport, "]") {
+		return ""
+	}
+	return hostport[colon+len(":"):]
+}
+
+// Returns true if the specified URI is using the standard port
+// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs)
+func isDefaultPort(scheme, port string) bool {
+	if port == "" {
+		return true
+	}
+
+	lowerCaseScheme := strings.ToLower(scheme)
+	if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") {
+		return true
+	}
+
+	return false
+}

aws/signer/internal/v4/scope.go

@@ -0,0 +1,13 @@
+package v4
+
+import "strings"
+
+// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope
+func BuildCredentialScope(signingTime SigningTime, region, service string) string {
+	return strings.Join([]string{
+		signingTime.ShortTimeFormat(),
+		region,
+		service,
+		"aws4_request",
+	}, "/")
+}

aws/signer/internal/v4/time.go

@@ -0,0 +1,36 @@
+package v4
+
+import "time"
+
+// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing.
+type SigningTime struct {
+	time.Time
+	timeFormat      string
+	shortTimeFormat string
+}
+
+// NewSigningTime creates a new SigningTime given a time.Time
+func NewSigningTime(t time.Time) SigningTime {
+	return SigningTime{
+		Time: t,
+	}
+}
+
+// TimeFormat provides a time formatted in the X-Amz-Date format.
+func (m *SigningTime) TimeFormat() string {
+	return m.format(&m.timeFormat, TimeFormat)
+}
+
+// ShortTimeFormat provides a time formatted of 20060102.
+func (m *SigningTime) ShortTimeFormat() string {
+	return m.format(&m.shortTimeFormat, ShortTimeFormat)
+}
+
+func (m *SigningTime) format(target *string, format string) string {
+	if len(*target) > 0 {
+		return *target
+	}
+	v := m.Time.Format(format)
+	*target = v
+	return v
+}

aws/signer/internal/v4/util.go

@@ -0,0 +1,80 @@
+package v4
+
+import (
+	"net/url"
+	"strings"
+)
+
+const doubleSpace = "  "
+
+// StripExcessSpaces will rewrite the passed in slice's string values to not
+// contain multiple side-by-side spaces.
+func StripExcessSpaces(str string) string {
+	var j, k, l, m, spaces int
+	// Trim trailing spaces
+	for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- {
+	}
+
+	// Trim leading spaces
+	for k = 0; k < j && str[k] == ' '; k++ {
+	}
+	str = str[k : j+1]
+
+	// Strip multiple spaces.
+	j = strings.Index(str, doubleSpace)
+	if j < 0 {
+		return str
+	}
+
+	buf := []byte(str)
+	for k, m, l = j, j, len(buf); k < l; k++ {
+		if buf[k] == ' ' {
+			if spaces == 0 {
+				// First space.
+				buf[m] = buf[k]
+				m++
+			}
+			spaces++
+		} else {
+			// End of multiple spaces.
+			spaces = 0
+			buf[m] = buf[k]
+			m++
+		}
+	}
+
+	return string(buf[:m])
+}
+
+// GetURIPath returns the escaped URI component from the provided URL.
+func GetURIPath(u *url.URL) string {
+	var uriPath string
+
+	if len(u.Opaque) > 0 {
+		const schemeSep, pathSep, queryStart = "//", "/", "?"
+
+		opaque := u.Opaque
+		// Cut off the query string if present.
+		if idx := strings.Index(opaque, queryStart); idx >= 0 {
+			opaque = opaque[:idx]
+		}
+
+		// Cutout the scheme separator if present.
+		if strings.Index(opaque, schemeSep) == 0 {
+			opaque = opaque[len(schemeSep):]
+		}
+
+		// capture URI path starting with first path separator.
+		if idx := strings.Index(opaque, pathSep); idx >= 0 {
+			uriPath = opaque[idx:]
+		}
+	} else {
+		uriPath = u.EscapedPath()
+	}
+
+	if len(uriPath) == 0 {
+		uriPath = "/"
+	}
+
+	return uriPath
+}

aws/signer/internal/v4/util_test.go

@@ -0,0 +1,158 @@
+package v4
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+func lazyURLParse(v string) func() (*url.URL, error) {
+	return func() (*url.URL, error) {
+		return url.Parse(v)
+	}
+}
+
+func TestGetURIPath(t *testing.T) {
+	cases := map[string]struct {
+		getURL func() (*url.URL, error)
+		expect string
+	}{
+		// Cases
+		"with scheme": {
+			getURL: lazyURLParse("https://localhost:9000"),
+			expect: "/",
+		},
+		"no port, with scheme": {
+			getURL: lazyURLParse("https://localhost"),
+			expect: "/",
+		},
+		"without scheme": {
+			getURL: lazyURLParse("localhost:9000"),
+			expect: "/",
+		},
+		"without scheme, with path": {
+			getURL: lazyURLParse("localhost:9000/abc123"),
+			expect: "/abc123",
+		},
+		"without scheme, with separator": {
+			getURL: lazyURLParse("//localhost:9000"),
+			expect: "/",
+		},
+		"no port, without scheme, with separator": {
+			getURL: lazyURLParse("//localhost"),
+			expect: "/",
+		},
+		"without scheme, with separator, with path": {
+			getURL: lazyURLParse("//localhost:9000/abc123"),
+			expect: "/abc123",
+		},
+		"no port, without scheme, with separator, with path": {
+			getURL: lazyURLParse("//localhost/abc123"),
+			expect: "/abc123",
+		},
+		"opaque with query string": {
+			getURL: lazyURLParse("localhost:9000/abc123?efg=456"),
+			expect: "/abc123",
+		},
+		"failing test": {
+			getURL: func() (*url.URL, error) {
+				endpoint := "https://service.region.amazonaws.com"
+				req, _ := http.NewRequest("POST", endpoint, nil)
+				u := req.URL
+
+				u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
+
+				query := u.Query()
+				query.Set("some-query-key", "value")
+				u.RawQuery = query.Encode()
+
+				return u, nil
+			},
+			expect: "/bucket/key-._~,!@#$%^&*()",
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			u, err := c.getURL()
+			if err != nil {
+				t.Fatalf("failed to get URL, %v", err)
+			}
+
+			actual := GetURIPath(u)
+			if e, a := c.expect, actual; e != a {
+				t.Errorf("expect %v path, got %v", e, a)
+			}
+		})
+	}
+}
+
+func TestStripExcessHeaders(t *testing.T) {
+	vals := []string{
+		"",
+		"123",
+		"1 2 3",
+		"1 2 3 ",
+		"  1 2 3",
+		"1  2 3",
+		"1  23",
+		"1  2  3",
+		"1  2  ",
+		" 1  2  ",
+		"12   3",
+		"12   3   1",
+		"12           3     1",
+		"12     3       1abc123",
+	}
+
+	expected := []string{
+		"",
+		"123",
+		"1 2 3",
+		"1 2 3",
+		"1 2 3",
+		"1 2 3",
+		"1 23",
+		"1 2 3",
+		"1 2",
+		"1 2",
+		"12 3",
+		"12 3 1",
+		"12 3 1",
+		"12 3 1abc123",
+	}
+
+	for i := 0; i < len(vals); i++ {
+		r := StripExcessSpaces(vals[i])
+		if e, a := expected[i], r; e != a {
+			t.Errorf("%d, expect %v, got %v", i, e, a)
+		}
+	}
+}
+
+var stripExcessSpaceCases = []string{
+	`AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`,
+	`123   321   123   321`,
+	`   123   321   123   321   `,
+	`   123    321    123          321   `,
+	"123",
+	"1 2 3",
+	"  1 2 3",
+	"1  2 3",
+	"1  23",
+	"1  2  3",
+	"1  2  ",
+	" 1  2  ",
+	"12   3",
+	"12   3   1",
+	"12           3     1",
+	"12     3       1abc123",
+}
+
+func BenchmarkStripExcessSpaces(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		for _, v := range stripExcessSpaceCases {
+			StripExcessSpaces(v)
+		}
+	}
+}

aws/signer/v4/functional_test.go

@@ -0,0 +1,139 @@
+package v4_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/versity/versitygw/aws/internal/awstesting/unit"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+var standaloneSignCases = []struct {
+	OrigURI                    string
+	OrigQuery                  string
+	Region, Service, SubDomain string
+	ExpSig                     string
+	EscapedURI                 string
+}{
+	{
+		OrigURI:   `/logs-*/_search`,
+		OrigQuery: `pretty=true`,
+		Region:    "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
+		EscapedURI: `/logs-%2A/_search`,
+		ExpSig:     `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
+	},
+}
+
+func TestStandaloneSign_CustomURIEscape(t *testing.T) {
+	var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967`
+
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner(func(signer *v4.SignerOptions) {
+		signer.DisableURIPathEscaping = true
+	})
+
+	host := "https://subdomain.us-east-1.es.amazonaws.com"
+	req, err := http.NewRequest("GET", host, nil)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	req.URL.Path = `/log-*/_search`
+	req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search"
+
+	err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0))
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	actual := req.Header.Get("Authorization")
+	if e, a := expectSig, actual; e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestStandaloneSign(t *testing.T) {
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner()
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		// URL.EscapedPath() will be used by the signer to get the
+		// escaped form of the request's URI path.
+		req.URL.Path = c.OrigURI
+		req.URL.RawQuery = c.OrigQuery
+
+		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		actual := req.Header.Get("Authorization")
+		if e, a := c.ExpSig, actual; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.OrigURI, req.URL.Path; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+	}
+}
+
+func TestStandaloneSign_RawPath(t *testing.T) {
+	creds, err := unit.Config().Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	signer := v4.NewSigner()
+
+	for _, c := range standaloneSignCases {
+		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
+			c.SubDomain, c.Region, c.Service)
+
+		req, err := http.NewRequest("GET", host, nil)
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		// URL.EscapedPath() will be used by the signer to get the
+		// escaped form of the request's URI path.
+		req.URL.Path = c.OrigURI
+		req.URL.RawPath = c.EscapedURI
+		req.URL.RawQuery = c.OrigQuery
+
+		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
+		if err != nil {
+			t.Errorf("expected no error, but received %v", err)
+		}
+
+		actual := req.Header.Get("Authorization")
+		if e, a := c.ExpSig, actual; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.OrigURI, req.URL.Path; e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
+			t.Errorf("expected %v, but recieved %v", e, a)
+		}
+	}
+}

aws/signer/v4/v4.go

@@ -0,0 +1,565 @@
+// Package v4 implements signing for AWS V4 signer
+//
+// Provides request signing for request that need to be signed with
+// AWS V4 Signatures.
+//
+// # Standalone Signer
+//
+// Generally using the signer outside of the SDK should not require any additional
+//
+//	The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires
+//
+// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent
+// to the service as.
+//
+// The signer will first check the URL.Opaque field, and use its value if set.
+// The signer does require the URL.Opaque field to be set in the form of:
+//
+//	"//<hostname>/<path>"
+//
+//	// e.g.
+//	"//example.com/some/path"
+//
+// The leading "//" and hostname are required or the URL.Opaque escaping will
+// not work correctly.
+//
+// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath()
+// method and using the returned value.
+//
+// AWS v4 signature validation requires that the canonical string's URI path
+// element must be the URI escaped form of the HTTP request's path.
+// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+//
+// The Go HTTP client will perform escaping automatically on the request. Some
+// of these escaping may cause signature validation errors because the HTTP
+// request differs from the URI path or query that the signature was generated.
+// https://golang.org/pkg/net/url/#URL.EscapedPath
+//
+// Because of this, it is recommended that when using the signer outside of the
+// SDK that explicitly escaping the request prior to being signed is preferable,
+// and will help prevent signature validation errors. This can be done by setting
+// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then
+// call URL.EscapedPath() if Opaque is not set.
+//
+// Test `TestStandaloneSign` provides a complete example of using the signer
+// outside of the SDK and pre-escaping the URI path.
+package v4
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"net/http"
+	"net/textproto"
+	"net/url"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go/encoding/httpbinding"
+	"github.com/aws/smithy-go/logging"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+const (
+	signingAlgorithm    = "AWS4-HMAC-SHA256"
+	authorizationHeader = "Authorization"
+
+	// Version of signing v4
+	Version = "SigV4"
+)
+
+// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests
+type HTTPSigner interface {
+	SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
+}
+
+type keyDerivator interface {
+	DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte
+}
+
+// SignerOptions is the SigV4 Signer options.
+type SignerOptions struct {
+	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
+	// request header to the request's query string. This is most commonly used
+	// with pre-signed requests preventing headers from being added to the
+	// request's query string.
+	DisableHeaderHoisting bool
+
+	// Disables the automatic escaping of the URI path of the request for the
+	// siganture's canonical string's path. For services that do not need additional
+	// escaping then use this to disable the signer escaping the path.
+	//
+	// S3 is an example of a service that does not need additional escaping.
+	//
+	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+	DisableURIPathEscaping bool
+
+	// The logger to send log messages to.
+	Logger logging.Logger
+
+	// Enable logging of signed requests.
+	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
+	// presigned URL.
+	LogSigning bool
+
+	// Disables setting the session token on the request as part of signing
+	// through X-Amz-Security-Token. This is needed for variations of v4 that
+	// present the token elsewhere.
+	DisableSessionToken bool
+}
+
+// Signer applies AWS v4 signing to given request. Use this to sign requests
+// that need to be signed with AWS V4 Signatures.
+type Signer struct {
+	options      SignerOptions
+	keyDerivator keyDerivator
+}
+
+// NewSigner returns a new SigV4 Signer
+func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
+	options := SignerOptions{}
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()}
+}
+
+type httpSigner struct {
+	Request      *http.Request
+	ServiceName  string
+	Region       string
+	Time         v4Internal.SigningTime
+	Credentials  aws.Credentials
+	KeyDerivator keyDerivator
+	IsPreSign    bool
+	SignedHdrs   []string
+
+	PayloadHash string
+
+	DisableHeaderHoisting  bool
+	DisableURIPathEscaping bool
+	DisableSessionToken    bool
+}
+
+func (s *httpSigner) Build() (signedRequest, error) {
+	req := s.Request
+
+	query := req.URL.Query()
+	headers := req.Header
+
+	s.setRequiredSigningFields(headers, query)
+
+	// Sort Each Query Key's Values
+	for key := range query {
+		sort.Strings(query[key])
+	}
+
+	v4Internal.SanitizeHostForHeader(req)
+
+	credentialScope := s.buildCredentialScope()
+	credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzCredentialKey, credentialStr)
+	}
+
+	unsignedHeaders := headers
+	if s.IsPreSign && !s.DisableHeaderHoisting {
+		var urlValues url.Values
+		urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers)
+		for k := range urlValues {
+			query[k] = urlValues[k]
+		}
+	}
+
+	host := req.URL.Host
+	if len(req.Host) > 0 {
+		host = req.Host
+	}
+
+	signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength)
+
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr)
+	}
+
+	var rawQuery strings.Builder
+	rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1))
+
+	canonicalURI := v4Internal.GetURIPath(req.URL)
+	if !s.DisableURIPathEscaping {
+		canonicalURI = httpbinding.EscapePath(canonicalURI, false)
+	}
+
+	canonicalString := s.buildCanonicalString(
+		req.Method,
+		canonicalURI,
+		rawQuery.String(),
+		signedHeadersStr,
+		canonicalHeaderStr,
+	)
+
+	strToSign := s.buildStringToSign(credentialScope, canonicalString)
+	signingSignature, err := s.buildSignature(strToSign)
+	if err != nil {
+		return signedRequest{}, err
+	}
+
+	if s.IsPreSign {
+		rawQuery.WriteString("&X-Amz-Signature=")
+		rawQuery.WriteString(signingSignature)
+	} else {
+		headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature))
+	}
+
+	req.URL.RawQuery = rawQuery.String()
+
+	return signedRequest{
+		Request:         req,
+		SignedHeaders:   signedHeaders,
+		CanonicalString: canonicalString,
+		StringToSign:    strToSign,
+		PreSigned:       s.IsPreSign,
+	}, nil
+}
+
+func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string {
+	const credential = "Credential="
+	const signedHeaders = "SignedHeaders="
+	const signature = "Signature="
+	const commaSpace = ", "
+
+	var parts strings.Builder
+	parts.Grow(len(signingAlgorithm) + 1 +
+		len(credential) + len(credentialStr) + 2 +
+		len(signedHeaders) + len(signedHeadersStr) + 2 +
+		len(signature) + len(signingSignature),
+	)
+	parts.WriteString(signingAlgorithm)
+	parts.WriteRune(' ')
+	parts.WriteString(credential)
+	parts.WriteString(credentialStr)
+	parts.WriteString(commaSpace)
+	parts.WriteString(signedHeaders)
+	parts.WriteString(signedHeadersStr)
+	parts.WriteString(commaSpace)
+	parts.WriteString(signature)
+	parts.WriteString(signingSignature)
+	return parts.String()
+}
+
+// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the
+// request is made to, and time the request is signed at. The signTime allows
+// you to specify that a request is signed for the future, and cannot be
+// used until then.
+//
+// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
+// must be provided. Even if the request has no payload (aka body). If the
+// request has no payload you should use the hex encoded SHA-256 of an empty
+// string as the payloadHash value.
+//
+//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+//
+// Some services such as Amazon S3 accept alternative values for the payload
+// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
+// included in the request signature.
+//
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+//
+// Sign differs from Presign in that it will sign the request using HTTP
+// header values. This type of signing is intended for http.Request values that
+// will not be shared, or are shared in a way the header values on the request
+// will not be lost.
+//
+// The passed in request will be modified in place.
+func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) error {
+	options := s.options
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	signer := &httpSigner{
+		Request:                r,
+		PayloadHash:            payloadHash,
+		ServiceName:            service,
+		Region:                 region,
+		Credentials:            credentials,
+		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
+		DisableHeaderHoisting:  options.DisableHeaderHoisting,
+		DisableURIPathEscaping: options.DisableURIPathEscaping,
+		DisableSessionToken:    options.DisableSessionToken,
+		KeyDerivator:           s.keyDerivator,
+		SignedHdrs:             signedHdrs,
+	}
+
+	signedRequest, err := signer.Build()
+	if err != nil {
+		return err
+	}
+
+	logSigningInfo(ctx, options, &signedRequest, false)
+
+	return nil
+}
+
+// PresignHTTP signs AWS v4 requests with the payload hash, service name, region
+// the request is made to, and time the request is signed at. The signTime
+// allows you to specify that a request is signed for the future, and cannot
+// be used until then.
+//
+// Returns the signed URL and the map of HTTP headers that were included in the
+// signature or an error if signing the request failed. For presigned requests
+// these headers and their values must be included on the HTTP request when it
+// is made. This is helpful to know what header values need to be shared with
+// the party the presigned request will be distributed to.
+//
+// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
+// must be provided. Even if the request has no payload (aka body). If the
+// request has no payload you should use the hex encoded SHA-256 of an empty
+// string as the payloadHash value.
+//
+//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+//
+// Some services such as Amazon S3 accept alternative values for the payload
+// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
+// included in the request signature.
+//
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+//
+// PresignHTTP differs from SignHTTP in that it will sign the request using
+// query string instead of header values. This allows you to share the
+// Presigned Request's URL with third parties, or distribute it throughout your
+// system with minimal dependencies.
+//
+// PresignHTTP will not set the expires time of the presigned request
+// automatically. To specify the expire duration for a request add the
+// "X-Amz-Expires" query parameter on the request with the value as the
+// duration in seconds the presigned URL should be considered valid for. This
+// parameter is not used by all AWS services, and is most notable used by
+// Amazon S3 APIs.
+//
+//	expires := 20 * time.Minute
+//	query := req.URL.Query()
+//	query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10))
+//	req.URL.RawQuery = query.Encode()
+//
+// This method does not modify the provided request.
+func (s *Signer) PresignHTTP(
+	ctx context.Context, credentials aws.Credentials, r *http.Request,
+	payloadHash string, service string, region string, signingTime time.Time,
+	signedHdrs []string,
+	optFns ...func(*SignerOptions),
+) (signedURI string, signedHeaders http.Header, err error) {
+	options := s.options
+
+	for _, fn := range optFns {
+		fn(&options)
+	}
+
+	signer := &httpSigner{
+		Request:                r.Clone(r.Context()),
+		PayloadHash:            payloadHash,
+		ServiceName:            service,
+		Region:                 region,
+		Credentials:            credentials,
+		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
+		IsPreSign:              true,
+		DisableHeaderHoisting:  options.DisableHeaderHoisting,
+		DisableURIPathEscaping: options.DisableURIPathEscaping,
+		DisableSessionToken:    options.DisableSessionToken,
+		KeyDerivator:           s.keyDerivator,
+		SignedHdrs:             signedHdrs,
+	}
+
+	signedRequest, err := signer.Build()
+	if err != nil {
+		return "", nil, err
+	}
+
+	logSigningInfo(ctx, options, &signedRequest, true)
+
+	signedHeaders = make(http.Header)
+
+	// For the signed headers we canonicalize the header keys in the returned map.
+	// This avoids situations where can standard library double headers like host header. For example the standard
+	// library will set the Host header, even if it is present in lower-case form.
+	for k, v := range signedRequest.SignedHeaders {
+		key := textproto.CanonicalMIMEHeaderKey(k)
+		signedHeaders[key] = append(signedHeaders[key], v...)
+	}
+
+	return signedRequest.Request.URL.String(), signedHeaders, nil
+}
+
+func (s *httpSigner) buildCredentialScope() string {
+	return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName)
+}
+
+func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) {
+	query := url.Values{}
+	unsignedHeaders := http.Header{}
+	for k, h := range header {
+		if r.IsValid(k) {
+			query[k] = h
+		} else {
+			unsignedHeaders[k] = h
+		}
+	}
+
+	return query, unsignedHeaders
+}
+
+func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) {
+	signed = make(http.Header)
+
+	var headers []string
+	const hostHeader = "host"
+	headers = append(headers, hostHeader)
+	signed[hostHeader] = append(signed[hostHeader], host)
+
+	const contentLengthHeader = "content-length"
+	if slices.Contains(s.SignedHdrs, contentLengthHeader) {
+		headers = append(headers, contentLengthHeader)
+		signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10))
+	}
+
+	for k, v := range header {
+		if !rule.IsValid(k) {
+			continue // ignored header
+		}
+		if strings.EqualFold(k, contentLengthHeader) {
+			// prevent signing already handled content-length header.
+			continue
+		}
+
+		lowerCaseKey := strings.ToLower(k)
+		if _, ok := signed[lowerCaseKey]; ok {
+			// include additional values
+			signed[lowerCaseKey] = append(signed[lowerCaseKey], v...)
+			continue
+		}
+
+		headers = append(headers, lowerCaseKey)
+		signed[lowerCaseKey] = v
+	}
+	sort.Strings(headers)
+
+	signedHeaders = strings.Join(headers, ";")
+
+	var canonicalHeaders strings.Builder
+	n := len(headers)
+	const colon = ':'
+	for i := 0; i < n; i++ {
+		if headers[i] == hostHeader {
+			canonicalHeaders.WriteString(hostHeader)
+			canonicalHeaders.WriteRune(colon)
+			canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host))
+		} else {
+			canonicalHeaders.WriteString(headers[i])
+			canonicalHeaders.WriteRune(colon)
+			// Trim out leading, trailing, and dedup inner spaces from signed header values.
+			values := signed[headers[i]]
+			for j, v := range values {
+				cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v))
+				canonicalHeaders.WriteString(cleanedValue)
+				if j < len(values)-1 {
+					canonicalHeaders.WriteRune(',')
+				}
+			}
+		}
+		canonicalHeaders.WriteRune('\n')
+	}
+	canonicalHeadersStr = canonicalHeaders.String()
+
+	return signed, signedHeaders, canonicalHeadersStr
+}
+
+func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string {
+	return strings.Join([]string{
+		method,
+		uri,
+		query,
+		canonicalHeaders,
+		signedHeaders,
+		s.PayloadHash,
+	}, "\n")
+}
+
+func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string {
+	return strings.Join([]string{
+		signingAlgorithm,
+		s.Time.TimeFormat(),
+		credentialScope,
+		hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))),
+	}, "\n")
+}
+
+func makeHash(hash hash.Hash, b []byte) []byte {
+	hash.Reset()
+	hash.Write(b)
+	return hash.Sum(nil)
+}
+
+func (s *httpSigner) buildSignature(strToSign string) (string, error) {
+	key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time)
+	return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil
+}
+
+func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) {
+	amzDate := s.Time.TimeFormat()
+
+	if s.IsPreSign {
+		query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm)
+		sessionToken := s.Credentials.SessionToken
+		if !s.DisableSessionToken && len(sessionToken) > 0 {
+			query.Set("X-Amz-Security-Token", sessionToken)
+		}
+
+		query.Set(v4Internal.AmzDateKey, amzDate)
+		return
+	}
+
+	headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate)
+
+	if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 {
+		headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken)
+	}
+}
+
+func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) {
+	if !options.LogSigning {
+		return
+	}
+	signedURLMsg := ""
+	if isPresign {
+		signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String())
+	}
+	logger := logging.WithContext(ctx, options.Logger)
+	logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg)
+}
+
+type signedRequest struct {
+	Request         *http.Request
+	SignedHeaders   http.Header
+	CanonicalString string
+	StringToSign    string
+	PreSigned       bool
+}
+
+const logSignInfoMsg = `Request Signature:
+---[ CANONICAL STRING  ]-----------------------------
+%s
+---[ STRING TO SIGN ]--------------------------------
+%s%s
+-----------------------------------------------------`
+const logSignedURLMsg = `
+---[ SIGNED URL ]------------------------------------
+%s`

aws/signer/v4/v4_test.go

@@ -0,0 +1,358 @@
+package v4
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/google/go-cmp/cmp"
+	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
+)
+
+var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"}
+
+func buildRequest(serviceName, region, body string) (*http.Request, string) {
+	reader := strings.NewReader(body)
+	return buildRequestWithBodyReader(serviceName, region, reader)
+}
+
+func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) {
+	var bodyLen int
+
+	type lenner interface {
+		Len() int
+	}
+	if lr, ok := body.(lenner); ok {
+		bodyLen = lr.Len()
+	}
+
+	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
+	req, _ := http.NewRequest("POST", endpoint, body)
+	req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
+	req.Header.Set("X-Amz-Target", "prefix.Operation")
+	req.Header.Set("Content-Type", "application/x-amz-json-1.0")
+
+	if bodyLen > 0 {
+		req.ContentLength = int64(bodyLen)
+	}
+
+	req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)")
+	req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
+	req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
+
+	h := sha256.New()
+	_, _ = io.Copy(h, body)
+	payloadHash := hex.EncodeToString(h.Sum(nil))
+
+	return req, payloadHash
+}
+
+func TestPresignRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "300")
+	req.URL.RawQuery = query.Encode()
+
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	signer := NewSigner()
+	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
+	expectedSig := "122f0b9e091e4ba84286097e2b3404a1f1f4c4aad479adda95b7dff0ccbe5581"
+	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
+	expectedTarget := "prefix.Operation"
+
+	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
+	if err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+
+	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
+		t.Errorf("expect %v to be empty", a)
+	}
+	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+
+	for _, h := range strings.Split(expectedHeaders, ";") {
+		v := headers.Get(h)
+		if len(v) == 0 {
+			t.Errorf("expect %v, to be present in header map", h)
+		}
+	}
+}
+
+func TestPresignBodyWithArrayRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "300")
+	req.URL.RawQuery = query.Encode()
+
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	signer := NewSigner()
+	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
+	if err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
+	expectedSig := "e3ac55addee8711b76c6d608d762cff285fe8b627a057f8b5ec9268cf82c08b1"
+	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
+	expectedTarget := "prefix.Operation"
+
+	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
+		t.Errorf("expect %v to be empty, was not", a)
+	}
+	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+
+	for _, h := range strings.Split(expectedHeaders, ";") {
+		v := headers.Get(h)
+		if len(v) == 0 {
+			t.Errorf("expect %v, to be present in header map", h)
+		}
+	}
+}
+
+func TestSignRequest(t *testing.T) {
+	req, body := buildRequest("dynamodb", "us-east-1", "{}")
+	signer := NewSigner()
+	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
+	err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	expectedDate := "19700101T000000Z"
+	expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9"
+
+	q := req.Header
+	if e, a := expectedSig, q.Get("Authorization"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestBuildCanonicalRequest(t *testing.T) {
+	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z"
+	if e, a := expected, build.Request.URL.String(); e != a {
+		t.Errorf("expect %v, got %v", e, a)
+	}
+}
+
+func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+	req.Body = io.NopCloser(bytes.NewReader([]byte{}))
+
+	s := NewSigner()
+
+	origBody := req.Body
+
+	err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	if req.Body != origBody {
+		t.Errorf("expect request body to not be chagned")
+	}
+}
+
+func TestRequestHost(t *testing.T) {
+	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+	req.Host = "myhost"
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if !strings.Contains(build.CanonicalString, "host:"+req.Host) {
+		t.Errorf("canonical host header invalid")
+	}
+}
+
+func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) {
+	body := `{"description": "this is a test"}`
+	req, _ := buildRequest("dynamodb", "us-east-1", body)
+	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
+	req.Host = "myhost"
+
+	contentLength := fmt.Sprintf("%d", len([]byte(body)))
+	req.Header.Add("Content-Length", contentLength)
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	ctx := &httpSigner{
+		ServiceName:  "dynamodb",
+		Region:       "us-east-1",
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Now()),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	_, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	//if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") {
+	//	t.Errorf("canonical header content-length invalid")
+	//}
+}
+
+func TestSign_buildCanonicalHeaders(t *testing.T) {
+	serviceName := "mockAPI"
+	region := "mock-region"
+	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
+
+	req, err := http.NewRequest("POST", endpoint, nil)
+	if err != nil {
+		t.Fatalf("failed to create request, %v", err)
+	}
+
+	req.Header.Set("FooInnerSpace", "   inner      space    ")
+	req.Header.Set("FooLeadingSpace", "    leading-space")
+	req.Header.Add("FooMultipleSpace", "no-space")
+	req.Header.Add("FooMultipleSpace", "\ttab-space")
+	req.Header.Add("FooMultipleSpace", "trailing-space    ")
+	req.Header.Set("FooNoSpace", "no-space")
+	req.Header.Set("FooTabSpace", "\ttab-space\t")
+	req.Header.Set("FooTrailingSpace", "trailing-space    ")
+	req.Header.Set("FooWrappedSpace", "   wrapped-space    ")
+
+	ctx := &httpSigner{
+		ServiceName:  serviceName,
+		Region:       region,
+		Request:      req,
+		Time:         v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)),
+		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
+	}
+
+	build, err := ctx.Build()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectCanonicalString := strings.Join([]string{
+		`POST`,
+		`/`,
+		``,
+		`fooinnerspace:inner space`,
+		`fooleadingspace:leading-space`,
+		`foomultiplespace:no-space,tab-space,trailing-space`,
+		`foonospace:no-space`,
+		`footabspace:tab-space`,
+		`footrailingspace:trailing-space`,
+		`foowrappedspace:wrapped-space`,
+		`host:mockAPI.mock-region.amazonaws.com`,
+		`x-amz-date:20211020T124200Z`,
+		``,
+		`fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`,
+		``,
+	}, "\n")
+	if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" {
+		t.Errorf("expect match, got\n%s", diff)
+	}
+}
+
+func BenchmarkPresignRequest(b *testing.B) {
+	signer := NewSigner()
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+
+	query := req.URL.Query()
+	query.Set("X-Amz-Expires", "5")
+	req.URL.RawQuery = query.Encode()
+
+	for i := 0; i < b.N; i++ {
+		signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	}
+}
+
+func BenchmarkSignRequest(b *testing.B) {
+	signer := NewSigner()
+	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
+	for i := 0; i < b.N; i++ {
+		signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
+	}
+}

backend/azure/azure.go

@@ -184,6 +184,63 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string,
 	return string(*uploadResp.ETag), nil
 }
 
+func (az *Azure) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	if tags == nil {
+		_, err := client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: map[string]*string{
+			string(aclKeyCapital): resp.Metadata[string(aclKeyCapital)],
+		}})
+		if err != nil {
+			return azureErrToS3Err(err)
+		}
+
+		return nil
+	}
+
+	_, ok := tags[string(aclKeyLower)]
+	if ok {
+		delete(tags, string(aclKeyLower))
+	}
+
+	tags[string(aclKeyCapital)] = *resp.Metadata[string(aclKeyCapital)]
+
+	_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: parseMetadata(tags)})
+	if err != nil {
+		return azureErrToS3Err(err)
+	}
+
+	return nil
+}
+
+func (az *Azure) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) {
+	client, err := az.getContainerClient(bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.GetProperties(ctx, &container.GetPropertiesOptions{})
+	if err != nil {
+		return nil, azureErrToS3Err(err)
+	}
+
+	delete(resp.Metadata, string(aclKeyCapital))
+
+	return parseAzMetadata(resp.Metadata), nil
+}
+
+func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error {
+	return az.PutBucketTagging(ctx, bucket, nil)
+}
+
 func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
 	var opts *azblob.DownloadStreamOptions
 	if *input.Range != "" {
@@ -365,7 +422,7 @@ func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput)
 	return azureErrToS3Err(err)
 }
 
-func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	delResult, errs := []types.DeletedObject{}, []types.Error{}
 	for _, obj := range input.Delete.Objects {
 		err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
@@ -392,7 +449,7 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
 		}
 	}
 
-	return s3response.DeleteObjectsResult{
+	return s3response.DeleteResult{
 		Deleted: delResult,
 		Error:   errs,
 	}, nil

backend/backend.go

@@ -38,7 +38,10 @@ type Backend interface {
 	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
 	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
-
+	PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
+	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
+	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
 	// multipart operations
 	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
@@ -58,14 +61,20 @@ type Backend interface {
 	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
 	DeleteObject(context.Context, *s3.DeleteObjectInput) error
-	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
+	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
 
 	// special case object operations
 	RestoreObject(context.Context, *s3.RestoreObjectInput) error
 	SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
 
-	// object tags operations
+	// bucket tagging operations
+	GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
+	PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
+	DeleteBucketTagging(_ context.Context, bucket string) error
+
+	// object tagging operations
 	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
 	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
 	DeleteObjectTagging(_ context.Context, bucket, object string) error
@@ -104,6 +113,18 @@ func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []
 func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
+func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
 
 func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -154,8 +175,8 @@ func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input)
 func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
-	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -179,6 +200,20 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
 	}
 }
 
+func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
+func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+
 func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }

backend/posix/posix.go

@@ -74,6 +74,12 @@ func New(rootdir string) (*Posix, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
+	_, err = xattr.FGet(f, "user.test")
+	if errors.Is(err, syscall.ENOTSUP) {
+		f.Close()
+		return nil, fmt.Errorf("xattr not supported on %v", rootdir)
+	}
+
 	return &Posix{rootfd: f, rootdir: rootdir}, nil
 }
 
@@ -369,10 +375,9 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
-			if err != nil {
-				return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
-			}
+		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 		}
 	}
 	err = f.link()
@@ -436,7 +441,7 @@ func loadUserMetaData(path string, m map[string]string) (contentType, contentEnc
 			continue
 		}
 		b, err := xattr.Get(path, e)
-		if err == syscall.ENODATA {
+		if err == errNoData {
 			m[strings.TrimPrefix(e, fmt.Sprintf("user.%v.", metaHdr))] = ""
 			continue
 		}
@@ -1189,7 +1194,7 @@ func (p *Posix) removeParents(bucket, object string) error {
 	return nil
 }
 
-func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	// delete object already checks bucket
 	delResult, errs := []types.DeletedObject{}, []types.Error{}
 	for _, obj := range input.Delete.Objects {
@@ -1218,7 +1223,7 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 		}
 	}
 
-	return s3response.DeleteObjectsResult{
+	return s3response.DeleteResult{
 		Deleted: delResult,
 		Error:   errs,
 	}, nil
@@ -1640,7 +1645,15 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
 	}
 	marker := ""
 	if input.ContinuationToken != nil {
-		marker = *input.ContinuationToken
+		if input.StartAfter != nil {
+			if *input.StartAfter > *input.ContinuationToken {
+				marker = *input.StartAfter
+			} else {
+				marker = *input.ContinuationToken
+			}
+		} else {
+			marker = *input.ContinuationToken
+		}
 	}
 	delim := ""
 	if input.Delimiter != nil {
@@ -1720,6 +1733,57 @@ func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]
 	return b, nil
 }
 
+func (p *Posix) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return fmt.Errorf("stat bucket: %w", err)
+	}
+
+	if tags == nil {
+		err = xattr.Remove(bucket, "user."+tagHdr)
+		if err != nil {
+			return fmt.Errorf("remove tags: %w", err)
+		}
+		return nil
+	}
+
+	b, err := json.Marshal(tags)
+	if err != nil {
+		return fmt.Errorf("marshal tags: %w", err)
+	}
+
+	err = xattr.Set(bucket, "user."+tagHdr, b)
+	if err != nil {
+		return fmt.Errorf("set tags: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Posix) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
+	_, err := os.Stat(bucket)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("stat bucket: %w", err)
+	}
+
+	tags, err := p.getXattrTags(bucket, "")
+	if err != nil {
+		return nil, err
+	}
+
+	return tags, nil
+}
+
+func (p *Posix) DeleteBucketTagging(ctx context.Context, bucket string) error {
+	return p.PutBucketTagging(ctx, bucket, nil)
+}
+
 func (p *Posix) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1876,7 +1940,7 @@ func isNoAttr(err error) bool {
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == syscall.ENODATA {
+	if err == errNoData {
 		return true
 	}
 	return false

backend/posix/posix_windows.go

@@ -1,89 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package posix
-
-import (
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-)
-
-type tmpfile struct {
-	f       *os.File
-	bucket  string
-	objname string
-	size    int64
-}
-
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
-	// Create a temp file for upload while in progress (see link comments below).
-	err := os.MkdirAll(dir, 0700)
-	if err != nil {
-		return nil, fmt.Errorf("make temp dir: %w", err)
-	}
-	f, err := os.CreateTemp(dir,
-		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
-	if err != nil {
-		return nil, err
-	}
-	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
-}
-
-func (tmp *tmpfile) link() error {
-	tempname := tmp.f.Name()
-	// cleanup in case anything goes wrong, if rename succeeds then
-	// this will no longer exist
-	defer os.Remove(tempname)
-
-	// We use Rename as the atomic operation for object puts. The upload is
-	// written to a temp file to not conflict with any other simultaneous
-	// uploads. The final operation is to move the temp file into place for
-	// the object. This ensures the object semantics of last upload completed
-	// wins and is not some combination of writes from simultaneous uploads.
-	objPath := filepath.Join(tmp.bucket, tmp.objname)
-	err := os.Remove(objPath)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove stale path: %w", err)
-	}
-
-	err = tmp.f.Close()
-	if err != nil {
-		return fmt.Errorf("close tmpfile: %w", err)
-	}
-
-	err = os.Rename(tempname, objPath)
-	if err != nil {
-		return fmt.Errorf("rename tmpfile: %w", err)
-	}
-
-	return nil
-}
-
-func (tmp *tmpfile) Write(b []byte) (int, error) {
-	if int64(len(b)) > tmp.size {
-		return 0, fmt.Errorf("write exceeds content length")
-	}
-
-	n, err := tmp.f.Write(b)
-	tmp.size -= int64(n)
-	return n, err
-}
-
-func (tmp *tmpfile) cleanup() {
-	tmp.f.Close()
-}

backend/posix/with_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !freebsd && !openbsd && !netbsd
+// +build !freebsd,!openbsd,!netbsd
+
+package posix
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENODATA
+)

backend/posix/with_otmpfile.go

@@ -12,6 +12,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build linux
+// +build linux
+
 package posix
 
 import (

backend/posix/without_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build freebsd || openbsd || netbsd
+// +build freebsd openbsd netbsd
+
+package posix
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENOATTR
+)

backend/posix/without_otmpfile.go

@@ -12,6 +12,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build !linux
+// +build !linux
+
 package posix
 
 import (

backend/s3proxy/s3.go

@@ -320,17 +320,17 @@ func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput)
 	return handleError(err)
 }
 
-func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	if len(input.Delete.Objects) == 0 {
 		input.Delete.Objects = []types.ObjectIdentifier{}
 	}
 
 	output, err := s.client.DeleteObjects(ctx, input)
 	if err != nil {
-		return s3response.DeleteObjectsResult{}, handleError(err)
+		return s3response.DeleteResult{}, handleError(err)
 	}
 
-	return s3response.DeleteObjectsResult{
+	return s3response.DeleteResult{
 		Deleted: output.Deleted,
 		Error:   output.Errors,
 	}, nil

backend/scoutfs/scoutfs.go

@@ -25,7 +25,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -201,10 +200,9 @@ func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteM
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
-			if err != nil {
-				return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
-			}
+		err = mkdirAll(dir, os.FileMode(0755), bucket, object)
+		if err != nil {
+			return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 		}
 	}
 	err = f.link()
@@ -268,7 +266,7 @@ func loadUserMetaData(path string, m map[string]string) (contentType, contentEnc
 			continue
 		}
 		b, err := xattr.Get(path, e)
-		if err == syscall.ENODATA {
+		if err == errNoData {
 			m[strings.TrimPrefix(e, "user.")] = ""
 			continue
 		}
@@ -806,7 +804,7 @@ func isNoAttr(err error) bool {
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == syscall.ENODATA {
+	if err == errNoData {
 		return true
 	}
 	return false

backend/scoutfs/scoutfs_incompat.go

@@ -34,7 +34,7 @@ var (
 	errNotSupported = errors.New("not supported")
 )
 
-func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+func openTmpFile(_, _, _ string, _ int64) (*tmpfile, error) {
 	return nil, errNotSupported
 }
 
@@ -49,10 +49,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
-func moveData(from *os.File, to *os.File) error {
+func moveData(_, _ *os.File) error {
 	return errNotSupported
 }
 
-func statMore(path string) (stat, error) {
+func statMore(_ string) (stat, error) {
 	return stat{}, errNotSupported
 }

backend/scoutfs/with_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !freebsd && !openbsd && !netbsd
+// +build !freebsd,!openbsd,!netbsd
+
+package scoutfs
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENODATA
+)

backend/scoutfs/without_enodata.go

@@ -0,0 +1,24 @@
+// Copyright 2024 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build freebsd || openbsd || netbsd
+// +build freebsd openbsd netbsd
+
+package scoutfs
+
+import "syscall"
+
+var (
+	errNoData = syscall.ENOATTR
+)

cmd/versitygw/admin.go

@@ -164,14 +164,14 @@ func createUser(ctx *cli.Context) error {
 	if access == "" || secret == "" {
 		return fmt.Errorf("invalid input parameters for the new user")
 	}
-	if role != "admin" && role != "user" {
-		return fmt.Errorf("invalid input parameter for role")
+	if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
+		return fmt.Errorf("invalid input parameter for role: %v", role)
 	}
 
 	acc := auth.Account{
 		Access:    access,
 		Secret:    secret,
-		Role:      role,
+		Role:      auth.Role(role),
 		UserID:    userID,
 		GroupID:   groupID,
 		ProjectID: projectID,
@@ -289,11 +289,14 @@ func listUsers(ctx *cli.Context) error {
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
+
 	var accs []auth.Account
 	if err := json.Unmarshal(body, &accs); err != nil {
 		return err
 	}
-	fmt.Println(accs)
 
 	printAcctTable(accs)
 
@@ -402,7 +405,7 @@ func listBuckets(ctx *cli.Context) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf(string(body))
+		return fmt.Errorf("%s", body)
 	}
 
 	var buckets []s3response.Bucket

cmd/versitygw/gateway_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/versity/versitygw/backend/posix"
-	"github.com/versity/versitygw/integration"
+	"github.com/versity/versitygw/tests/integration"
 )
 
 const (

cmd/versitygw/main.go

@@ -42,6 +42,7 @@ var (
 	natsURL, natsTopic                     string
 	logWebhookURL                          string
 	accessLog                              string
+	healthPath                             string
 	debug                                  bool
 	quiet                                  bool
 	iamDir                                 string
@@ -122,6 +123,7 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "port",
 			Usage:       "gateway listen address <ip>:<port> or :<port>",
+			EnvVars:     []string{"VGW_PORT"},
 			Value:       ":7070",
 			Destination: &port,
 			Aliases:     []string{"p"},
@@ -143,6 +145,7 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "region",
 			Usage:       "s3 region string",
+			EnvVars:     []string{"VGW_REGION"},
 			Value:       "us-east-1",
 			Destination: &region,
 			Aliases:     []string{"r"},
@@ -150,184 +153,226 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "cert",
 			Usage:       "TLS cert file",
+			EnvVars:     []string{"VGW_CERT"},
 			Destination: &certFile,
 		},
 		&cli.StringFlag{
 			Name:        "key",
 			Usage:       "TLS key file",
+			EnvVars:     []string{"VGW_KEY"},
 			Destination: &keyFile,
 		},
 		&cli.StringFlag{
 			Name:        "admin-port",
 			Usage:       "gateway admin server listen address <ip>:<port> or :<port>",
+			EnvVars:     []string{"VGW_ADMIN_PORT"},
 			Destination: &admPort,
 			Aliases:     []string{"ap"},
 		},
 		&cli.StringFlag{
 			Name:        "admin-cert",
 			Usage:       "TLS cert file for admin server",
+			EnvVars:     []string{"VGW_ADMIN_CERT"},
 			Destination: &admCertFile,
 		},
 		&cli.StringFlag{
 			Name:        "admin-cert-key",
 			Usage:       "TLS key file for admin server",
+			EnvVars:     []string{"VGW_ADMIN_CERT_KEY"},
 			Destination: &admKeyFile,
 		},
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
+			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
 		&cli.BoolFlag{
 			Name:        "quiet",
 			Usage:       "silence stdout request logging output",
+			EnvVars:     []string{"VGW_QUIET"},
 			Destination: &quiet,
 			Aliases:     []string{"q"},
 		},
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
-			EnvVars:     []string{"LOGFILE"},
+			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
 			Destination: &accessLog,
 		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
-			EnvVars:     []string{"WEBHOOK"},
+			EnvVars:     []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"},
 			Destination: &logWebhookURL,
 		},
 		&cli.StringFlag{
 			Name:        "event-kafka-url",
 			Usage:       "kafka server url to send the bucket notifications.",
+			EnvVars:     []string{"VGW_EVENT_KAFKA_URL"},
 			Destination: &kafkaURL,
 			Aliases:     []string{"eku"},
 		},
 		&cli.StringFlag{
 			Name:        "event-kafka-topic",
 			Usage:       "kafka server pub-sub topic to send the bucket notifications to",
+			EnvVars:     []string{"VGW_EVENT_KAFKA_TOPIC"},
 			Destination: &kafkaTopic,
 			Aliases:     []string{"ekt"},
 		},
 		&cli.StringFlag{
 			Name:        "event-kafka-key",
 			Usage:       "kafka server put-sub topic key to send the bucket notifications to",
+			EnvVars:     []string{"VGW_EVENT_KAFKA_KEY"},
 			Destination: &kafkaKey,
 			Aliases:     []string{"ekk"},
 		},
 		&cli.StringFlag{
 			Name:        "event-nats-url",
 			Usage:       "nats server url to send the bucket notifications",
+			EnvVars:     []string{"VGW_EVENT_NATS_URL"},
 			Destination: &natsURL,
 			Aliases:     []string{"enu"},
 		},
 		&cli.StringFlag{
 			Name:        "event-nats-topic",
 			Usage:       "nats server pub-sub topic to send the bucket notifications to",
+			EnvVars:     []string{"VGW_EVENT_NATS_TOPIC"},
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
 		&cli.StringFlag{
 			Name:        "iam-dir",
 			Usage:       "if defined, run internal iam service within this directory",
+			EnvVars:     []string{"VGW_IAM_DIR"},
 			Destination: &iamDir,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-url",
 			Usage:       "ldap server url to store iam data",
+			EnvVars:     []string{"VGW_IAM_LDAP_URL"},
 			Destination: &ldapURL,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-bind-dn",
 			Usage:       "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
+			EnvVars:     []string{"VGW_IAM_LDAP_BIND_DN"},
 			Destination: &ldapBindDN,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-bind-pass",
 			Usage:       "ldap server user password",
+			EnvVars:     []string{"VGW_IAM_LDAP_BIND_PASS"},
 			Destination: &ldapPassword,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-query-base",
 			Usage:       "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
+			EnvVars:     []string{"VGW_IAM_LDAP_QUERY_BASE"},
 			Destination: &ldapQueryBase,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-object-classes",
 			Usage:       "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
+			EnvVars:     []string{"VGW_IAM_LDAP_OBJECT_CLASSES"},
 			Destination: &ldapObjClasses,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-access-atr",
 			Usage:       "ldap server user access key id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_ACCESS_ATR"},
 			Destination: &ldapAccessAtr,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-secret-atr",
 			Usage:       "ldap server user secret access key attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_SECRET_ATR"},
 			Destination: &ldapSecAtr,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-role-atr",
 			Usage:       "ldap server user role attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
 		&cli.StringFlag{
 			Name:        "s3-iam-access",
 			Usage:       "s3 IAM access key",
+			EnvVars:     []string{"VGW_S3_IAM_ACCESS_KEY"},
 			Destination: &s3IamAccess,
 		},
 		&cli.StringFlag{
 			Name:        "s3-iam-secret",
 			Usage:       "s3 IAM secret key",
+			EnvVars:     []string{"VGW_S3_IAM_SECRET_KEY"},
 			Destination: &s3IamSecret,
 		},
 		&cli.StringFlag{
 			Name:        "s3-iam-region",
 			Usage:       "s3 IAM region",
+			EnvVars:     []string{"VGW_S3_IAM_REGION"},
 			Destination: &s3IamRegion,
 			Value:       "us-east-1",
 		},
 		&cli.StringFlag{
 			Name:        "s3-iam-bucket",
 			Usage:       "s3 IAM bucket",
+			EnvVars:     []string{"VGW_S3_IAM_BUCKET"},
 			Destination: &s3IamBucket,
 		},
 		&cli.StringFlag{
 			Name:        "s3-iam-endpoint",
 			Usage:       "s3 IAM endpoint",
+			EnvVars:     []string{"VGW_S3_IAM_ENDPOINT"},
 			Destination: &s3IamEndpoint,
 		},
 		&cli.BoolFlag{
 			Name:        "s3-iam-noverify",
 			Usage:       "s3 IAM disable ssl verification",
+			EnvVars:     []string{"VGW_S3_IAM_NO_VERIFY"},
 			Destination: &s3IamSslNoVerify,
 		},
 		&cli.BoolFlag{
 			Name:        "s3-iam-debug",
 			Usage:       "s3 IAM debug output",
+			EnvVars:     []string{"VGW_S3_IAM_DEBUG"},
 			Destination: &s3IamDebug,
 		},
 		&cli.BoolFlag{
 			Name:        "iam-cache-disable",
 			Usage:       "disable local iam cache",
+			EnvVars:     []string{"VGW_IAM_CACHE_DISABLE"},
 			Destination: &iamCacheDisable,
 		},
 		&cli.IntFlag{
 			Name:        "iam-cache-ttl",
 			Usage:       "local iam cache entry ttl (seconds)",
+			EnvVars:     []string{"VGW_IAM_CACHE_TTL"},
 			Value:       120,
 			Destination: &iamCacheTTL,
 		},
 		&cli.IntFlag{
 			Name:        "iam-cache-prune",
 			Usage:       "local iam cache cleanup interval (seconds)",
+			EnvVars:     []string{"VGW_IAM_CACHE_PRUNE"},
 			Value:       3600,
 			Destination: &iamCachePrune,
 		},
+		&cli.StringFlag{
+			Name: "health",
+			Usage: `health check endpoint path. Health endpoint will be configured on GET http method: GET <health>
+					NOTICE: the path has to be specified with '/'. e.g /health`,
+			EnvVars:     []string{"VGW_HEALTH"},
+			Destination: &healthPath,
+		},
 	}
 }
 
 func runGateway(ctx context.Context, be backend.Backend) error {
+	if rootUserAccess == "" || rootUserSecret == "" {
+		return fmt.Errorf("root user access and secret key must be provided")
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:           "versitygw",
 		ServerHeader:      "VERSITYGW",
@@ -360,6 +405,9 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if quiet {
 		opts = append(opts, s3api.WithQuiet())
 	}
+	if healthPath != "" {
+		opts = append(opts, s3api.WithHealth(healthPath))
+	}
 
 	admApp := fiber.New(fiber.Config{
 		AppName:      "versitygw",
@@ -448,7 +496,6 @@ Loop:
 	for {
 		select {
 		case <-ctx.Done():
-			err = ctx.Err()
 			break Loop
 		case err = <-c:
 			break Loop
@@ -468,12 +515,18 @@ Loop:
 
 	err = iam.Shutdown()
 	if err != nil {
+		if saveErr == nil {
+			saveErr = err
+		}
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
 	if logger != nil {
 		err := logger.Shutdown()
 		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
 			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
 		}
 	}

cmd/versitygw/s3.go

@@ -44,6 +44,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server access key id",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_ACCESS_KEY"},
 				Destination: &s3proxyAccess,
 				Aliases:     []string{"a"},
 			},
@@ -52,6 +53,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server secret access key",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_SECRET_KEY"},
 				Destination: &s3proxySecret,
 				Aliases:     []string{"s"},
 			},
@@ -59,23 +61,27 @@ to an s3 storage backend service.`,
 				Name:        "endpoint",
 				Usage:       "s3 service endpoint, default AWS if not specified",
 				Value:       "",
+				EnvVars:     []string{"VGW_S3_ENDPOINT"},
 				Destination: &s3proxyEndpoint,
 			},
 			&cli.StringFlag{
 				Name:        "region",
 				Usage:       "s3 service region, default 'us-east-1' if not specified",
 				Value:       "us-east-1",
+				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
+				EnvVars:     []string{"VGW_S3_SSL_SKIP_VERIFY"},
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
@@ -83,6 +89,7 @@ to an s3 storage backend service.`,
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DEBUG"},
 				Destination: &s3proxyDebug,
 			},
 		},

cmd/versitygw/scoutfs.go

@@ -48,6 +48,7 @@ move interfaces as well as support for tiered filesystems.`,
 				Name:        "glacier",
 				Usage:       "enable glacier emulation mode",
 				Aliases:     []string{"g"},
+				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
 		},

cmd/versitygw/test.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/integration"
+	"github.com/versity/versitygw/tests/integration"
 )
 
 var (
@@ -79,6 +79,11 @@ func initTestCommands() []*cli.Command {
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
 		},
+		{
+			Name:   "iam",
+			Usage:  "Tests iam service",
+			Action: getAction(integration.TestIAM),
+		},
 		{
 			Name:  "bench",
 			Usage: "Runs download/upload performance test on the gateway",
@@ -268,11 +273,11 @@ func getAction(tf testFunc) func(*cli.Context) error {
 func extractIntTests() (commands []*cli.Command) {
 	tests := integration.GetIntTests()
 	for key, val := range tests {
-		testKey := key
+		k := key
 		testFunc := val
 		commands = append(commands, &cli.Command{
-			Name:  testKey,
-			Usage: fmt.Sprintf("Runs %v integration test", testKey),
+			Name:  k,
+			Usage: fmt.Sprintf("Runs %v integration test", key),
 			Action: func(ctx *cli.Context) error {
 				opts := []integration.Option{
 					integration.WithAccess(awsID),

docker-compose.yml

@@ -27,6 +27,11 @@ services:
       - "10000:10000"
       - "10001:10001"
       - "10002:10002"
+    restart: always
+    hostname: azurite
+    command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
+    volumes:
+      - ./certs:/certs
   azuritegw:
     build:
       context: .
@@ -35,4 +40,4 @@ services:
       - ./:/app
     ports:
       - 7070:7070
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url http://azurite:10000/$AZ_ACCOUNT_NAME"]
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]

extra/example.conf

@@ -0,0 +1,272 @@
+###################################
+# VersityGW systemd configuration #
+###################################
+
+# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
+# For example, if the service name is "mygateway", then the file should be
+# named /etc/versitygw.d/mygateway.conf.
+# The systemd template file /lib/systemd/system/versitygw@.service will
+# automatically load the configuration file for the service name.
+# To start the gateway, use the following command:
+# systemctl start versitygw@mygateway
+# To enable the gateway to start on boot, use the following command:
+# systemctl enable versitygw@mygateway
+# To stop the gateway, use the following command:
+# systemctl stop versitygw@mygateway
+
+# There can be multiple gateway services running on the same host. Each
+# gateway service must have a unique service name with a unique configuration
+# file in /etc/versitygw.d/. They must also listen on different ports and/or
+# interfaces using the VGW_PORT option.
+
+##############################
+# VersityGW Required Options #
+##############################
+
+# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3
+# This defines the backend that the VGW will use for data access.
+VGW_BACKEND=posix
+
+# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
+# as the the top level directory for the gateway.
+# All sub directories of the top level directory are treated as buckets,
+# and all files/directories below the "bucket directory" are treated as
+# the objects. The object name is split on "/" separator to translate
+# to posix storage.
+# For example:
+# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+VGW_BACKEND_ARG=
+
+############################
+# VersityGW Global Options #
+############################
+
+# commented options are the default values
+
+# The following must be set, and do not have default values
+# The access and secret options will specify the root account credentials.
+# The root account is granted full authorization to all API requests after
+# authentication.
+ROOT_ACCESS_KEY_ID=
+ROOT_SECRET_ACCESS_KEY=
+
+# The following are optional, and have the default values as listed
+
+# The VGW_PORT option will specify the listening port for the S3 server.
+# This option can use either the form <ip>:<port> which will listen only
+# on the network interface that matches the IP on the specified port, or
+# :<port> which will listen on all network interfaces on the specified port.
+# The <ip> spec can either be IP dotted notation or a resolvable hostname.
+# The <port> spec can either be a numeric port or the service name typically
+# in /etc/services.
+#VGW_PORT=:7070
+
+# The VGW_REGION option will specify the region that the S3 server will
+# report to clients. This option is optional, and defaults to "us-east-1".
+#VGW_REGION=us-east-1
+
+# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
+# private key that the S3 server will use for SSL connections. This option
+# is optional, and defaults to not using SSL.
+#VGW_CERT=
+#VGW_KEY=
+
+# The VGW_ADMIN_PORT option will specify the listening port for the admin
+# server. The admin server endpoint can optionally be set to listen on a
+# different interface or port than the S3 service. This allows for better
+# control of firewall restrictions to the admin endpoint. The certs for this
+# can be different certs than specified for the S3 service. The default when
+# these are not specified is to have the admin server listen on the same
+# endpoint as the S3 service.
+# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
+# server will use SSL.
+#VGW_ADMIN_PORT=
+#VGW_ADMIN_CERT=
+#VGW_ADMIN_CERT_KEY=
+
+# The VGW_QUIET option when set will supress the S3 server request summary
+# logging to stdout.
+#VGW_QUIET=false
+
+# The VGW_HEALTH option when set will specify the URL to accept health checks
+# on. The health check endpoint is often used for load balancers to verify
+# gateway is alive. The health endpoint masks any bucket with this setting.
+# For example, if the health endpoint is set to /health, the gateway will not
+# allow creating or listing contents of a bucket called "health". The health
+# endpoint is unauthenticated, and returns a 200 status for GET.
+#VGW_HEALTH=
+
+###############
+# Access Logs #
+###############
+
+# The VGW_ACCESS_LOG option when set will specify the file to log all S3
+# server requests to. This option is optional, and defaults to not logging.
+# It is suggested to use absolute paths for the server log file because the
+# server may chdir into the backend root directory and change locations for
+# relative paths.
+# The log file format follows the AWS S3 access log format documented in
+# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
+#VGW_ACCESS_LOG=
+
+# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
+# S3 server request access logs to. The access logs are JSON encoded when
+# sent to the webhook.
+#VGW_LOG_WEBHOOK_URL=
+
+##############
+# Event Logs #
+##############
+
+# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
+# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
+# bucket events will be sent to the kafka service. The gateway events are
+# similar to AWS S3 events, and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_KAFKA_URL=
+#VGW_EVENT_KAFKA_TOPIC=
+#VGW_EVENT_KAFKA_KEY=
+
+# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
+# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the
+# the NATS messaging service. The gateway events are  similar to AWS S3 events,
+# and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_NATS_URL=
+#VGW_EVENT_NATS_TOPIC=
+
+# The VGW_DEBUG option enables verbose debug log output to stdout. This output
+# includes details for signature verification steps. This is generally only
+# useful for debugging the S3 server, and should not be used in production.
+#VGW_DEBUG=false
+
+################
+# IAM services #
+################
+
+# The VGW_IAM_DIR option will enable the internal IAM service with accounts
+# stored in a file under the specified directory. This is provided to minimize
+# dependencies on outside services for basic functionality. The local account
+# files are plain text and only protected with file permissions. This IAM
+# service is added for convenience, but is not considered as secure or scalable
+# as a dedicated IAM service.
+#VGW_IAM_DIR=
+
+# The ldap options will enable the LDAP IAM service with accounts stored in an
+# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
+# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
+# secret credentials and role respectively. The other options are used to
+# connect to the LDAP service.
+#VGW_IAM_LDAP_URL=
+#VGW_IAM_LDAP_BASE_DN=
+#VGW_IAM_LDAP_BIND_DN=
+#VGW_IAM_LDAP_BIND_PASS=
+#VGW_IAM_LDAP_QUERY_BASE=
+#VGW_IAM_LDAP_OBJECT_CLASSES=
+#VGW_IAM_LDAP_ACCESS_ATR=
+#VGW_IAM_LDAP_SECRET_ATR=
+#VGW_IAM_LDAP_ROLE_ATR=
+
+# The VGW_S3 IAM service is similar to the internal IAM service, but instead
+# stores the account information JSON encoded in an S3 object. This should use
+# a bucket that is not accessible to general users when using s3 backend to
+# prevent access to account credentials. This IAM service is added for
+# convenience, but is not considered as secure or scalable as a dedicated IAM
+# service.
+#VGW_S3_IAM_ACCESS_KEY=
+#VGW_S3_IAM_SECRET_KEY=
+#VGW_S3_IAM_REGION=
+#VGW_S3_IAM_ENDPOINT=
+#VGW_S3_IAM_BUCKET=
+#VGW_S3_IAM_NO_VERIFY=
+
+###############
+# IAM caching #
+###############
+
+# The IAM cache is intended to ease the load on the IAM service and increase
+# the Gateway performance by caching accounts and credentials for the TTL time
+# interval. Disabling this will cause a request to the configured IAM service
+# for each incoming request to retrieve the corresponding account credentials.
+# The cache is enabled by default. The TTL specifies how long to cache
+# credentials, and the prune value determines the interval for expired entries
+# to be removed from the cache. Increasing the TTL may lessen the load on the
+# IAM service backend, but may have out of date account info until the next
+# interval. Increasing the prune value may reduce memory use at the cost of
+# added CPU to check cache expirations.
+#VGW_IAM_CACHE_DISABLE=false
+#VGW_IAM_CACHE_TTL=120
+#VGW_IAM_CACHE_PRUNE=3600
+
+######################################
+# VersityGW Backend Specific Options #
+######################################
+
+#########
+# posix #
+#########
+
+# The posix backend translates S3 requests to file access in a local filesystem.
+# The posix backend requires a filesystem that supports extended attributes.
+# The top level directory for the gateway must be provided. All sub directories
+# of the top level directory are treated as buckets, and all files/directories
+# below the "bucket directory" are treated as the objects. The object
+# name is split on "/" separator to translate to posix storage.
+# For example:
+# top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+
+# There are currently no further options other than VGW_BACKEND_ARG for the
+# posix backend.
+
+###########
+# scoutfs #
+###########
+
+# The scoutfs backend requires a ScoutFS filesystem type for the backend
+# path. The glacier mode functionality requires ScoutAM to be configured
+# for tiering data from the ScoutFS filesystem to a mass stroage system.
+# The mass storage system is often one or more tape libraries. Due to the
+# high latency of tape, the glacier mode functionality is designed to
+# give feedback to clients about object state and offer ability to request
+# data to be staged back to disk without the client dealing with long
+# request timeout settings.
+
+# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
+# GET object:  if file offline, return invalid object state
+# HEAD object: if file offline, set obj storage class to GLACIER
+#              if file offline and staging, x-amz-restore: ongoing-request="true"
+#              if file offline and not staging, x-amz-restore: ongoing-request="false"
+#              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
+#              note: this expiry-date is not used but provided for client glacier compatibility
+# ListObjects: if file offline, set obj storage class to GLACIER
+# RestoreObject: add batch stage request to file
+#VGW_SCOUTFS_GLACIER=false
+
+######
+# s3 #
+######
+
+# The s3 backend allows the gateway to forward requests to an S3 compatible
+# service. This allows the gateway to act as a proxy for another S3 service.
+# The backend S3 access is all done with a single configured account. The
+# gateway will manage incoming multi-tenant access with the gateway configured
+# IAM service. This gives stroage admins the ability to manage local gateway
+# accounts while maintaining full control and a single account for the backend
+# S3 service.
+
+# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
+# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
+# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
+#VGW_S3_ACCESS_KEY=
+#VGW_S3_SECRET_KEY=
+#VGW_S3_REGION=
+#VGW_S3_ENDPOINT=
+#VGW_S3_DISABLE_CHECKSUM=false
+#VGW_S3_SSL_SKIP_VERIFY=false
+#VGW_S3_DEBUG=false

extra/posttrans.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+systemctl daemon-reload

extra/versitygw@.service

@@ -0,0 +1,33 @@
+[Unit]
+Description=VersityGW
+Documentation=https://github.com/versity/versitygw/wiki
+Wants=network-online.target
+After=network-online.target remote-fs.target
+AssertFileIsExecutable=/usr/bin/versitygw
+AssertPathExists=/etc/versitygw.d/%i.conf
+
+[Service]
+WorkingDirectory=/root
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=versitygw-%i
+
+User=root
+Group=root
+
+EnvironmentFile=/etc/versitygw.d/%i.conf
+
+ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"'
+
+# Let systemd restart this service always
+Restart=always
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65536
+
+# Specifies the maximum number of threads this process can create
+TasksMax=infinity
+
+[Install]
+WantedBy=multi-user.target
+

go.mod

@@ -1,35 +1,36 @@
 module github.com/versity/versitygw
 
-go 1.20
+go 1.21
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1
-	github.com/aws/aws-sdk-go-v2 v1.24.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
-	github.com/aws/smithy-go v1.19.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
+	github.com/aws/aws-sdk-go-v2 v1.26.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
+	github.com/aws/smithy-go v1.20.1
 	github.com/go-ldap/ldap/v3 v3.4.6
-	github.com/gofiber/fiber/v2 v2.52.0
+	github.com/gofiber/fiber/v2 v2.52.2
+	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/nats-io/nats.go v1.32.0
+	github.com/nats-io/nats.go v1.33.1
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/urfave/cli/v2 v2.27.1
-	github.com/valyala/fasthttp v1.51.0
+	github.com/valyala/fasthttp v1.52.0
 	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.16.0
+	golang.org/x/sys v0.18.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -38,26 +39,26 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	golang.org/x/crypto v0.18.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.26.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.16
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.8
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.8
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect

go.sum

@@ -1,73 +1,76 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
-github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
-github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
-github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o=
-github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15 h1:2MUXyGW6dVaQz6aqycpbdLIH1NMcUI6kW6vQ0RabGYg=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15/go.mod h1:aHbhbR6WEQgHAiRj41EQ2W47yOYwNtIkWTXmcAtYqj8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10/go.mod h1:FHbKWQtRBYUz4vO5WBWjzMD2by126ny5y/1EoaWoLfI=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 h1:L0ai8WICYHozIKK+OtPzVJBugL7culcuM4E4JOpIEm8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10/go.mod h1:byqfyxJBshFk0fF9YmK0M0ugIO8OWjzH2T3bPG4eGuA=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 h1:KOxnQeWy5sXyS37fdKEvAsGHOr9fa/qvwxfJurR/BzE=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10/go.mod h1:jMx5INQFYFYB3lQD9W0D8Ohgq6Wnl7NYOJ2TQndbulI=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1 h1:5XNlsBsEvBZBMO6p82y+sqpWg8j5aBCe+5C2GBFgqBQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
-github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
-github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
+github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
+github.com/aws/aws-sdk-go-v2/config v1.27.8 h1:0r8epOsiJ7YJz65MGcb8i91ehFp4kvvFe2qkq5oYeRI=
+github.com/aws/aws-sdk-go-v2/config v1.27.8/go.mod h1:XsmYKxYNuIhLsFddpNds+j9H5XKzjWDdg/SZngiwFio=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.8 h1:WUdNLXbyNbU07V/WFrSOBXqZTDgmmMNMgUFzpYOKJhw=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.8/go.mod h1:iPZzLpaBIfhyvVS/XGD3JvR1GP3YdHTqpySKDlqkfs8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 h1:S+L2QSKhUuShih3aq9P/mkzDBiOO5tTyVg+vXREfsfg=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12 h1:rfAytUY7OgbOMDkzxdiigZkbTe9SDER2dIpO/Fzi9+0=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12/go.mod h1:BaY3WWSgUwV/zq0K3HePyXhRYZxGnDATYERkR0f1RTs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
+github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
+github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
 github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
-github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE=
-github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo=
+github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
 github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -76,8 +79,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -87,8 +90,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/nats-io/nats.go v1.32.0 h1:Bx9BZS+aXYlxW08k8Gd3yR2s73pV5XSoAQUyp1Kwvp0=
-github.com/nats-io/nats.go v1.32.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
+github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -114,12 +117,13 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
 github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
-github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
+github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
@@ -137,8 +141,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
-golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -147,8 +151,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -165,8 +169,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -190,6 +194,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration/group-tests.go

@@ -1,355 +0,0 @@
-package integration
-
-func TestAuthentication(s *S3Conf) {
-	Authentication_empty_auth_header(s)
-	Authentication_invalid_auth_header(s)
-	Authentication_unsupported_signature_version(s)
-	Authentication_malformed_credentials(s)
-	Authentication_malformed_credentials_invalid_parts(s)
-	Authentication_credentials_terminated_string(s)
-	Authentication_credentials_incorrect_service(s)
-	Authentication_credentials_incorrect_region(s)
-	Authentication_credentials_invalid_date(s)
-	Authentication_credentials_future_date(s)
-	Authentication_credentials_past_date(s)
-	Authentication_credentials_non_existing_access_key(s)
-	Authentication_invalid_signed_headers(s)
-	Authentication_missing_date_header(s)
-	Authentication_invalid_date_header(s)
-	Authentication_date_mismatch(s)
-	Authentication_incorrect_payload_hash(s)
-	Authentication_incorrect_md5(s)
-	Authentication_signature_error_incorrect_secret_key(s)
-}
-
-func TestCreateBucket(s *S3Conf) {
-	CreateBucket_invalid_bucket_name(s)
-	CreateBucket_existing_bucket(s)
-	CreateBucket_as_user(s)
-	CreateDeleteBucket_success(s)
-}
-
-func TestHeadBucket(s *S3Conf) {
-	HeadBucket_non_existing_bucket(s)
-	HeadBucket_success(s)
-}
-
-func TestListBuckets(s *S3Conf) {
-	ListBuckets_as_user(s)
-	ListBuckets_as_admin(s)
-	ListBuckets_success(s)
-}
-
-func TestDeleteBucket(s *S3Conf) {
-	DeleteBucket_non_existing_bucket(s)
-	DeleteBucket_non_empty_bucket(s)
-	DeleteBucket_success_status_code(s)
-}
-
-func TestPutObject(s *S3Conf) {
-	PutObject_non_existing_bucket(s)
-	PutObject_special_chars(s)
-	PutObject_invalid_long_tags(s)
-	PutObject_success(s)
-	PutObject_invalid_credentials(s)
-}
-
-func TestHeadObject(s *S3Conf) {
-	HeadObject_non_existing_object(s)
-	HeadObject_success(s)
-}
-
-func TestGetObject(s *S3Conf) {
-	GetObject_non_existing_key(s)
-	GetObject_invalid_ranges(s)
-	GetObject_with_meta(s)
-	GetObject_success(s)
-	GetObject_by_range_success(s)
-}
-
-func TestListObjects(s *S3Conf) {
-	ListObjects_non_existing_bucket(s)
-	ListObjects_with_prefix(s)
-	ListObject_truncated(s)
-	ListObjects_invalid_max_keys(s)
-	ListObjects_max_keys_0(s)
-	ListObjects_delimiter(s)
-	ListObjects_max_keys_none(s)
-	ListObjects_marker_not_from_obj_list(s)
-}
-
-func TestDeleteObject(s *S3Conf) {
-	DeleteObject_non_existing_object(s)
-	DeleteObject_success(s)
-	DeleteObject_success_status_code(s)
-}
-
-func TestDeleteObjects(s *S3Conf) {
-	DeleteObjects_empty_input(s)
-	DeleteObjects_non_existing_objects(s)
-	DeleteObjects_success(s)
-}
-
-func TestCopyObject(s *S3Conf) {
-	CopyObject_non_existing_dst_bucket(s)
-	CopyObject_not_owned_source_bucket(s)
-	CopyObject_copy_to_itself(s)
-	CopyObject_to_itself_with_new_metadata(s)
-	CopyObject_success(s)
-}
-
-func TestPutObjectTagging(s *S3Conf) {
-	PutObjectTagging_non_existing_object(s)
-	PutObjectTagging_long_tags(s)
-	PutObjectTagging_success(s)
-}
-
-func TestGetObjectTagging(s *S3Conf) {
-	GetObjectTagging_non_existing_object(s)
-	GetObjectTagging_success(s)
-}
-
-func TestDeleteObjectTagging(s *S3Conf) {
-	DeleteObjectTagging_non_existing_object(s)
-	DeleteObjectTagging_success_status(s)
-	DeleteObjectTagging_success(s)
-}
-
-func TestCreateMultipartUpload(s *S3Conf) {
-	CreateMultipartUpload_non_existing_bucket(s)
-	CreateMultipartUpload_success(s)
-}
-
-func TestUploadPart(s *S3Conf) {
-	UploadPart_non_existing_bucket(s)
-	UploadPart_invalid_part_number(s)
-	UploadPart_non_existing_key(s)
-	UploadPart_non_existing_mp_upload(s)
-	UploadPart_success(s)
-}
-
-func TestUploadPartCopy(s *S3Conf) {
-	UploadPartCopy_non_existing_bucket(s)
-	UploadPartCopy_incorrect_uploadId(s)
-	UploadPartCopy_incorrect_object_key(s)
-	UploadPartCopy_invalid_part_number(s)
-	UploadPartCopy_invalid_copy_source(s)
-	UploadPartCopy_non_existing_source_bucket(s)
-	UploadPartCopy_non_existing_source_object_key(s)
-	UploadPartCopy_success(s)
-	UploadPartCopy_by_range_invalid_range(s)
-	UploadPartCopy_greater_range_than_obj_size(s)
-	UploadPartCopy_by_range_success(s)
-}
-
-func TestListParts(s *S3Conf) {
-	ListParts_incorrect_uploadId(s)
-	ListParts_incorrect_object_key(s)
-	ListParts_success(s)
-}
-
-func TestListMultipartUploads(s *S3Conf) {
-	ListMultipartUploads_non_existing_bucket(s)
-	ListMultipartUploads_empty_result(s)
-	ListMultipartUploads_invalid_max_uploads(s)
-	ListMultipartUploads_max_uploads(s)
-	ListMultipartUploads_incorrect_next_key_marker(s)
-	ListMultipartUploads_ignore_upload_id_marker(s)
-	ListMultipartUploads_success(s)
-}
-
-func TestAbortMultipartUpload(s *S3Conf) {
-	AbortMultipartUpload_non_existing_bucket(s)
-	AbortMultipartUpload_incorrect_uploadId(s)
-	AbortMultipartUpload_incorrect_object_key(s)
-	AbortMultipartUpload_success(s)
-	AbortMultipartUpload_success_status_code(s)
-}
-
-func TestCompleteMultipartUpload(s *S3Conf) {
-	CompletedMultipartUpload_non_existing_bucket(s)
-	CompleteMultipartUpload_invalid_part_number(s)
-	CompleteMultipartUpload_invalid_ETag(s)
-	CompleteMultipartUpload_success(s)
-}
-
-func TestPutBucketAcl(s *S3Conf) {
-	PutBucketAcl_non_existing_bucket(s)
-	PutBucketAcl_invalid_acl_canned_and_acp(s)
-	PutBucketAcl_invalid_acl_canned_and_grants(s)
-	PutBucketAcl_invalid_acl_acp_and_grants(s)
-	PutBucketAcl_invalid_owner(s)
-	PutBucketAcl_success_access_denied(s)
-	PutBucketAcl_success_grants(s)
-	PutBucketAcl_success_canned_acl(s)
-	PutBucketAcl_success_acp(s)
-}
-
-func TestGetBucketAcl(s *S3Conf) {
-	GetBucketAcl_non_existing_bucket(s)
-	GetBucketAcl_access_denied(s)
-	GetBucketAcl_success(s)
-}
-
-func TestFullFlow(s *S3Conf) {
-	TestAuthentication(s)
-	TestCreateBucket(s)
-	TestHeadBucket(s)
-	TestListBuckets(s)
-	TestDeleteBucket(s)
-	TestPutObject(s)
-	TestHeadObject(s)
-	TestGetObject(s)
-	TestListObjects(s)
-	TestDeleteObject(s)
-	TestDeleteObjects(s)
-	TestCopyObject(s)
-	TestPutObjectTagging(s)
-	TestDeleteObjectTagging(s)
-	TestCreateMultipartUpload(s)
-	TestUploadPart(s)
-	TestUploadPartCopy(s)
-	TestListParts(s)
-	TestListMultipartUploads(s)
-	TestAbortMultipartUpload(s)
-	TestCompleteMultipartUpload(s)
-	TestPutBucketAcl(s)
-	TestGetBucketAcl(s)
-}
-
-func TestPosix(s *S3Conf) {
-	PutObject_overwrite_dir_obj(s)
-	PutObject_overwrite_file_obj(s)
-	PutObject_dir_obj_with_data(s)
-	CreateMultipartUpload_dir_obj(s)
-}
-
-type IntTests map[string]func(s *S3Conf) error
-
-func GetIntTests() IntTests {
-	return IntTests{
-		"Authentication_empty_auth_header":                    Authentication_empty_auth_header,
-		"Authentication_invalid_auth_header":                  Authentication_invalid_auth_header,
-		"Authentication_unsupported_signature_version":        Authentication_unsupported_signature_version,
-		"Authentication_malformed_credentials":                Authentication_malformed_credentials,
-		"Authentication_malformed_credentials_invalid_parts":  Authentication_malformed_credentials_invalid_parts,
-		"Authentication_credentials_terminated_string":        Authentication_credentials_terminated_string,
-		"Authentication_credentials_incorrect_service":        Authentication_credentials_incorrect_service,
-		"Authentication_credentials_incorrect_region":         Authentication_credentials_incorrect_region,
-		"Authentication_credentials_invalid_date":             Authentication_credentials_invalid_date,
-		"Authentication_credentials_future_date":              Authentication_credentials_future_date,
-		"Authentication_credentials_past_date":                Authentication_credentials_past_date,
-		"Authentication_credentials_non_existing_access_key":  Authentication_credentials_non_existing_access_key,
-		"Authentication_invalid_signed_headers":               Authentication_invalid_signed_headers,
-		"Authentication_missing_date_header":                  Authentication_missing_date_header,
-		"Authentication_invalid_date_header":                  Authentication_invalid_date_header,
-		"Authentication_date_mismatch":                        Authentication_date_mismatch,
-		"Authentication_incorrect_payload_hash":               Authentication_incorrect_payload_hash,
-		"Authentication_incorrect_md5":                        Authentication_incorrect_md5,
-		"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
-		"CreateBucket_invalid_bucket_name":                    CreateBucket_invalid_bucket_name,
-		"CreateBucket_existing_bucket":                        CreateBucket_existing_bucket,
-		"CreateBucket_as_user":                                CreateBucket_as_user,
-		"CreateDeleteBucket_success":                          CreateDeleteBucket_success,
-		"HeadBucket_non_existing_bucket":                      HeadBucket_non_existing_bucket,
-		"HeadBucket_success":                                  HeadBucket_success,
-		"ListBuckets_as_user":                                 ListBuckets_as_user,
-		"ListBuckets_as_admin":                                ListBuckets_as_admin,
-		"ListBuckets_success":                                 ListBuckets_success,
-		"DeleteBucket_non_existing_bucket":                    DeleteBucket_non_existing_bucket,
-		"DeleteBucket_non_empty_bucket":                       DeleteBucket_non_empty_bucket,
-		"DeleteBucket_success_status_code":                    DeleteBucket_success_status_code,
-		"PutObject_non_existing_bucket":                       PutObject_non_existing_bucket,
-		"PutObject_special_chars":                             PutObject_special_chars,
-		"PutObject_invalid_long_tags":                         PutObject_invalid_long_tags,
-		"PutObject_success":                                   PutObject_success,
-		"PutObject_invalid_credentials":                       PutObject_invalid_credentials,
-		"HeadObject_non_existing_object":                      HeadObject_non_existing_object,
-		"HeadObject_success":                                  HeadObject_success,
-		"GetObject_non_existing_key":                          GetObject_non_existing_key,
-		"GetObject_invalid_ranges":                            GetObject_invalid_ranges,
-		"GetObject_with_meta":                                 GetObject_with_meta,
-		"GetObject_success":                                   GetObject_success,
-		"GetObject_by_range_success":                          GetObject_by_range_success,
-		"ListObjects_non_existing_bucket":                     ListObjects_non_existing_bucket,
-		"ListObjects_with_prefix":                             ListObjects_with_prefix,
-		"ListObject_truncated":                                ListObject_truncated,
-		"ListObjects_invalid_max_keys":                        ListObjects_invalid_max_keys,
-		"ListObjects_max_keys_0":                              ListObjects_max_keys_0,
-		"ListObjects_delimiter":                               ListObjects_delimiter,
-		"ListObjects_max_keys_none":                           ListObjects_max_keys_none,
-		"ListObjects_marker_not_from_obj_list":                ListObjects_marker_not_from_obj_list,
-		"DeleteObject_non_existing_object":                    DeleteObject_non_existing_object,
-		"DeleteObject_success":                                DeleteObject_success,
-		"DeleteObject_success_status_code":                    DeleteObject_success_status_code,
-		"DeleteObjects_empty_input":                           DeleteObjects_empty_input,
-		"DeleteObjects_non_existing_objects":                  DeleteObjects_non_existing_objects,
-		"DeleteObjects_success":                               DeleteObjects_success,
-		"CopyObject_non_existing_dst_bucket":                  CopyObject_non_existing_dst_bucket,
-		"CopyObject_not_owned_source_bucket":                  CopyObject_not_owned_source_bucket,
-		"CopyObject_copy_to_itself":                           CopyObject_copy_to_itself,
-		"CopyObject_to_itself_with_new_metadata":              CopyObject_to_itself_with_new_metadata,
-		"CopyObject_success":                                  CopyObject_success,
-		"PutObjectTagging_non_existing_object":                PutObjectTagging_non_existing_object,
-		"PutObjectTagging_long_tags":                          PutObjectTagging_long_tags,
-		"PutObjectTagging_success":                            PutObjectTagging_success,
-		"GetObjectTagging_non_existing_object":                GetObjectTagging_non_existing_object,
-		"GetObjectTagging_success":                            GetObjectTagging_success,
-		"DeleteObjectTagging_non_existing_object":             DeleteObjectTagging_non_existing_object,
-		"DeleteObjectTagging_success_status":                  DeleteObjectTagging_success_status,
-		"DeleteObjectTagging_success":                         DeleteObjectTagging_success,
-		"CreateMultipartUpload_non_existing_bucket":           CreateMultipartUpload_non_existing_bucket,
-		"CreateMultipartUpload_success":                       CreateMultipartUpload_success,
-		"UploadPart_non_existing_bucket":                      UploadPart_non_existing_bucket,
-		"UploadPart_invalid_part_number":                      UploadPart_invalid_part_number,
-		"UploadPart_non_existing_key":                         UploadPart_non_existing_key,
-		"UploadPart_non_existing_mp_upload":                   UploadPart_non_existing_mp_upload,
-		"UploadPart_success":                                  UploadPart_success,
-		"UploadPartCopy_non_existing_bucket":                  UploadPartCopy_non_existing_bucket,
-		"UploadPartCopy_incorrect_uploadId":                   UploadPartCopy_incorrect_uploadId,
-		"UploadPartCopy_incorrect_object_key":                 UploadPartCopy_incorrect_object_key,
-		"UploadPartCopy_invalid_part_number":                  UploadPartCopy_invalid_part_number,
-		"UploadPartCopy_invalid_copy_source":                  UploadPartCopy_invalid_copy_source,
-		"UploadPartCopy_non_existing_source_bucket":           UploadPartCopy_non_existing_source_bucket,
-		"UploadPartCopy_non_existing_source_object_key":       UploadPartCopy_non_existing_source_object_key,
-		"UploadPartCopy_success":                              UploadPartCopy_success,
-		"UploadPartCopy_by_range_invalid_range":               UploadPartCopy_by_range_invalid_range,
-		"UploadPartCopy_greater_range_than_obj_size":          UploadPartCopy_greater_range_than_obj_size,
-		"UploadPartCopy_by_range_success":                     UploadPartCopy_by_range_success,
-		"ListParts_incorrect_uploadId":                        ListParts_incorrect_uploadId,
-		"ListParts_incorrect_object_key":                      ListParts_incorrect_object_key,
-		"ListParts_success":                                   ListParts_success,
-		"ListMultipartUploads_non_existing_bucket":            ListMultipartUploads_non_existing_bucket,
-		"ListMultipartUploads_empty_result":                   ListMultipartUploads_empty_result,
-		"ListMultipartUploads_invalid_max_uploads":            ListMultipartUploads_invalid_max_uploads,
-		"ListMultipartUploads_max_uploads":                    ListMultipartUploads_max_uploads,
-		"ListMultipartUploads_incorrect_next_key_marker":      ListMultipartUploads_incorrect_next_key_marker,
-		"ListMultipartUploads_ignore_upload_id_marker":        ListMultipartUploads_ignore_upload_id_marker,
-		"ListMultipartUploads_success":                        ListMultipartUploads_success,
-		"AbortMultipartUpload_non_existing_bucket":            AbortMultipartUpload_non_existing_bucket,
-		"AbortMultipartUpload_incorrect_uploadId":             AbortMultipartUpload_incorrect_uploadId,
-		"AbortMultipartUpload_incorrect_object_key":           AbortMultipartUpload_incorrect_object_key,
-		"AbortMultipartUpload_success":                        AbortMultipartUpload_success,
-		"AbortMultipartUpload_success_status_code":            AbortMultipartUpload_success_status_code,
-		"CompletedMultipartUpload_non_existing_bucket":        CompletedMultipartUpload_non_existing_bucket,
-		"CompleteMultipartUpload_invalid_part_number":         CompleteMultipartUpload_invalid_part_number,
-		"CompleteMultipartUpload_invalid_ETag":                CompleteMultipartUpload_invalid_ETag,
-		"CompleteMultipartUpload_success":                     CompleteMultipartUpload_success,
-		"PutBucketAcl_non_existing_bucket":                    PutBucketAcl_non_existing_bucket,
-		"PutBucketAcl_invalid_acl_canned_and_acp":             PutBucketAcl_invalid_acl_canned_and_acp,
-		"PutBucketAcl_invalid_acl_canned_and_grants":          PutBucketAcl_invalid_acl_canned_and_grants,
-		"PutBucketAcl_invalid_acl_acp_and_grants":             PutBucketAcl_invalid_acl_acp_and_grants,
-		"PutBucketAcl_invalid_owner":                          PutBucketAcl_invalid_owner,
-		"PutBucketAcl_success_access_denied":                  PutBucketAcl_success_access_denied,
-		"PutBucketAcl_success_grants":                         PutBucketAcl_success_grants,
-		"PutBucketAcl_success_canned_acl":                     PutBucketAcl_success_canned_acl,
-		"PutBucketAcl_success_acp":                            PutBucketAcl_success_acp,
-		"GetBucketAcl_non_existing_bucket":                    GetBucketAcl_non_existing_bucket,
-		"GetBucketAcl_access_denied":                          GetBucketAcl_access_denied,
-		"GetBucketAcl_success":                                GetBucketAcl_success,
-		"PutObject_overwrite_dir_obj":                         PutObject_overwrite_dir_obj,
-		"PutObject_overwrite_file_obj":                        PutObject_overwrite_file_obj,
-		"PutObject_dir_obj_with_data":                         PutObject_dir_obj_with_data,
-		"CreateMultipartUpload_dir_obj":                       CreateMultipartUpload_dir_obj,
-	}
-}

runtests.sh

@@ -20,8 +20,21 @@ if ! kill -0 $GW_PID; then
 fi
 
 # run tests
+# full flow tests
 if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
-	echo "tests failed"
+	echo "full flow tests failed"
+	kill $GW_PID
+	exit 1
+fi
+# posix tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 posix; then
+	echo "posix tests failed"
+	kill $GW_PID
+	exit 1
+fi
+# iam tests
+if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
+	echo "iam tests failed"
 	kill $GW_PID
 	exit 1
 fi

s3api/controllers/admin.go

@@ -43,8 +43,8 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 		return fmt.Errorf("failed to parse request body: %w", err)
 	}
 
-	if usr.Role != "user" && usr.Role != "admin" {
-		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
+	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
+		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
 	}
 
 	err = c.iam.CreateAccount(usr)

s3api/controllers/backend_moq_test.go

@@ -44,6 +44,9 @@ var _ backend.Backend = &BackendMock{}
 //			DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
 //				panic("mock out the DeleteBucket method")
 //			},
+//			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
+//				panic("mock out the DeleteBucketTagging method")
+//			},
 //			DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
 //				panic("mock out the DeleteObject method")
 //			},
@@ -56,6 +59,15 @@ var _ backend.Backend = &BackendMock{}
 //			GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
 //				panic("mock out the GetBucketAcl method")
 //			},
+//			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+//				panic("mock out the GetBucketPolicy method")
+//			},
+//			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
+//				panic("mock out the GetBucketTagging method")
+//			},
+//			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+//				panic("mock out the GetBucketVersioning method")
+//			},
 //			GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
 //				panic("mock out the GetObject method")
 //			},
@@ -83,6 +95,9 @@ var _ backend.Backend = &BackendMock{}
 //			ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 //				panic("mock out the ListMultipartUploads method")
 //			},
+//			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+//				panic("mock out the ListObjectVersions method")
+//			},
 //			ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 //				panic("mock out the ListObjects method")
 //			},
@@ -95,6 +110,15 @@ var _ backend.Backend = &BackendMock{}
 //			PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
 //				panic("mock out the PutBucketAcl method")
 //			},
+//			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
+//				panic("mock out the PutBucketPolicy method")
+//			},
+//			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
+//				panic("mock out the PutBucketTagging method")
+//			},
+//			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+//				panic("mock out the PutBucketVersioning method")
+//			},
 //			PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
 //				panic("mock out the PutObject method")
 //			},
@@ -150,6 +174,9 @@ type BackendMock struct {
 	// DeleteBucketFunc mocks the DeleteBucket method.
 	DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
 
+	// DeleteBucketTaggingFunc mocks the DeleteBucketTagging method.
+	DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
+
 	// DeleteObjectFunc mocks the DeleteObject method.
 	DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error
 
@@ -157,11 +184,20 @@ type BackendMock struct {
 	DeleteObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) error
 
 	// DeleteObjectsFunc mocks the DeleteObjects method.
-	DeleteObjectsFunc func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
+	DeleteObjectsFunc func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
 
 	// GetBucketAclFunc mocks the GetBucketAcl method.
 	GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
 
+	// GetBucketPolicyFunc mocks the GetBucketPolicy method.
+	GetBucketPolicyFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
+
+	// GetBucketTaggingFunc mocks the GetBucketTagging method.
+	GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
+
+	// GetBucketVersioningFunc mocks the GetBucketVersioning method.
+	GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
+
 	// GetObjectFunc mocks the GetObject method.
 	GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
 
@@ -189,6 +225,9 @@ type BackendMock struct {
 	// ListMultipartUploadsFunc mocks the ListMultipartUploads method.
 	ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 
+	// ListObjectVersionsFunc mocks the ListObjectVersions method.
+	ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
+
 	// ListObjectsFunc mocks the ListObjects method.
 	ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
 
@@ -201,6 +240,15 @@ type BackendMock struct {
 	// PutBucketAclFunc mocks the PutBucketAcl method.
 	PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
 
+	// PutBucketPolicyFunc mocks the PutBucketPolicy method.
+	PutBucketPolicyFunc func(contextMoqParam context.Context, bucket string, policy []byte) error
+
+	// PutBucketTaggingFunc mocks the PutBucketTagging method.
+	PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error
+
+	// PutBucketVersioningFunc mocks the PutBucketVersioning method.
+	PutBucketVersioningFunc func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error
+
 	// PutObjectFunc mocks the PutObject method.
 	PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error)
 
@@ -283,6 +331,13 @@ type BackendMock struct {
 			// DeleteBucketInput is the deleteBucketInput argument value.
 			DeleteBucketInput *s3.DeleteBucketInput
 		}
+		// DeleteBucketTagging holds details about calls to the DeleteBucketTagging method.
+		DeleteBucketTagging []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// DeleteObject holds details about calls to the DeleteObject method.
 		DeleteObject []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -313,6 +368,27 @@ type BackendMock struct {
 			// GetBucketAclInput is the getBucketAclInput argument value.
 			GetBucketAclInput *s3.GetBucketAclInput
 		}
+		// GetBucketPolicy holds details about calls to the GetBucketPolicy method.
+		GetBucketPolicy []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
+		// GetBucketTagging holds details about calls to the GetBucketTagging method.
+		GetBucketTagging []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
+		// GetBucketVersioning holds details about calls to the GetBucketVersioning method.
+		GetBucketVersioning []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+		}
 		// GetObject holds details about calls to the GetObject method.
 		GetObject []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -380,6 +456,13 @@ type BackendMock struct {
 			// ListMultipartUploadsInput is the listMultipartUploadsInput argument value.
 			ListMultipartUploadsInput *s3.ListMultipartUploadsInput
 		}
+		// ListObjectVersions holds details about calls to the ListObjectVersions method.
+		ListObjectVersions []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// ListObjectVersionsInput is the listObjectVersionsInput argument value.
+			ListObjectVersionsInput *s3.ListObjectVersionsInput
+		}
 		// ListObjects holds details about calls to the ListObjects method.
 		ListObjects []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -410,6 +493,31 @@ type BackendMock struct {
 			// Data is the data argument value.
 			Data []byte
 		}
+		// PutBucketPolicy holds details about calls to the PutBucketPolicy method.
+		PutBucketPolicy []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Policy is the policy argument value.
+			Policy []byte
+		}
+		// PutBucketTagging holds details about calls to the PutBucketTagging method.
+		PutBucketTagging []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// Bucket is the bucket argument value.
+			Bucket string
+			// Tags is the tags argument value.
+			Tags map[string]string
+		}
+		// PutBucketVersioning holds details about calls to the PutBucketVersioning method.
+		PutBucketVersioning []struct {
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// PutBucketVersioningInput is the putBucketVersioningInput argument value.
+			PutBucketVersioningInput *s3.PutBucketVersioningInput
+		}
 		// PutObject holds details about calls to the PutObject method.
 		PutObject []struct {
 			// ContextMoqParam is the contextMoqParam argument value.
@@ -477,10 +585,14 @@ type BackendMock struct {
 	lockCreateBucket            sync.RWMutex
 	lockCreateMultipartUpload   sync.RWMutex
 	lockDeleteBucket            sync.RWMutex
+	lockDeleteBucketTagging     sync.RWMutex
 	lockDeleteObject            sync.RWMutex
 	lockDeleteObjectTagging     sync.RWMutex
 	lockDeleteObjects           sync.RWMutex
 	lockGetBucketAcl            sync.RWMutex
+	lockGetBucketPolicy         sync.RWMutex
+	lockGetBucketTagging        sync.RWMutex
+	lockGetBucketVersioning     sync.RWMutex
 	lockGetObject               sync.RWMutex
 	lockGetObjectAcl            sync.RWMutex
 	lockGetObjectAttributes     sync.RWMutex
@@ -490,10 +602,14 @@ type BackendMock struct {
 	lockListBuckets             sync.RWMutex
 	lockListBucketsAndOwners    sync.RWMutex
 	lockListMultipartUploads    sync.RWMutex
+	lockListObjectVersions      sync.RWMutex
 	lockListObjects             sync.RWMutex
 	lockListObjectsV2           sync.RWMutex
 	lockListParts               sync.RWMutex
 	lockPutBucketAcl            sync.RWMutex
+	lockPutBucketPolicy         sync.RWMutex
+	lockPutBucketTagging        sync.RWMutex
+	lockPutBucketVersioning     sync.RWMutex
 	lockPutObject               sync.RWMutex
 	lockPutObjectAcl            sync.RWMutex
 	lockPutObjectTagging        sync.RWMutex
@@ -765,6 +881,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
 	return calls
 }
 
+// DeleteBucketTagging calls DeleteBucketTaggingFunc.
+func (mock *BackendMock) DeleteBucketTagging(contextMoqParam context.Context, bucket string) error {
+	if mock.DeleteBucketTaggingFunc == nil {
+		panic("BackendMock.DeleteBucketTaggingFunc: method is nil but Backend.DeleteBucketTagging was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockDeleteBucketTagging.Lock()
+	mock.calls.DeleteBucketTagging = append(mock.calls.DeleteBucketTagging, callInfo)
+	mock.lockDeleteBucketTagging.Unlock()
+	return mock.DeleteBucketTaggingFunc(contextMoqParam, bucket)
+}
+
+// DeleteBucketTaggingCalls gets all the calls that were made to DeleteBucketTagging.
+// Check the length with:
+//
+//	len(mockedBackend.DeleteBucketTaggingCalls())
+func (mock *BackendMock) DeleteBucketTaggingCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockDeleteBucketTagging.RLock()
+	calls = mock.calls.DeleteBucketTagging
+	mock.lockDeleteBucketTagging.RUnlock()
+	return calls
+}
+
 // DeleteObject calls DeleteObjectFunc.
 func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
 	if mock.DeleteObjectFunc == nil {
@@ -842,7 +994,7 @@ func (mock *BackendMock) DeleteObjectTaggingCalls() []struct {
 }
 
 // DeleteObjects calls DeleteObjectsFunc.
-func (mock *BackendMock) DeleteObjects(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+func (mock *BackendMock) DeleteObjects(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
 	if mock.DeleteObjectsFunc == nil {
 		panic("BackendMock.DeleteObjectsFunc: method is nil but Backend.DeleteObjects was just called")
 	}
@@ -913,6 +1065,114 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
 	return calls
 }
 
+// GetBucketPolicy calls GetBucketPolicyFunc.
+func (mock *BackendMock) GetBucketPolicy(contextMoqParam context.Context, bucket string) ([]byte, error) {
+	if mock.GetBucketPolicyFunc == nil {
+		panic("BackendMock.GetBucketPolicyFunc: method is nil but Backend.GetBucketPolicy was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketPolicy.Lock()
+	mock.calls.GetBucketPolicy = append(mock.calls.GetBucketPolicy, callInfo)
+	mock.lockGetBucketPolicy.Unlock()
+	return mock.GetBucketPolicyFunc(contextMoqParam, bucket)
+}
+
+// GetBucketPolicyCalls gets all the calls that were made to GetBucketPolicy.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketPolicyCalls())
+func (mock *BackendMock) GetBucketPolicyCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketPolicy.RLock()
+	calls = mock.calls.GetBucketPolicy
+	mock.lockGetBucketPolicy.RUnlock()
+	return calls
+}
+
+// GetBucketTagging calls GetBucketTaggingFunc.
+func (mock *BackendMock) GetBucketTagging(contextMoqParam context.Context, bucket string) (map[string]string, error) {
+	if mock.GetBucketTaggingFunc == nil {
+		panic("BackendMock.GetBucketTaggingFunc: method is nil but Backend.GetBucketTagging was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketTagging.Lock()
+	mock.calls.GetBucketTagging = append(mock.calls.GetBucketTagging, callInfo)
+	mock.lockGetBucketTagging.Unlock()
+	return mock.GetBucketTaggingFunc(contextMoqParam, bucket)
+}
+
+// GetBucketTaggingCalls gets all the calls that were made to GetBucketTagging.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketTaggingCalls())
+func (mock *BackendMock) GetBucketTaggingCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketTagging.RLock()
+	calls = mock.calls.GetBucketTagging
+	mock.lockGetBucketTagging.RUnlock()
+	return calls
+}
+
+// GetBucketVersioning calls GetBucketVersioningFunc.
+func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+	if mock.GetBucketVersioningFunc == nil {
+		panic("BackendMock.GetBucketVersioningFunc: method is nil but Backend.GetBucketVersioning was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+	}
+	mock.lockGetBucketVersioning.Lock()
+	mock.calls.GetBucketVersioning = append(mock.calls.GetBucketVersioning, callInfo)
+	mock.lockGetBucketVersioning.Unlock()
+	return mock.GetBucketVersioningFunc(contextMoqParam, bucket)
+}
+
+// GetBucketVersioningCalls gets all the calls that were made to GetBucketVersioning.
+// Check the length with:
+//
+//	len(mockedBackend.GetBucketVersioningCalls())
+func (mock *BackendMock) GetBucketVersioningCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+	}
+	mock.lockGetBucketVersioning.RLock()
+	calls = mock.calls.GetBucketVersioning
+	mock.lockGetBucketVersioning.RUnlock()
+	return calls
+}
+
 // GetObject calls GetObjectFunc.
 func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
 	if mock.GetObjectFunc == nil {
@@ -1245,6 +1505,42 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct {
 	return calls
 }
 
+// ListObjectVersions calls ListObjectVersionsFunc.
+func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+	if mock.ListObjectVersionsFunc == nil {
+		panic("BackendMock.ListObjectVersionsFunc: method is nil but Backend.ListObjectVersions was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam         context.Context
+		ListObjectVersionsInput *s3.ListObjectVersionsInput
+	}{
+		ContextMoqParam:         contextMoqParam,
+		ListObjectVersionsInput: listObjectVersionsInput,
+	}
+	mock.lockListObjectVersions.Lock()
+	mock.calls.ListObjectVersions = append(mock.calls.ListObjectVersions, callInfo)
+	mock.lockListObjectVersions.Unlock()
+	return mock.ListObjectVersionsFunc(contextMoqParam, listObjectVersionsInput)
+}
+
+// ListObjectVersionsCalls gets all the calls that were made to ListObjectVersions.
+// Check the length with:
+//
+//	len(mockedBackend.ListObjectVersionsCalls())
+func (mock *BackendMock) ListObjectVersionsCalls() []struct {
+	ContextMoqParam         context.Context
+	ListObjectVersionsInput *s3.ListObjectVersionsInput
+} {
+	var calls []struct {
+		ContextMoqParam         context.Context
+		ListObjectVersionsInput *s3.ListObjectVersionsInput
+	}
+	mock.lockListObjectVersions.RLock()
+	calls = mock.calls.ListObjectVersions
+	mock.lockListObjectVersions.RUnlock()
+	return calls
+}
+
 // ListObjects calls ListObjectsFunc.
 func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 	if mock.ListObjectsFunc == nil {
@@ -1393,6 +1689,122 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
 	return calls
 }
 
+// PutBucketPolicy calls PutBucketPolicyFunc.
+func (mock *BackendMock) PutBucketPolicy(contextMoqParam context.Context, bucket string, policy []byte) error {
+	if mock.PutBucketPolicyFunc == nil {
+		panic("BackendMock.PutBucketPolicyFunc: method is nil but Backend.PutBucketPolicy was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Policy          []byte
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Policy:          policy,
+	}
+	mock.lockPutBucketPolicy.Lock()
+	mock.calls.PutBucketPolicy = append(mock.calls.PutBucketPolicy, callInfo)
+	mock.lockPutBucketPolicy.Unlock()
+	return mock.PutBucketPolicyFunc(contextMoqParam, bucket, policy)
+}
+
+// PutBucketPolicyCalls gets all the calls that were made to PutBucketPolicy.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketPolicyCalls())
+func (mock *BackendMock) PutBucketPolicyCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Policy          []byte
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Policy          []byte
+	}
+	mock.lockPutBucketPolicy.RLock()
+	calls = mock.calls.PutBucketPolicy
+	mock.lockPutBucketPolicy.RUnlock()
+	return calls
+}
+
+// PutBucketTagging calls PutBucketTaggingFunc.
+func (mock *BackendMock) PutBucketTagging(contextMoqParam context.Context, bucket string, tags map[string]string) error {
+	if mock.PutBucketTaggingFunc == nil {
+		panic("BackendMock.PutBucketTaggingFunc: method is nil but Backend.PutBucketTagging was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Tags            map[string]string
+	}{
+		ContextMoqParam: contextMoqParam,
+		Bucket:          bucket,
+		Tags:            tags,
+	}
+	mock.lockPutBucketTagging.Lock()
+	mock.calls.PutBucketTagging = append(mock.calls.PutBucketTagging, callInfo)
+	mock.lockPutBucketTagging.Unlock()
+	return mock.PutBucketTaggingFunc(contextMoqParam, bucket, tags)
+}
+
+// PutBucketTaggingCalls gets all the calls that were made to PutBucketTagging.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketTaggingCalls())
+func (mock *BackendMock) PutBucketTaggingCalls() []struct {
+	ContextMoqParam context.Context
+	Bucket          string
+	Tags            map[string]string
+} {
+	var calls []struct {
+		ContextMoqParam context.Context
+		Bucket          string
+		Tags            map[string]string
+	}
+	mock.lockPutBucketTagging.RLock()
+	calls = mock.calls.PutBucketTagging
+	mock.lockPutBucketTagging.RUnlock()
+	return calls
+}
+
+// PutBucketVersioning calls PutBucketVersioningFunc.
+func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+	if mock.PutBucketVersioningFunc == nil {
+		panic("BackendMock.PutBucketVersioningFunc: method is nil but Backend.PutBucketVersioning was just called")
+	}
+	callInfo := struct {
+		ContextMoqParam          context.Context
+		PutBucketVersioningInput *s3.PutBucketVersioningInput
+	}{
+		ContextMoqParam:          contextMoqParam,
+		PutBucketVersioningInput: putBucketVersioningInput,
+	}
+	mock.lockPutBucketVersioning.Lock()
+	mock.calls.PutBucketVersioning = append(mock.calls.PutBucketVersioning, callInfo)
+	mock.lockPutBucketVersioning.Unlock()
+	return mock.PutBucketVersioningFunc(contextMoqParam, putBucketVersioningInput)
+}
+
+// PutBucketVersioningCalls gets all the calls that were made to PutBucketVersioning.
+// Check the length with:
+//
+//	len(mockedBackend.PutBucketVersioningCalls())
+func (mock *BackendMock) PutBucketVersioningCalls() []struct {
+	ContextMoqParam          context.Context
+	PutBucketVersioningInput *s3.PutBucketVersioningInput
+} {
+	var calls []struct {
+		ContextMoqParam          context.Context
+		PutBucketVersioningInput *s3.PutBucketVersioningInput
+	}
+	mock.lockPutBucketVersioning.RLock()
+	calls = mock.calls.PutBucketVersioning
+	mock.lockPutBucketVersioning.RUnlock()
+	return calls
+}
+
 // PutObject calls PutObjectFunc.
 func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
 	if mock.PutObjectFunc == nil {

s3api/controllers/base_test.go

@@ -343,6 +343,18 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 				return &s3.ListObjectsOutput{}, nil
 			},
+			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
+				return map[string]string{}, nil
+			},
+			GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+				return &s3.GetBucketVersioningOutput{}, nil
+			},
+			ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+				return &s3.ListObjectVersionsOutput{}, nil
+			},
+			GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
+				return []byte{}, nil
+			},
 		},
 	}
 
@@ -365,6 +377,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 			},
+			GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
+				return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+			},
 		},
 	}
 	appError := fiber.New()
@@ -384,6 +399,24 @@ func TestS3ApiController_ListActions(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
+		{
+			name: "Get-bucket-tagging-non-existing-bucket",
+			app:  appError,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil),
+			},
+			wantErr:    false,
+			statusCode: 404,
+		},
+		{
+			name: "Get-bucket-tagging-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Get-bucket-acl-success",
 			app:  app,
@@ -429,6 +462,33 @@ func TestS3ApiController_ListActions(t *testing.T) {
 			wantErr:    false,
 			statusCode: 501,
 		},
+		{
+			name: "List-actions-get-bucket-versioning-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?versioning", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "List-actions-get-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "List-actions-list-object-versions-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodGet, "/my-bucket?versions", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -492,6 +552,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	</AccessControlPolicy>
 	`
 
+	tagBody := `
+	<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<TagSet>
+			<Tag>
+				<Key>organization</Key>
+				<Value>marketing</Value>
+			</Tag>
+		</TagSet>
+	</Tagging>
+	`
+
+	versioningBody := `
+	<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> 
+		<Status>Enabled</Status> 
+		<MfaDelete>Enabled</MfaDelete>
+	</VersioningConfiguration>
+	`
+
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -503,6 +581,15 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 			CreateBucketFunc: func(context.Context, *s3.CreateBucketInput, []byte) error {
 				return nil
 			},
+			PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
+				return nil
+			},
+			PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
+				return nil
+			},
+			PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
+				return nil
+			},
 		},
 	}
 	// Mock ctx.Locals
@@ -543,6 +630,51 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 		wantErr    bool
 		statusCode int
 	}{
+		{
+			name: "Put-bucket-tagging-invalid-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-tagging-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Put-bucket-versioning-invalid-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Put-bucket-versioning-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", strings.NewReader(versioningBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Put-bucket-policy-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", nil),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
 		{
 			name: "Put-bucket-acl-invalid-acl",
 			app:  app,
@@ -850,12 +982,12 @@ func TestS3ApiController_PutActions(t *testing.T) {
 		resp, err := tt.app.Test(tt.args.req)
 
 		if (err != nil) != tt.wantErr {
-			t.Errorf("S3ApiController.GetActions() %v error = %v, wantErr %v",
+			t.Errorf("S3ApiController.PutActions() %v error = %v, wantErr %v",
 				tt.name, err, tt.wantErr)
 		}
 
 		if resp.StatusCode != tt.statusCode {
-			t.Errorf("S3ApiController.GetActions() %v statusCode = %v, wantStatusCode = %v",
+			t.Errorf("S3ApiController.PutActions() %v statusCode = %v, wantStatusCode = %v",
 				tt.name, resp.StatusCode, tt.statusCode)
 		}
 	}
@@ -869,12 +1001,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app := fiber.New()
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
-			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
-				return acldata, nil
-			},
 			DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
 				return nil
 			},
+			DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
+				return nil
+			},
 		},
 	}
 
@@ -904,6 +1036,15 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 			wantErr:    false,
 			statusCode: 204,
 		},
+		{
+			name: "Delete-bucket-tagging-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodDelete, "/my-bucket?tagging", nil),
+			},
+			wantErr:    false,
+			statusCode: 204,
+		},
 	}
 	for _, tt := range tests {
 		resp, err := tt.app.Test(tt.args.req)
@@ -929,8 +1070,8 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 				return acldata, nil
 			},
-			DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
-				return s3response.DeleteObjectsResult{}, nil
+			DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
+				return s3response.DeleteResult{}, nil
 			},
 		},
 	}

s3api/middlewares/acl-parser.go

@@ -16,6 +16,7 @@ package middlewares
 
 import (
 	"net/http"
+	"regexp"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -26,6 +27,10 @@ import (
 	"github.com/versity/versitygw/s3log"
 )
 
+var (
+	singlePath = regexp.MustCompile(`^/[^/]+/?$`)
+)
+
 func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
@@ -38,8 +43,13 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
 		if ctx.Method() == http.MethodPatch {
 			return ctx.Next()
 		}
-		if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
-			if err := auth.IsAdmin(acct, isRoot); err != nil {
+		if singlePath.MatchString(path) &&
+			ctx.Method() == http.MethodPut &&
+			!ctx.Request().URI().QueryArgs().Has("acl") &&
+			!ctx.Request().URI().QueryArgs().Has("tagging") &&
+			!ctx.Request().URI().QueryArgs().Has("versioning") &&
+			!ctx.Request().URI().QueryArgs().Has("policy") {
+			if err := auth.MayCreateBucket(acct, isRoot); err != nil {
 				return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
 			}
 			return ctx.Next()

s3api/middlewares/authentication.go

@@ -44,6 +44,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 	acct := accounts{root: root, iam: iam}
 
 	return func(ctx *fiber.Ctx) error {
+		// If account is set in context locals, it means it was presigned url case
+		_, ok := ctx.Locals("account").(auth.Account)
+		if ok {
+			return ctx.Next()
+		}
+
 		ctx.Locals("region", region)
 		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
@@ -96,7 +102,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		}
 
 		// Validate the dates difference
-		err = validateDate(tdate)
+		err = utils.ValidateDate(tdate)
 		if err != nil {
 			return sendResponse(ctx, err, logger)
 		}
@@ -158,29 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
 	return a.iam.GetUserAccount(access)
 }
 
-func validateDate(date time.Time) error {
-	now := time.Now().UTC()
-	diff := date.Unix() - now.Unix()
-
-	// Checks the dates difference to be less than a minute
-	if diff > 60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
-	}
-	if diff < -60 {
-		return s3err.APIError{
-			Code:           "SignatureDoesNotMatch",
-			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-			HTTPStatusCode: http.StatusForbidden,
-		}
-	}
-
-	return nil
-}
-
 func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
 	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
 }

s3api/middlewares/presign-auth.go

@@ -0,0 +1,69 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3log"
+)
+
+func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
+	acct := accounts{root: root, iam: iam}
+
+	return func(ctx *fiber.Ctx) error {
+		if ctx.Query("X-Amz-Signature") == "" {
+			return ctx.Next()
+		}
+
+		ctx.Locals("region", region)
+		ctx.Locals("startTime", time.Now())
+
+		authData, err := utils.ParsePresignedURIParts(ctx)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		ctx.Locals("isRoot", authData.Access == root.Access)
+		account, err := acct.getAccount(authData.Access)
+		if err == auth.ErrNoSuchUser {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
+		}
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+		ctx.Locals("account", account)
+
+		if utils.IsBigDataAction(ctx) {
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
+			})
+
+			return ctx.Next()
+		}
+
+		err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
+		}
+
+		return ctx.Next()
+	}
+}

s3api/server.go

@@ -16,6 +16,7 @@ package s3api
 
 import (
 	"crypto/tls"
+	"net/http"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/logger"
@@ -34,6 +35,7 @@ type S3ApiServer struct {
 	cert    *tls.Certificate
 	quiet   bool
 	debug   bool
+	health  string
 }
 
 func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
@@ -52,10 +54,17 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 	if !server.quiet {
 		app.Use(logger.New())
 	}
+	// Set up health endpoint if specified
+	if server.health != "" {
+		app.Get(server.health, func(ctx *fiber.Ctx) error {
+			return ctx.SendStatus(http.StatusOK)
+		})
+	}
 	app.Use(middlewares.DecodeURL(l))
 	app.Use(middlewares.RequestLogger(server.debug))
 
 	// Authentication middlewares
+	app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
 	app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
 	app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
 	app.Use(middlewares.VerifyMD5Body(l))
@@ -89,6 +98,11 @@ func WithQuiet() Option {
 	return func(s *S3ApiServer) { s.quiet = true }
 }
 
+// WithHealth sets up a GET health endpoint
+func WithHealth(health string) Option {
+	return func(s *S3ApiServer) { s.health = health }
+}
+
 func (sa *S3ApiServer) Serve() (err error) {
 	if sa.cert != nil {
 		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

s3api/utils/auth-reader.go

@@ -24,9 +24,9 @@ import (
 	"unicode"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/smithy-go/logging"
 	"github.com/gofiber/fiber/v2"
+	v4 "github.com/versity/versitygw/aws/signer/v4"
 	"github.com/versity/versitygw/s3err"
 )
 
@@ -126,16 +126,19 @@ func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string,
 
 	signer := v4.NewSigner()
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{
-		AccessKeyID:     auth.Access,
-		SecretAccessKey: secret,
-	}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
-		options.DisableURIPathEscaping = true
-		if debug {
-			options.LogSigning = true
-			options.Logger = logging.NewStandardLogger(os.Stderr)
-		}
-	})
+	signErr := signer.SignHTTP(req.Context(),
+		aws.Credentials{
+			AccessKeyID:     auth.Access,
+			SecretAccessKey: secret,
+		},
+		req, checksum, service, auth.Region, tdate, signedHdrs,
+		func(options *v4.SignerOptions) {
+			options.DisableURIPathEscaping = true
+			if debug {
+				options.LogSigning = true
+				options.Logger = logging.NewStandardLogger(os.Stderr)
+			}
+		})
 	if signErr != nil {
 		return fmt.Errorf("sign generated http request: %w", err)
 	}

s3api/utils/auth_test.go

@@ -1,7 +1,14 @@
 package utils
 
 import (
+	"net"
 	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp/fasthttputil"
+	v4 "github.com/versity/versitygw/aws/signer/v4"
 )
 
 func TestAuthParse(t *testing.T) {
@@ -10,18 +17,26 @@ func TestAuthParse(t *testing.T) {
 		authstr string // Authorization string
 		algo    string
 		sig     string
-	}{{
-		name:    "restic",
-		authstr: "AWS4-HMAC-SHA256 Credential=user/20240116/us-east-1/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
-		algo:    "AWS4-HMAC-SHA256",
-		sig:     "d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
-	},
+	}{
+		{
+			name:    "restic",
+			authstr: "AWS4-HMAC-SHA256 Credential=user/20240116/us-east-1/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
+			algo:    "AWS4-HMAC-SHA256",
+			sig:     "d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
+		},
 		{
 			name:    "aws eaxample",
 			authstr: "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-date, Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
 			algo:    "AWS4-HMAC-SHA256",
 			sig:     "fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
-		}}
+		},
+		{
+			name:    "s3browser",
+			authstr: "AWS4-HMAC-SHA256 Credential=access_key/20240206/us-east-1/s3/aws4_request,SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, Signature=37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6",
+			algo:    "AWS4-HMAC-SHA256",
+			sig:     "37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6",
+		},
+	}
 
 	for _, v := range vectors {
 		t.Run(v.name, func(t *testing.T) {
@@ -38,3 +53,78 @@ func TestAuthParse(t *testing.T) {
 		})
 	}
 }
+
+// 2024/02/06 21:03:28 Request headers:
+// 2024/02/06 21:03:28 Host: 172.21.0.160:11000
+// 2024/02/06 21:03:28 User-Agent: S3 Browser/11.5.7 (https://s3browser.com)
+// 2024/02/06 21:03:28 Authorization: AWS4-HMAC-SHA256 Credential=access_key/20240206/us-east-1/s3/aws4_request,SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, Signature=37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6
+// 2024/02/06 21:03:28 X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+// 2024/02/06 21:03:28 X-Amz-Date: 20240206T210328Z
+func Test_Client_UserAgent(t *testing.T) {
+	signedHdrs := []string{"host", "user-agent", "x-amz-content-sha256", "x-amz-date"}
+	access := "access_key"
+	secret := "secret_key"
+	region := "us-east-1"
+	host := "172.21.0.160:11000"
+	agent := "S3 Browser/11.5.7 (https://s3browser.com)"
+	expectedSig := "37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6"
+	dateStr := "20240206T210328Z"
+
+	app := fiber.New(fiber.Config{DisableStartupMessage: true})
+
+	tdate, err := time.Parse(iso8601Format, dateStr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	app.Get("/", func(c *fiber.Ctx) error {
+		req, err := createHttpRequestFromCtx(c, signedHdrs, int64(c.Request().Header.ContentLength()))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		req.Host = host
+		req.Header.Set("X-Amz-Content-Sha256", zeroLenSig)
+
+		signer := v4.NewSigner()
+
+		signErr := signer.SignHTTP(req.Context(),
+			aws.Credentials{
+				AccessKeyID:     access,
+				SecretAccessKey: secret,
+			},
+			req, zeroLenSig, service, region, tdate, signedHdrs,
+			func(options *v4.SignerOptions) {
+				options.DisableURIPathEscaping = true
+			})
+		if signErr != nil {
+			t.Fatalf("sign generated http request: %v", err)
+		}
+
+		genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
+		if err != nil {
+			return err
+		}
+
+		if genAuth.Signature != expectedSig {
+			t.Errorf("SIG: %v\nexpected: %v\n", genAuth.Signature, expectedSig)
+		}
+
+		return c.Send(c.Request().Header.UserAgent())
+	})
+
+	ln := fasthttputil.NewInmemoryListener()
+	go func() {
+		err := app.Listener(ln)
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	c := fiber.AcquireClient()
+	c.UserAgent = agent
+	a := c.Get("http://example.com")
+	a.HostClient.Dial = func(_ string) (net.Conn, error) { return ln.Dial() }
+	a.String()
+	fiber.ReleaseClient(c)
+}

s3api/utils/presign-auth-reader.go

@@ -0,0 +1,243 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go/logging"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	unsignedPayload string = "UNSIGNED-PAYLOAD"
+)
+
+// PresignedAuthReader is an io.Reader that validates presigned request authorization
+// once the underlying reader returns io.EOF.  This is needed for streaming
+// data requests where the data size is not known until
+// the data is completely read.
+type PresignedAuthReader struct {
+	ctx    *fiber.Ctx
+	auth   AuthData
+	secret string
+	r      io.Reader
+	debug  bool
+}
+
+func NewPresignedAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *PresignedAuthReader {
+	return &PresignedAuthReader{
+		ctx:    ctx,
+		r:      r,
+		auth:   auth,
+		secret: secret,
+		debug:  debug,
+	}
+}
+
+// Read allows *PresignedAuthReader to be used as an io.Reader
+func (pr *PresignedAuthReader) Read(p []byte) (int, error) {
+	n, err := pr.r.Read(p)
+
+	if errors.Is(err, io.EOF) {
+		cerr := CheckPresignedSignature(pr.ctx, pr.auth, pr.secret, pr.debug)
+		if cerr != nil {
+			return n, cerr
+		}
+	}
+
+	return n, err
+}
+
+// CheckPresignedSignature validates presigned request signature
+func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, debug bool) error {
+	signedHdrs := strings.Split(auth.SignedHeaders, ";")
+
+	var contentLength int64
+	var err error
+	contentLengthStr := ctx.Get("Content-Length")
+	if contentLengthStr != "" {
+		contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+		if err != nil {
+			return s3err.GetAPIError(s3err.ErrInvalidRequest)
+		}
+	}
+
+	// Create a new http request instance from fasthttp request
+	req, err := createPresignedHttpRequestFromCtx(ctx, signedHdrs, contentLength)
+	if err != nil {
+		return fmt.Errorf("create http request from context: %w", err)
+	}
+
+	date, _ := time.Parse(iso8601Format, auth.Date)
+
+	signer := v4.NewSigner()
+	uri, _, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
+		AccessKeyID:     auth.Access,
+		SecretAccessKey: secret,
+	}, req, unsignedPayload, service, auth.Region, date, func(options *v4.SignerOptions) {
+		options.DisableURIPathEscaping = true
+		if debug {
+			options.LogSigning = true
+			options.Logger = logging.NewStandardLogger(os.Stderr)
+		}
+	})
+	if signErr != nil {
+		return fmt.Errorf("presign generated http request: %w", err)
+	}
+
+	urlParts, err := url.Parse(uri)
+	if err != nil {
+		return fmt.Errorf("parse presigned url: %w", err)
+	}
+
+	signature := urlParts.Query().Get("X-Amz-Signature")
+	if signature != auth.Signature {
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+
+	return nil
+}
+
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+//
+// # ParsePresignedURIParts parses and validates request URL query parameters
+//
+// ?X-Amz-Algorithm=AWS4-HMAC-SHA256
+// &X-Amz-Credential=access-key-id/20130721/us-east-1/s3/aws4_request
+// &X-Amz-Date=20130721T201207Z
+// &X-Amz-Expires=86400
+// &X-Amz-SignedHeaders=host
+// &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8
+func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
+	a := AuthData{}
+
+	// Get and verify algorithm query parameter
+	algo := ctx.Query("X-Amz-Algorithm")
+	if algo == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+	if algo != "AWS4-HMAC-SHA256" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)
+	}
+
+	// Parse and validate credentials query parameter
+	credsQuery := ctx.Query("X-Amz-Credential")
+	if credsQuery == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	creds := strings.Split(credsQuery, "/")
+	if len(creds) != 5 {
+		return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+	}
+	if creds[3] != "s3" {
+		return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
+	}
+	if creds[4] != "aws4_request" {
+		return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
+	}
+	_, err := time.Parse(yyyymmdd, creds[1])
+	if err != nil {
+		return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
+	}
+
+	// Parse and validate Date query param
+	date := ctx.Query("X-Amz-Date")
+	if date == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	tdate, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return a, s3err.GetAPIError(s3err.ErrMalformedDate)
+	}
+
+	if date[:8] != creds[1] {
+		return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
+	}
+
+	if ctx.Locals("region") != creds[2] {
+		return a, s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+
+	signature := ctx.Query("X-Amz-Signature")
+	if signature == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	signedHdrs := ctx.Query("X-Amz-SignedHeaders")
+	if signedHdrs == "" {
+		return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	// Validate X-Amz-Expires query param and check if request is expired
+	err = validateExpiration(ctx.Query("X-Amz-Expires"), tdate)
+	if err != nil {
+		return a, err
+	}
+
+	a.Signature = signature
+	a.Access = creds[0]
+	a.Algorithm = algo
+	a.Region = creds[2]
+	a.SignedHeaders = signedHdrs
+	a.Date = date
+
+	return a, nil
+}
+
+func validateExpiration(str string, date time.Time) error {
+	if str == "" {
+		return s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+	}
+
+	exp, err := strconv.Atoi(str)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrMalformedExpires)
+	}
+
+	if exp < 0 {
+		return s3err.GetAPIError(s3err.ErrNegativeExpires)
+	}
+
+	if exp > 604800 {
+		return s3err.GetAPIError(s3err.ErrMaximumExpires)
+	}
+
+	now := time.Now()
+	passed := int(now.Sub(date).Seconds())
+
+	if passed > exp {
+		return s3err.GetAPIError(s3err.ErrExpiredPresignRequest)
+	}
+
+	return nil
+}

s3api/utils/presign-auth-reader_test.go

@@ -0,0 +1,100 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"testing"
+	"time"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+func Test_validateExpiration(t *testing.T) {
+	type args struct {
+		str  string
+		date time.Time
+	}
+	tests := []struct {
+		name string
+		args args
+		err  error
+	}{
+		{
+			name: "empty-expiration",
+			args: args{
+				str:  "",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrInvalidQueryParams),
+		},
+		{
+			name: "invalid-expiration",
+			args: args{
+				str:  "invalid_expiration",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrMalformedExpires),
+		},
+		{
+			name: "negative-expiration",
+			args: args{
+				str:  "-320",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrNegativeExpires),
+		},
+		{
+			name: "exceeding-expiration",
+			args: args{
+				str:  "6048000",
+				date: time.Now(),
+			},
+			err: s3err.GetAPIError(s3err.ErrMaximumExpires),
+		},
+		{
+			name: "expired value",
+			args: args{
+				str:  "200",
+				date: time.Now().AddDate(0, 0, -1),
+			},
+			err: s3err.GetAPIError(s3err.ErrExpiredPresignRequest),
+		},
+		{
+			name: "valid expiration",
+			args: args{
+				str:  "300",
+				date: time.Now(),
+			},
+			err: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateExpiration(tt.args.str, tt.args.date)
+			// Check for nil case
+			if tt.err == nil && err != nil {
+				t.Errorf("Expected nil error, got: %v", err)
+				return
+			} else if tt.err == nil && err == nil {
+				// Both are nil, no need for further comparison
+				return
+			}
+
+			if err.Error() != tt.err.Error() {
+				t.Errorf("Expected error: %v, got: %v", tt.err, err)
+			}
+		})
+	}
+}

s3api/utils/utils.go

@@ -20,10 +20,13 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/aws/smithy-go/encoding/httpbinding"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/s3err"
@@ -72,6 +75,75 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		}
 	})
 
+	// make sure all headers in the signed headers are present
+	for _, header := range signedHdrs {
+		if httpReq.Header.Get(header) == "" {
+			httpReq.Header.Set(header, "")
+		}
+	}
+
+	// Check if Content-Length in signed headers
+	// If content length is non 0, then the header will be included
+	if !includeHeader("Content-Length", signedHdrs) {
+		httpReq.ContentLength = 0
+	} else {
+		httpReq.ContentLength = contentLength
+	}
+
+	// Set the Host header
+	httpReq.Host = string(req.Header.Host())
+
+	return httpReq, nil
+}
+
+var (
+	signedQueryArgs = map[string]bool{
+		"X-Amz-Algorithm":     true,
+		"X-Amz-Credential":    true,
+		"X-Amz-Date":          true,
+		"X-Amz-SignedHeaders": true,
+		"X-Amz-Signature":     true,
+	}
+)
+
+func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
+	req := ctx.Request()
+	var body io.Reader
+	if IsBigDataAction(ctx) {
+		body = req.BodyStream()
+	} else {
+		body = bytes.NewReader(req.Body())
+	}
+
+	uri := string(ctx.Request().URI().Path())
+	uri = httpbinding.EscapePath(uri, false)
+	isFirst := true
+
+	ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
+		_, ok := signedQueryArgs[string(key)]
+		if !ok {
+			escapeValue := url.QueryEscape(string(value))
+			if isFirst {
+				uri += fmt.Sprintf("?%s=%s", key, escapeValue)
+				isFirst = false
+			} else {
+				uri += fmt.Sprintf("&%s=%s", key, escapeValue)
+			}
+		}
+	})
+
+	httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
+	if err != nil {
+		return nil, errors.New("error in creating an http request")
+	}
+	// Set the request headers
+	req.Header.VisitAll(func(key, value []byte) {
+		keyStr := string(key)
+		if includeHeader(keyStr, signedHdrs) {
+			httpReq.Header.Add(keyStr, string(value))
+		}
+	})
+
 	// Check if Content-Length in signed headers
 	// If content length is non 0, then the header will be included
 	if !includeHeader("Content-Length", signedHdrs) {
@@ -149,3 +221,26 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
 	}
 	return false
 }
+
+func ValidateDate(date time.Time) error {
+	now := time.Now().UTC()
+	diff := date.Unix() - now.Unix()
+
+	// Checks the dates difference to be less than a minute
+	if diff > 60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+	if diff < -60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+
+	return nil
+}

s3api/utils/utils_test.go

@@ -35,6 +35,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 		args    args
 		want    *http.Request
 		wantErr bool
+		hdrs    []string
 	}{
 		{
 			name: "Success-response",
@@ -43,6 +44,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request,
 			wantErr: false,
+			hdrs:    []string{},
 		},
 		{
 			name: "Success-response-With-Headers",
@@ -51,11 +53,12 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request2,
 			wantErr: false,
+			hdrs:    []string{"X-Amz-Mfa"},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
+			got, err := createHttpRequestFromCtx(tt.args.ctx, tt.hdrs, 0)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
 				return

s3response/README.txt

@@ -1,6 +0,0 @@
-https://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd
-
-see https://blog.aqwari.net/xml-schema-go/
-
-go install aqwari.net/xml/cmd/xsdgen@latest
-xsdgen -o s3api_xsd_generated.go -pkg s3response AmazonS3.xsd

s3response/s3response.go

@@ -16,6 +16,7 @@ package s3response
 
 import (
 	"encoding/xml"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
@@ -107,6 +108,11 @@ type TagSet struct {
 }
 
 type Tagging struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging" json:"-"`
+	TagSet  TagSet   `xml:"TagSet"`
+}
+
+type TaggingInput struct {
 	TagSet TagSet `xml:"TagSet"`
 }
 
@@ -114,7 +120,7 @@ type DeleteObjects struct {
 	Objects []types.ObjectIdentifier `xml:"Object"`
 }
 
-type DeleteObjectsResult struct {
+type DeleteResult struct {
 	Deleted []types.DeletedObject
 	Error   []types.Error
 }
@@ -139,3 +145,58 @@ type Bucket struct {
 	Name  string `json:"name"`
 	Owner string `json:"owner"`
 }
+
+type ListAllMyBucketsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	Owner   CanonicalUser
+	Buckets ListAllMyBucketsList
+}
+
+type ListAllMyBucketsEntry struct {
+	Name         string
+	CreationDate time.Time
+}
+
+type ListAllMyBucketsList struct {
+	Bucket []ListAllMyBucketsEntry
+}
+
+type CanonicalUser struct {
+	ID          string
+	DisplayName string
+}
+
+type CopyObjectResult struct {
+	XMLName      xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	LastModified time.Time
+	ETag         string
+}
+
+type AccessControlPolicy struct {
+	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
+	Owner             CanonicalUser
+	AccessControlList AccessControlList
+}
+
+type AccessControlList struct {
+	Grant []Grant
+}
+
+type Grant struct {
+	Grantee    Grantee
+	Permission string
+}
+
+// Set the following to encode correctly:
+//
+//	Grantee: s3response.Grantee{
+//		Xsi:         "http://www.w3.org/2001/XMLSchema-instance",
+//		Type:        "CanonicalUser",
+//	},
+type Grantee struct {
+	XMLName     xml.Name `xml:"Grantee"`
+	Xsi         string   `xml:"xmlns:xsi,attr,omitempty"`
+	Type        string   `xml:"xsi:type,attr,omitempty"`
+	ID          string
+	DisplayName string
+}

s3select/arch32.go

@@ -0,0 +1,24 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !amd64 && !arm64 && !ppc64le && !riscv64
+// +build !amd64,!arm64,!ppc64le,!riscv64
+
+package s3select
+
+import "math"
+
+const (
+	maxMessageSize = math.MaxInt
+)

s3select/arch64.go

@@ -0,0 +1,22 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build amd64 || arm64 || ppc64le || riscv64
+// +build amd64 arm64 ppc64le riscv64
+
+package s3select
+
+const (
+	maxMessageSize = 5 * 1024 * 1024 * 1024
+)

s3select/message-handler.go

@@ -123,8 +123,7 @@ func generatePrelude(msgLen int, headerLen int) []byte {
 }
 
 const (
-	maxHeaderSize  = 1024 * 1024
-	maxMessageSize = 5 * 1024 * 1024 * 1024
+	maxHeaderSize = 1024 * 1024
 )
 
 func genMessage(header, payload []byte) []byte {

tests/.env.default

@@ -1,6 +1,14 @@
-AWS_REGION=us-west-2
 AWS_PROFILE=versity
+AWS_ENDPOINT_URL=https://127.0.0.1:7070
 VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
 BACKEND=posix
 LOCAL_FOLDER=/tmp/gw
-AWS_ENDPOINT_URL=http://127.0.0.1:7070
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+#RECREATE_BUCKETS=true
+CERT=$PWD/cert.pem
+KEY=$PWD/versitygw.pem
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets
+MC_ALIAS=versity

tests/.env.versitygw

@@ -1,6 +1,8 @@
-AWS_REGION=us-east-1
 AWS_PROFILE=versity
+AWS_ENDPOINT_URL=http://127.0.0.1:7070
 VERSITY_EXE=./versitygw
 BACKEND=posix
 LOCAL_FOLDER=/tmp/gw
-AWS_ENDPOINT_URL=http://127.0.0.1:7070
+BUCKET_ONE_NAME=versity-gwtest-bucket-one
+BUCKET_TWO_NAME=versity-gwtest-bucket-two
+RECREATE_BUCKETS=true

tests/README.md

@@ -1,13 +1,34 @@
 # Command-Line Tests
 
-Instructions:
+## Instructions - Running Locally
+
 1. Build the `versitygw` binary.
-2. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values above to the profile.
-3. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
-4. In the root repo folder, run with `VERSITYGW_TEST_ENV=<env file> tests/s3_bucket_tests.sh`.
-5. If running/testing the GitHub workflow locally, create a `.secrets` file, and set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` parameters here to the values of your AWS S3 IAM account.
+2. Install the command-line interface(s) you want to test if unavailable on your machine.  
+   * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
+   * **s3cmd**:  Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
+   * **mc**:  Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
+3. Install BATS.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
+4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
+5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
 ```
-AWS_ACCESS_KEY_ID=<key_id>
-AWS_SECRET_ACCESS_KEY=<secret_key>
+    export AWS_PROFILE=versity-test
+    export AWS_ACCESS_KEY_ID=<your account ID>
+    export AWS_SECRET_ACCESS_KEY=<your account key>
+    export AWS_REGION=<your account region>
+    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE
+    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE
+    aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
 ```
-6. To run the workflow locally, install **act** and run with `act -W .github/workflows/system.yml`.
+6. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
+7. If using SSL, create a local private key and certificate, such as with the commands below.  Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively.
+```
+    openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
+    openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
+```
+8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
+9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+
+## Instructions - Running With Docker
+
+1.  Create a `.secrets` file in the `tests` folder, and add the `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and the `AWS_PROFILE` fields.
+2.  Build and run the `Dockerfile_test_bats` file.

tests/certs/azurite-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDUaEoThzf6bWNU
+R7+V0sYGND0iArE8dx33jCqWBzUCeeAnf8VjsKTVKWK8Cernjot7jP/GlKHCXKVC
+Dpaw2SuekVn84gXr0UDD0clrPYYcytm8TWrfNmcPnmLARH6bes+5Glt21gug5fc7
+gvd/EFI/eNTAmU6ZPzVjKVP5plvkSZYXePgmgKzY8hs8j9IjjOZB+TzalRlQoMa4
+hPYTzNpvdS3aQCM/8PTefC8sxI0H9inx1utax22MPNCaGl+BvD9yqvFwQl7n0d9u
+3ZLJ8Br9UKRPX47NDerpsYsmJAAtjevei84v4liCKtxIzC6b/KuKpgEDhiza+TL/
+VoJ34xK9AgMBAAECggEBAISIPhBJQshjEKM50XTuetjMJ4jdHTGZMX2QW9IY1R6i
+ZRbARq2ZPoAyvoSNu6CX9Lg2ljGV9AiOPh8lcykNmIXsM7XyYbdubXbBo2fij5fP
+imRP+eskytGYBg3prwXcb1gT9hYEIGVYmBbt9Pe3e1pXToiOH9jG88zXsKoI/zVF
+ASfE73L9xlO9roaFWe+sQlNsHJ5SVhKVNq4FcjvIbZTi3JqMJvuGqgIjyOeKlRdH
+FszPjmn8tmlbT7BQIfeo4uIJwDrUxdp4Do9txScSOhV4xcdVjpOrbD+HaTIl18u8
+J32pf0n8E90ntKJlPyfSt68VIkIT1VtE0SH42UgxcAECgYEA2Ctw58hSKFSQtSDk
+HAOuxh9gSM4eIvWG8Ni4L+S6n66hCPYYj+1ZGBM1xIKqfug25STP2qV54zB34CMM
+RfPEJhKglgzN1VdTvTkdLfqSnrEDyF+67/X0IhomV54CLN52i7S6ra39OVClirQc
+5FbjX7Dwy57vanNW7E7ZKHyxRwECgYEA+4tiCw8+NeCIjLiOldB/R9P6CguFCIFY
+B+MUXc2K690ysRcVnHCVLGVrdYlmlAjiEEOo6e8nhl0pLJecOLUkLnEdZkC9x5je
+oy1itan7Y/WjdSPIpLE5sBnrWzzpy5YVoKi4rjIkPXYDHcgbvEcKUxN/qUpOpKk6
+rTeaAPOwp70CgYEAwPIFVNz4eAcDIqi48khXN3/J8TIItCtyxoap4BXIfb7g/Z6r
+TcwMOfDrjPsUMzIRzXWOERqiMKaSWPzvd4CdE16M92F2V3YayEqyQNfnBr35ImBP
++t8NiWLN1mayiloGdaxa86rY2s+g8qzRHP5w9Hh6dUTnbZyFeWbnbbvegAECgYAG
+T1zKQjuhRlymiwqon25R8vNWxSs1J2l56SxdngZaHFZlMtsL7ZcQYgrsC+JS3FYv
+akMWezVWnYem4ra8hW6+6399TSp1k1Qia8UKIZV40HSlP5yM5RU5Ya0RwNlsjftE
+6HaZiBB4qjkxyg9IDdAofVi6em62mrgqGDb4xyQrUQKBgHfzBE/pxKTZVCyjJgq+
+XKtvm7awq6tVByZzKVbFb+kqle7n0Kb+MjDSiH0gXqrntkhquEEpo8ViXenHeDq1
+I2Fgp6Tw8pKW7RANcWaxd2KLYDxUkqwNWCMYGQZoW9lhTy1/e5stXSTdqd+3CJkp
+qBbZOasECIEGL8XUpFEBmrTA
+-----END PRIVATE KEY-----

tests/certs/azurite.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEMzCCApugAwIBAgIQbYDa23tko31nfiGrCJ4XfDANBgkqhkiG9w0BAQsFADBj
+MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExHDAaBgNVBAsME2JlbkBC
+eXRlTmV4dXMubG9jYWwxIzAhBgNVBAMMGm1rY2VydCBiZW5AQnl0ZU5leHVzLmxv
+Y2FsMB4XDTI0MDIyMTA2MDY1OFoXDTI2MDUyMTA1MDY1OFowRzEnMCUGA1UEChMe
+bWtjZXJ0IGRldmVsb3BtZW50IGNlcnRpZmljYXRlMRwwGgYDVQQLDBNiZW5AQnl0
+ZU5leHVzLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1GhK
+E4c3+m1jVEe/ldLGBjQ9IgKxPHcd94wqlgc1AnngJ3/FY7Ck1SlivAnq546Le4z/
+xpShwlylQg6WsNkrnpFZ/OIF69FAw9HJaz2GHMrZvE1q3zZnD55iwER+m3rPuRpb
+dtYLoOX3O4L3fxBSP3jUwJlOmT81YylT+aZb5EmWF3j4JoCs2PIbPI/SI4zmQfk8
+2pUZUKDGuIT2E8zab3Ut2kAjP/D03nwvLMSNB/Yp8dbrWsdtjDzQmhpfgbw/cqrx
+cEJe59Hfbt2SyfAa/VCkT1+OzQ3q6bGLJiQALY3r3ovOL+JYgircSMwum/yriqYB
+A4Ys2vky/1aCd+MSvQIDAQABo38wfTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAww
+CgYIKwYBBQUHAwEwHwYDVR0jBBgwFoAUHWzgJTYXxrKTuaJyaD2ddH4g43QwNQYD
+VR0RBC4wLIIJbG9jYWxob3N0ggdhenVyaXRlhwR/AAABhxAAAAAAAAAAAAAAAAAA
+AAABMA0GCSqGSIb3DQEBCwUAA4IBgQBANjSWFEE2XrQfCtqKjHRtYn+WnjfeUmJW
+4qfNl971spR+ui+WMNob53EKGV9O55Ai8zWUKhL14GSDo4bqygcqCSP7esUaqW5x
+LgqGkIDPiPN18OfWxZonlvcSJYaWJ9dL4Xix7OmPXQV88oNUYh5zASJRZ8mN3vZK
+v0BElS5w7X4pM8jWkIEVUjmgSQJAxapDnbDPCIntH13rjYPHZW3zF1wjWW/mfW59
++thLVpG0U9w8knDU9TJQWPX9rQB8lLBHYfBy14ebuMWmDl25Rr+wcUjFp76HP8f9
+D8wX7ZTJlMlaH60vbTi8ylsKMlu6eHxJ8TBNdRI5+Xa94Pi6A81a9EiagpU2uZs1
+P/v/3ug9AtCFoEBB2groj3waC6QTOrJArHhvioLKBz7yMsfGRQCSA+VK+KGuqE6f
+E/Knv1Phf8HIe6OpDNXqlJo9+5m0sz/y6F2OaeNqM643Y78OQRFijtDKu+TmB+t0
+4lo/rMVz6A0vIOBBQkiJz82xezvCiHQ=
+-----END CERTIFICATE-----

tests/integration/group-tests.go

@@ -0,0 +1,457 @@
+package integration
+
+func TestAuthentication(s *S3Conf) {
+	Authentication_empty_auth_header(s)
+	Authentication_invalid_auth_header(s)
+	Authentication_unsupported_signature_version(s)
+	Authentication_malformed_credentials(s)
+	Authentication_malformed_credentials_invalid_parts(s)
+	Authentication_credentials_terminated_string(s)
+	Authentication_credentials_incorrect_service(s)
+	Authentication_credentials_incorrect_region(s)
+	Authentication_credentials_invalid_date(s)
+	Authentication_credentials_future_date(s)
+	Authentication_credentials_past_date(s)
+	Authentication_credentials_non_existing_access_key(s)
+	Authentication_invalid_signed_headers(s)
+	Authentication_missing_date_header(s)
+	Authentication_invalid_date_header(s)
+	Authentication_date_mismatch(s)
+	Authentication_incorrect_payload_hash(s)
+	Authentication_incorrect_md5(s)
+	Authentication_signature_error_incorrect_secret_key(s)
+}
+
+func TestPresignedAuthentication(s *S3Conf) {
+	PresignedAuth_missing_algo_query_param(s)
+	PresignedAuth_unsupported_algorithm(s)
+	PresignedAuth_missing_credentials_query_param(s)
+	PresignedAuth_malformed_creds_invalid_parts(s)
+	PresignedAuth_malformed_creds_invalid_parts(s)
+	PresignedAuth_creds_incorrect_service(s)
+	PresignedAuth_creds_incorrect_region(s)
+	PresignedAuth_creds_invalid_date(s)
+	PresignedAuth_missing_date_query(s)
+	PresignedAuth_dates_mismatch(s)
+	PresignedAuth_non_existing_access_key_id(s)
+	PresignedAuth_missing_signed_headers_query_param(s)
+	PresignedAuth_missing_expiration_query_param(s)
+	PresignedAuth_invalid_expiration_query_param(s)
+	PresignedAuth_negative_expiration_query_param(s)
+	PresignedAuth_exceeding_expiration_query_param(s)
+	PresignedAuth_expired_request(s)
+	PresignedAuth_incorrect_secret_key(s)
+	PresignedAuth_PutObject_success(s)
+	PresignedAuth_Put_GetObject_with_data(s)
+	PresignedAuth_Put_GetObject_with_UTF8_chars(s)
+	PresignedAuth_UploadPart(s)
+}
+
+func TestCreateBucket(s *S3Conf) {
+	CreateBucket_invalid_bucket_name(s)
+	CreateBucket_existing_bucket(s)
+	CreateBucket_as_user(s)
+	CreateBucket_default_acl(s)
+	CreateBucket_non_default_acl(s)
+	CreateDeleteBucket_success(s)
+}
+
+func TestHeadBucket(s *S3Conf) {
+	HeadBucket_non_existing_bucket(s)
+	HeadBucket_success(s)
+}
+
+func TestListBuckets(s *S3Conf) {
+	ListBuckets_as_user(s)
+	ListBuckets_as_admin(s)
+	ListBuckets_success(s)
+}
+
+func TestDeleteBucket(s *S3Conf) {
+	DeleteBucket_non_existing_bucket(s)
+	DeleteBucket_non_empty_bucket(s)
+	DeleteBucket_success_status_code(s)
+}
+
+func TestPutBucketTagging(s *S3Conf) {
+	PutBucketTagging_non_existing_bucket(s)
+	PutBucketTagging_long_tags(s)
+	PutBucketTagging_success(s)
+}
+
+func TestGetBucketTagging(s *S3Conf) {
+	GetBucketTagging_non_existing_bucket(s)
+	GetBucketTagging_success(s)
+}
+
+func TestDeleteBucketTagging(s *S3Conf) {
+	DeleteBucketTagging_non_existing_object(s)
+	DeleteBucketTagging_success_status(s)
+	DeleteBucketTagging_success(s)
+}
+
+func TestPutObject(s *S3Conf) {
+	PutObject_non_existing_bucket(s)
+	PutObject_special_chars(s)
+	PutObject_invalid_long_tags(s)
+	PutObject_success(s)
+	PutObject_invalid_credentials(s)
+}
+
+func TestHeadObject(s *S3Conf) {
+	HeadObject_non_existing_object(s)
+	HeadObject_success(s)
+}
+
+func TestGetObject(s *S3Conf) {
+	GetObject_non_existing_key(s)
+	GetObject_invalid_ranges(s)
+	GetObject_with_meta(s)
+	GetObject_success(s)
+	GetObject_by_range_success(s)
+}
+
+func TestListObjects(s *S3Conf) {
+	ListObjects_non_existing_bucket(s)
+	ListObjects_with_prefix(s)
+	ListObject_truncated(s)
+	ListObjects_invalid_max_keys(s)
+	ListObjects_max_keys_0(s)
+	ListObjects_delimiter(s)
+	ListObjects_max_keys_none(s)
+	ListObjects_marker_not_from_obj_list(s)
+}
+
+func TestListObjectsV2(s *S3Conf) {
+	ListObjectsV2_start_after(s)
+	ListObjectsV2_both_start_after_and_continuation_token(s)
+	ListObjectsV2_start_after_not_in_list(s)
+	ListObjectsV2_start_after_empty_result(s)
+}
+
+func TestDeleteObject(s *S3Conf) {
+	DeleteObject_non_existing_object(s)
+	DeleteObject_success(s)
+	DeleteObject_success_status_code(s)
+}
+
+func TestDeleteObjects(s *S3Conf) {
+	DeleteObjects_empty_input(s)
+	DeleteObjects_non_existing_objects(s)
+	DeleteObjects_success(s)
+}
+
+func TestCopyObject(s *S3Conf) {
+	CopyObject_non_existing_dst_bucket(s)
+	CopyObject_not_owned_source_bucket(s)
+	CopyObject_copy_to_itself(s)
+	CopyObject_to_itself_with_new_metadata(s)
+	CopyObject_success(s)
+}
+
+func TestPutObjectTagging(s *S3Conf) {
+	PutObjectTagging_non_existing_object(s)
+	PutObjectTagging_long_tags(s)
+	PutObjectTagging_success(s)
+}
+
+func TestGetObjectTagging(s *S3Conf) {
+	GetObjectTagging_non_existing_object(s)
+	GetObjectTagging_success(s)
+}
+
+func TestDeleteObjectTagging(s *S3Conf) {
+	DeleteObjectTagging_non_existing_object(s)
+	DeleteObjectTagging_success_status(s)
+	DeleteObjectTagging_success(s)
+}
+
+func TestCreateMultipartUpload(s *S3Conf) {
+	CreateMultipartUpload_non_existing_bucket(s)
+	CreateMultipartUpload_success(s)
+}
+
+func TestUploadPart(s *S3Conf) {
+	UploadPart_non_existing_bucket(s)
+	UploadPart_invalid_part_number(s)
+	UploadPart_non_existing_key(s)
+	UploadPart_non_existing_mp_upload(s)
+	UploadPart_success(s)
+}
+
+func TestUploadPartCopy(s *S3Conf) {
+	UploadPartCopy_non_existing_bucket(s)
+	UploadPartCopy_incorrect_uploadId(s)
+	UploadPartCopy_incorrect_object_key(s)
+	UploadPartCopy_invalid_part_number(s)
+	UploadPartCopy_invalid_copy_source(s)
+	UploadPartCopy_non_existing_source_bucket(s)
+	UploadPartCopy_non_existing_source_object_key(s)
+	UploadPartCopy_success(s)
+	UploadPartCopy_by_range_invalid_range(s)
+	UploadPartCopy_greater_range_than_obj_size(s)
+	UploadPartCopy_by_range_success(s)
+}
+
+func TestListParts(s *S3Conf) {
+	ListParts_incorrect_uploadId(s)
+	ListParts_incorrect_object_key(s)
+	ListParts_success(s)
+}
+
+func TestListMultipartUploads(s *S3Conf) {
+	ListMultipartUploads_non_existing_bucket(s)
+	ListMultipartUploads_empty_result(s)
+	ListMultipartUploads_invalid_max_uploads(s)
+	ListMultipartUploads_max_uploads(s)
+	ListMultipartUploads_incorrect_next_key_marker(s)
+	ListMultipartUploads_ignore_upload_id_marker(s)
+	ListMultipartUploads_success(s)
+}
+
+func TestAbortMultipartUpload(s *S3Conf) {
+	AbortMultipartUpload_non_existing_bucket(s)
+	AbortMultipartUpload_incorrect_uploadId(s)
+	AbortMultipartUpload_incorrect_object_key(s)
+	AbortMultipartUpload_success(s)
+	AbortMultipartUpload_success_status_code(s)
+}
+
+func TestCompleteMultipartUpload(s *S3Conf) {
+	CompletedMultipartUpload_non_existing_bucket(s)
+	CompleteMultipartUpload_invalid_part_number(s)
+	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_success(s)
+}
+
+func TestPutBucketAcl(s *S3Conf) {
+	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_invalid_acl_canned_and_acp(s)
+	PutBucketAcl_invalid_acl_canned_and_grants(s)
+	PutBucketAcl_invalid_acl_acp_and_grants(s)
+	PutBucketAcl_invalid_owner(s)
+	PutBucketAcl_success_access_denied(s)
+	PutBucketAcl_success_grants(s)
+	PutBucketAcl_success_canned_acl(s)
+	PutBucketAcl_success_acp(s)
+}
+
+func TestGetBucketAcl(s *S3Conf) {
+	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_access_denied(s)
+	GetBucketAcl_success(s)
+}
+
+func TestFullFlow(s *S3Conf) {
+	TestAuthentication(s)
+	TestPresignedAuthentication(s)
+	TestCreateBucket(s)
+	TestHeadBucket(s)
+	TestListBuckets(s)
+	TestDeleteBucket(s)
+	TestPutBucketTagging(s)
+	TestGetBucketTagging(s)
+	TestDeleteBucketTagging(s)
+	TestPutObject(s)
+	TestHeadObject(s)
+	TestGetObject(s)
+	TestListObjects(s)
+	TestListObjectsV2(s)
+	TestDeleteObject(s)
+	TestDeleteObjects(s)
+	TestCopyObject(s)
+	TestPutObjectTagging(s)
+	TestDeleteObjectTagging(s)
+	TestCreateMultipartUpload(s)
+	TestUploadPart(s)
+	TestUploadPartCopy(s)
+	TestListParts(s)
+	TestListMultipartUploads(s)
+	TestAbortMultipartUpload(s)
+	TestCompleteMultipartUpload(s)
+	TestPutBucketAcl(s)
+	TestGetBucketAcl(s)
+}
+
+func TestPosix(s *S3Conf) {
+	PutObject_overwrite_dir_obj(s)
+	PutObject_overwrite_file_obj(s)
+	PutObject_dir_obj_with_data(s)
+	CreateMultipartUpload_dir_obj(s)
+}
+
+func TestIAM(s *S3Conf) {
+	IAM_user_access_denied(s)
+	IAM_userplus_access_denied(s)
+	IAM_userplus_CreateBucket(s)
+	IAM_admin_ChangeBucketOwner(s)
+}
+
+type IntTests map[string]func(s *S3Conf) error
+
+func GetIntTests() IntTests {
+	return IntTests{
+		"Authentication_empty_auth_header":                      Authentication_empty_auth_header,
+		"Authentication_invalid_auth_header":                    Authentication_invalid_auth_header,
+		"Authentication_unsupported_signature_version":          Authentication_unsupported_signature_version,
+		"Authentication_malformed_credentials":                  Authentication_malformed_credentials,
+		"Authentication_malformed_credentials_invalid_parts":    Authentication_malformed_credentials_invalid_parts,
+		"Authentication_credentials_terminated_string":          Authentication_credentials_terminated_string,
+		"Authentication_credentials_incorrect_service":          Authentication_credentials_incorrect_service,
+		"Authentication_credentials_incorrect_region":           Authentication_credentials_incorrect_region,
+		"Authentication_credentials_invalid_date":               Authentication_credentials_invalid_date,
+		"Authentication_credentials_future_date":                Authentication_credentials_future_date,
+		"Authentication_credentials_past_date":                  Authentication_credentials_past_date,
+		"Authentication_credentials_non_existing_access_key":    Authentication_credentials_non_existing_access_key,
+		"Authentication_invalid_signed_headers":                 Authentication_invalid_signed_headers,
+		"Authentication_missing_date_header":                    Authentication_missing_date_header,
+		"Authentication_invalid_date_header":                    Authentication_invalid_date_header,
+		"Authentication_date_mismatch":                          Authentication_date_mismatch,
+		"Authentication_incorrect_payload_hash":                 Authentication_incorrect_payload_hash,
+		"Authentication_incorrect_md5":                          Authentication_incorrect_md5,
+		"Authentication_signature_error_incorrect_secret_key":   Authentication_signature_error_incorrect_secret_key,
+		"PresignedAuth_missing_algo_query_param":                PresignedAuth_missing_algo_query_param,
+		"PresignedAuth_unsupported_algorithm":                   PresignedAuth_unsupported_algorithm,
+		"PresignedAuth_missing_credentials_query_param":         PresignedAuth_missing_credentials_query_param,
+		"PresignedAuth_malformed_creds_invalid_parts":           PresignedAuth_malformed_creds_invalid_parts,
+		"PresignedAuth_creds_invalid_terminator":                PresignedAuth_creds_invalid_terminator,
+		"PresignedAuth_creds_incorrect_service":                 PresignedAuth_creds_incorrect_service,
+		"PresignedAuth_creds_incorrect_region":                  PresignedAuth_creds_incorrect_region,
+		"PresignedAuth_creds_invalid_date":                      PresignedAuth_creds_invalid_date,
+		"PresignedAuth_missing_date_query":                      PresignedAuth_missing_date_query,
+		"PresignedAuth_dates_mismatch":                          PresignedAuth_dates_mismatch,
+		"PresignedAuth_non_existing_access_key_id":              PresignedAuth_non_existing_access_key_id,
+		"PresignedAuth_missing_signed_headers_query_param":      PresignedAuth_missing_signed_headers_query_param,
+		"PresignedAuth_missing_expiration_query_param":          PresignedAuth_missing_expiration_query_param,
+		"PresignedAuth_invalid_expiration_query_param":          PresignedAuth_invalid_expiration_query_param,
+		"PresignedAuth_negative_expiration_query_param":         PresignedAuth_negative_expiration_query_param,
+		"PresignedAuth_exceeding_expiration_query_param":        PresignedAuth_exceeding_expiration_query_param,
+		"PresignedAuth_expired_request":                         PresignedAuth_expired_request,
+		"PresignedAuth_incorrect_secret_key":                    PresignedAuth_incorrect_secret_key,
+		"PresignedAuth_PutObject_success":                       PresignedAuth_PutObject_success,
+		"PresignedAuth_Put_GetObject_with_data":                 PresignedAuth_Put_GetObject_with_data,
+		"PresignedAuth_Put_GetObject_with_UTF8_chars":           PresignedAuth_Put_GetObject_with_UTF8_chars,
+		"PresignedAuth_UploadPart":                              PresignedAuth_UploadPart,
+		"CreateBucket_invalid_bucket_name":                      CreateBucket_invalid_bucket_name,
+		"CreateBucket_existing_bucket":                          CreateBucket_existing_bucket,
+		"CreateBucket_as_user":                                  CreateBucket_as_user,
+		"CreateDeleteBucket_success":                            CreateDeleteBucket_success,
+		"CreateBucket_default_acl":                              CreateBucket_default_acl,
+		"CreateBucket_non_default_acl":                          CreateBucket_non_default_acl,
+		"HeadBucket_non_existing_bucket":                        HeadBucket_non_existing_bucket,
+		"HeadBucket_success":                                    HeadBucket_success,
+		"ListBuckets_as_user":                                   ListBuckets_as_user,
+		"ListBuckets_as_admin":                                  ListBuckets_as_admin,
+		"ListBuckets_success":                                   ListBuckets_success,
+		"DeleteBucket_non_existing_bucket":                      DeleteBucket_non_existing_bucket,
+		"DeleteBucket_non_empty_bucket":                         DeleteBucket_non_empty_bucket,
+		"DeleteBucket_success_status_code":                      DeleteBucket_success_status_code,
+		"PutBucketTagging_non_existing_bucket":                  PutBucketTagging_non_existing_bucket,
+		"PutBucketTagging_long_tags":                            PutBucketTagging_long_tags,
+		"PutBucketTagging_success":                              PutBucketTagging_success,
+		"GetBucketTagging_non_existing_bucket":                  GetBucketTagging_non_existing_bucket,
+		"GetBucketTagging_success":                              GetBucketTagging_success,
+		"DeleteBucketTagging_non_existing_object":               DeleteBucketTagging_non_existing_object,
+		"DeleteBucketTagging_success_status":                    DeleteBucketTagging_success_status,
+		"DeleteBucketTagging_success":                           DeleteBucketTagging_success,
+		"PutObject_non_existing_bucket":                         PutObject_non_existing_bucket,
+		"PutObject_special_chars":                               PutObject_special_chars,
+		"PutObject_invalid_long_tags":                           PutObject_invalid_long_tags,
+		"PutObject_success":                                     PutObject_success,
+		"HeadObject_non_existing_object":                        HeadObject_non_existing_object,
+		"HeadObject_success":                                    HeadObject_success,
+		"GetObject_non_existing_key":                            GetObject_non_existing_key,
+		"GetObject_invalid_ranges":                              GetObject_invalid_ranges,
+		"GetObject_with_meta":                                   GetObject_with_meta,
+		"GetObject_success":                                     GetObject_success,
+		"GetObject_by_range_success":                            GetObject_by_range_success,
+		"ListObjects_non_existing_bucket":                       ListObjects_non_existing_bucket,
+		"ListObjects_with_prefix":                               ListObjects_with_prefix,
+		"ListObject_truncated":                                  ListObject_truncated,
+		"ListObjects_invalid_max_keys":                          ListObjects_invalid_max_keys,
+		"ListObjects_max_keys_0":                                ListObjects_max_keys_0,
+		"ListObjects_delimiter":                                 ListObjects_delimiter,
+		"ListObjects_max_keys_none":                             ListObjects_max_keys_none,
+		"ListObjects_marker_not_from_obj_list":                  ListObjects_marker_not_from_obj_list,
+		"ListObjectsV2_start_after":                             ListObjectsV2_start_after,
+		"ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token,
+		"ListObjectsV2_start_after_not_in_list":                 ListObjectsV2_start_after_not_in_list,
+		"ListObjectsV2_start_after_empty_result":                ListObjectsV2_start_after_empty_result,
+		"DeleteObject_non_existing_object":                      DeleteObject_non_existing_object,
+		"DeleteObject_success":                                  DeleteObject_success,
+		"DeleteObject_success_status_code":                      DeleteObject_success_status_code,
+		"DeleteObjects_empty_input":                             DeleteObjects_empty_input,
+		"DeleteObjects_non_existing_objects":                    DeleteObjects_non_existing_objects,
+		"DeleteObjects_success":                                 DeleteObjects_success,
+		"CopyObject_non_existing_dst_bucket":                    CopyObject_non_existing_dst_bucket,
+		"CopyObject_not_owned_source_bucket":                    CopyObject_not_owned_source_bucket,
+		"CopyObject_copy_to_itself":                             CopyObject_copy_to_itself,
+		"CopyObject_to_itself_with_new_metadata":                CopyObject_to_itself_with_new_metadata,
+		"CopyObject_success":                                    CopyObject_success,
+		"PutObjectTagging_non_existing_object":                  PutObjectTagging_non_existing_object,
+		"PutObjectTagging_long_tags":                            PutObjectTagging_long_tags,
+		"PutObjectTagging_success":                              PutObjectTagging_success,
+		"GetObjectTagging_non_existing_object":                  GetObjectTagging_non_existing_object,
+		"GetObjectTagging_success":                              GetObjectTagging_success,
+		"DeleteObjectTagging_non_existing_object":               DeleteObjectTagging_non_existing_object,
+		"DeleteObjectTagging_success_status":                    DeleteObjectTagging_success_status,
+		"DeleteObjectTagging_success":                           DeleteObjectTagging_success,
+		"CreateMultipartUpload_non_existing_bucket":             CreateMultipartUpload_non_existing_bucket,
+		"CreateMultipartUpload_success":                         CreateMultipartUpload_success,
+		"UploadPart_non_existing_bucket":                        UploadPart_non_existing_bucket,
+		"UploadPart_invalid_part_number":                        UploadPart_invalid_part_number,
+		"UploadPart_non_existing_key":                           UploadPart_non_existing_key,
+		"UploadPart_non_existing_mp_upload":                     UploadPart_non_existing_mp_upload,
+		"UploadPart_success":                                    UploadPart_success,
+		"UploadPartCopy_non_existing_bucket":                    UploadPartCopy_non_existing_bucket,
+		"UploadPartCopy_incorrect_uploadId":                     UploadPartCopy_incorrect_uploadId,
+		"UploadPartCopy_incorrect_object_key":                   UploadPartCopy_incorrect_object_key,
+		"UploadPartCopy_invalid_part_number":                    UploadPartCopy_invalid_part_number,
+		"UploadPartCopy_invalid_copy_source":                    UploadPartCopy_invalid_copy_source,
+		"UploadPartCopy_non_existing_source_bucket":             UploadPartCopy_non_existing_source_bucket,
+		"UploadPartCopy_non_existing_source_object_key":         UploadPartCopy_non_existing_source_object_key,
+		"UploadPartCopy_success":                                UploadPartCopy_success,
+		"UploadPartCopy_by_range_invalid_range":                 UploadPartCopy_by_range_invalid_range,
+		"UploadPartCopy_greater_range_than_obj_size":            UploadPartCopy_greater_range_than_obj_size,
+		"UploadPartCopy_by_range_success":                       UploadPartCopy_by_range_success,
+		"ListParts_incorrect_uploadId":                          ListParts_incorrect_uploadId,
+		"ListParts_incorrect_object_key":                        ListParts_incorrect_object_key,
+		"ListParts_success":                                     ListParts_success,
+		"ListMultipartUploads_non_existing_bucket":              ListMultipartUploads_non_existing_bucket,
+		"ListMultipartUploads_empty_result":                     ListMultipartUploads_empty_result,
+		"ListMultipartUploads_invalid_max_uploads":              ListMultipartUploads_invalid_max_uploads,
+		"ListMultipartUploads_max_uploads":                      ListMultipartUploads_max_uploads,
+		"ListMultipartUploads_incorrect_next_key_marker":        ListMultipartUploads_incorrect_next_key_marker,
+		"ListMultipartUploads_ignore_upload_id_marker":          ListMultipartUploads_ignore_upload_id_marker,
+		"ListMultipartUploads_success":                          ListMultipartUploads_success,
+		"AbortMultipartUpload_non_existing_bucket":              AbortMultipartUpload_non_existing_bucket,
+		"AbortMultipartUpload_incorrect_uploadId":               AbortMultipartUpload_incorrect_uploadId,
+		"AbortMultipartUpload_incorrect_object_key":             AbortMultipartUpload_incorrect_object_key,
+		"AbortMultipartUpload_success":                          AbortMultipartUpload_success,
+		"AbortMultipartUpload_success_status_code":              AbortMultipartUpload_success_status_code,
+		"CompletedMultipartUpload_non_existing_bucket":          CompletedMultipartUpload_non_existing_bucket,
+		"CompleteMultipartUpload_invalid_part_number":           CompleteMultipartUpload_invalid_part_number,
+		"CompleteMultipartUpload_invalid_ETag":                  CompleteMultipartUpload_invalid_ETag,
+		"CompleteMultipartUpload_success":                       CompleteMultipartUpload_success,
+		"PutBucketAcl_non_existing_bucket":                      PutBucketAcl_non_existing_bucket,
+		"PutBucketAcl_invalid_acl_canned_and_acp":               PutBucketAcl_invalid_acl_canned_and_acp,
+		"PutBucketAcl_invalid_acl_canned_and_grants":            PutBucketAcl_invalid_acl_canned_and_grants,
+		"PutBucketAcl_invalid_acl_acp_and_grants":               PutBucketAcl_invalid_acl_acp_and_grants,
+		"PutBucketAcl_invalid_owner":                            PutBucketAcl_invalid_owner,
+		"PutBucketAcl_success_access_denied":                    PutBucketAcl_success_access_denied,
+		"PutBucketAcl_success_grants":                           PutBucketAcl_success_grants,
+		"PutBucketAcl_success_canned_acl":                       PutBucketAcl_success_canned_acl,
+		"PutBucketAcl_success_acp":                              PutBucketAcl_success_acp,
+		"GetBucketAcl_non_existing_bucket":                      GetBucketAcl_non_existing_bucket,
+		"GetBucketAcl_access_denied":                            GetBucketAcl_access_denied,
+		"GetBucketAcl_success":                                  GetBucketAcl_success,
+		"PutObject_overwrite_dir_obj":                           PutObject_overwrite_dir_obj,
+		"PutObject_overwrite_file_obj":                          PutObject_overwrite_file_obj,
+		"PutObject_dir_obj_with_data":                           PutObject_dir_obj_with_data,
+		"CreateMultipartUpload_dir_obj":                         CreateMultipartUpload_dir_obj,
+		"IAM_user_access_denied":                                IAM_user_access_denied,
+		"IAM_userplus_access_denied":                            IAM_userplus_access_denied,
+		"IAM_userplus_CreateBucket":                             IAM_userplus_CreateBucket,
+		"IAM_admin_ChangeBucketOwner":                           IAM_admin_ChangeBucketOwner,
+	}
+}

tests/integration/utils.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	rnd "math/rand"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"strings"
@@ -27,9 +28,11 @@ import (
 )
 
 var (
-	bcktCount  = 0
-	succUsrCrt = "The user has been created successfully"
-	failUsrCrt = "failed to create a user: update iam data: account already exists"
+	bcktCount            = 0
+	succUsrCrt           = "The user has been created successfully"
+	failUsrCrt           = "failed to create a user: update iam data: account already exists"
+	adminAccessDeniedMsg = "access denied: only admin users have access to this resource"
+	succDeleteUserMsg    = "The user has been deleted successfully"
 )
 
 func getBucketName() string {
@@ -150,6 +153,20 @@ func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) err
 	return nil
 }
 
+func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error {
+	runF(testName)
+	clt := s3.NewPresignClient(s3.NewFromConfig(s.Config()))
+
+	err := handler(clt)
+	if err != nil {
+		failF("%v: %v", testName, err)
+		return fmt.Errorf("%v: %w", testName, err)
+	}
+
+	passF(testName)
+	return nil
+}
+
 func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
 	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
 	if err != nil {
@@ -526,6 +543,18 @@ func createUsers(s *S3Conf, users []user) error {
 	return nil
 }
 
+func deleteUser(s *S3Conf, access string) error {
+	out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "delete-user", "-a", access)
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(string(out), succDeleteUserMsg) {
+		return fmt.Errorf("failed to delete the user account")
+	}
+
+	return nil
+}
+
 func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
 	for _, bucket := range buckets {
 		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
@@ -551,3 +580,26 @@ func genRandString(length int) string {
 	}
 	return string(result)
 }
+
+const (
+	credAccess int = iota
+	credDate
+	credRegion
+	credService
+	credTerminator
+)
+
+func changeAuthCred(uri, newVal string, index int) (string, error) {
+	urlParsed, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+
+	queries := urlParsed.Query()
+	creds := strings.Split(queries.Get("X-Amz-Credential"), "/")
+	creds[index] = newVal
+	queries.Set("X-Amz-Credential", strings.Join(creds, "/"))
+	urlParsed.RawQuery = queries.Encode()
+
+	return urlParsed.String(), nil
+}

tests/run.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Function to display help information
+show_help() {
+    echo "Usage: $0 [option...]"
+    echo "   -h, --help          Display this help message and exit"
+    echo "   -s, --static        Don't remove buckets between tests"
+    echo "   aws                 Run tests with aws cli"
+    echo "   aws-posix           Run posix tests with aws cli"
+    echo "   s3cmd               Run tests with s3cmd utility"
+    echo "   mc                  Run tests with mc utility"
+}
+
+handle_param() {
+  case $1 in
+      -h|--help)
+          show_help
+          exit 0
+          ;;
+      -s|--static)
+          export RECREATE_BUCKETS=false
+          ;;
+      aws|aws-posix|s3cmd|mc)
+          set_command_type "$1"
+          ;;
+      *) # Handle unrecognized options or positional arguments
+          echo "Unrecognized option: $1" >&2
+          exit 1
+          ;;
+  esac
+}
+
+set_command_type() {
+  if [[ -n $command_type ]]; then
+    echo "Error:  command type already set"
+    exit 1
+  fi
+  command_type=$1
+  export command_type
+}
+
+export RECREATE_BUCKETS=true
+while [[ "$#" -gt 0 ]]; do
+  handle_param "$1"
+  shift # past argument or value
+done
+
+if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+  exit 1
+fi
+
+if [[ $RECREATE_BUCKETS == false ]]; then
+  ./tests/setup_static.sh || exit_code=$?
+  if [[ exit_code -ne 0 ]]; then
+    exit 1
+  fi
+fi
+
+case $command_type in
+  aws)
+    "$HOME"/bin/bats ./tests/test_aws.sh || exit_code=$?
+    ;;
+  aws-posix)
+    "$HOME"/bin/bats ./tests/test_aws_posix.sh || exit_code=$?
+    ;;
+  s3cmd)
+    "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
+    ;;
+  mc)
+    "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
+    ;;
+esac
+
+exit $exit_code

tests/run_all.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+  exit 1
+fi
+if ! ./tests/run.sh aws; then
+  exit 1
+fi
+if ! ./tests/run.sh aws-posix; then
+  exit 1
+fi
+if ! ./tests/run.sh s3cmd; then
+  exit 1
+fi
+if ! ./tests/run.sh mc; then
+  exit 1
+fi
+if ! ./tests/run.sh -s aws; then
+  exit 1
+fi
+if ! ./tests/run.sh -s aws-posix; then
+  exit 1
+fi
+if ! ./tests/run.sh -s s3cmd; then
+  exit 1
+fi
+if ! ./tests/run.sh -s mc; then
+  exit 1
+fi
+exit 0

tests/s3_bucket_tests.sh

@@ -1,353 +0,0 @@
-#!/usr/bin/env bats
-
-source ./tests/tests.sh
-
-# create an AWS bucket
-# param:  bucket name
-# return 0 for success, 1 for failure
-create_bucket() {
-  if [ $# -ne 1 ]; then
-    echo "create bucket missing bucket name"
-    return 1
-  fi
-  local exit_code=0
-  local error
-  error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$?
-  if [ $exit_code -ne 0 ]; then
-    echo "error creating bucket: $error"
-    return 1
-  fi
-  return 0
-}
-
-# delete an AWS bucket
-# param:  bucket name
-# return 0 for success, 1 for failure
-delete_bucket() {
-  if [ $# -ne 1 ]; then
-    echo "delete bucket missing bucket name"
-    return 1
-  fi
-  local exit_code=0
-  local error
-  error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?"
-  if [ $exit_code -ne 0 ]; then
-    if [[ "$error" == *"The specified bucket does not exist"* ]]; then
-      return 0
-    else
-      echo "error deleting bucket: $error"
-      return 1
-    fi
-  fi
-  return 0
-}
-
-# check if bucket exists
-# param:  bucket name
-# return 0 for true, 1 for false, 2 for error
-bucket_exists() {
-  if [ $# -ne 1 ]; then
-    echo "bucket exists check missing bucket name"
-    return 2
-  fi
-  local exit_code=0
-  local error
-  error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
-  echo "Exit code: $exit_code, error: $error"
-  if [ $exit_code -ne 0 ]; then
-    if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
-      return 1
-    else
-      echo "error checking if bucket exists: $error"
-      return 2
-    fi
-  fi
-  return 0
-}
-
-# create bucket if it doesn't exist
-# param:  bucket name
-# return 0 for success, 1 for failure
-check_and_create_bucket() {
-  if [ $# -ne 1 ]; then
-    echo "bucket creation function requires bucket name"
-    return 1
-  fi
-  local exists_result
-  bucket_exists "$1" || exists_result=$?
-  if [[ $exists_result -eq 2 ]]; then
-    echo "Bucket existence check error"
-    return 1
-  fi
-  local create_result
-  if [[ $exists_result -eq 1 ]]; then
-    create_bucket "$1" || create_result=$?
-    if [[ $create_result -ne 0 ]]; then
-      echo "Error creating bucket"
-      return 1
-    fi
-  fi
-  return 0
-}
-
-# check if object exists on S3 via gateway
-# param:  object path
-# return 0 for true, 1 for false, 2 for error
-object_exists() {
-  if [ $# -ne 1 ]; then
-    echo "object exists check missing object name"
-    return 2
-  fi
-  local exit_code=0
-  local error
-  error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
-  if [ $exit_code -ne 0 ]; then
-    if [[ "$error" == "" ]]; then
-      return 1
-    else
-      echo "error checking if object exists: $error"
-      return 2
-    fi
-  fi
-  return 0
-}
-
-# add object to versitygw
-# params:  source file, destination copy location
-# return 0 for success, 1 for failure
-put_object() {
-  if [ $# -ne 2 ]; then
-    echo "put object command requires source, destination"
-    return 1
-  fi
-  local exit_code=0
-  local error
-  error=$(aws s3 cp "$1" s3://"$2" 2>&1) || exit_code=$?
-  if [ $exit_code -ne 0 ]; then
-    echo "error copying object to bucket: $error"
-    return 1
-  fi
-  return 0
-}
-
-# add object to versitygw if it doesn't exist
-# params:  source file, destination copy location
-# return 0 for success or already exists, 1 for failure
-check_and_put_object() {
-  if [ $# -ne 2 ]; then
-    echo "check and put object function requires source, destination"
-    return 1
-  fi
-  object_exists "$2" || local exists_result=$?
-  if [ $exists_result -eq 2 ]; then
-    echo "error checking if object exists"
-    return 1
-  fi
-  if [ $exists_result -eq 1 ]; then
-    put_object "$1" "$2" || local put_result=$?
-    if [ $put_result -ne 0 ]; then
-      echo "error adding object"
-      return 1
-    fi
-  fi
-  return 0
-}
-
-# delete object from versitygw
-# param:  object location
-# return 0 for success, 1 for failure
-delete_object() {
-  if [ $# -ne 1 ]; then
-    echo "delete object command requires object parameter"
-    return 1
-  fi
-  local exit_code=0
-  local error
-  error=$(aws s3 rm s3://"$1" 2>&1) || exit_code=$?
-  if [ $exit_code -ne 0 ]; then
-    echo "error deleting object: $error"
-    return 1
-  fi
-  return 0
-}
-
-# list buckets on versitygw
-# no params
-# export bucket_array (bucket names) on success, return 1 for failure
-list_buckets() {
-  local exit_code=0
-  local output
-  output=$(aws s3 ls 2>&1) || exit_code=$?
-  if [ $exit_code -ne 0 ]; then
-    echo "error listing buckets: $output"
-    return 1
-  fi
-
-  bucket_array=()
-  while IFS= read -r line; do
-    bucket_name=$(echo "$line" | awk '{print $NF}')
-    bucket_array+=("$bucket_name")
-  done <<< "$output"
-
-  export bucket_array
-}
-
-# list objects on versitygw, in bucket or folder
-# param:  path of bucket or folder
-# export object_array (object names) on success, return 1 for failure
-list_objects() {
-  if [ $# -ne 1 ]; then
-    echo "list objects command requires bucket or folder"
-    return 1
-  fi
-  local exit_code=0
-  local output
-  output=$(aws s3 ls s3://"$1" 2>&1) || exit_code=$?
-  if [ $exit_code -ne 0 ]; then
-    echo "error listing objects: $output"
-    return 1
-  fi
-
-  object_array=()
-  while IFS= read -r line; do
-    object_name=$(echo "$line" | awk '{print $NF}')
-    object_array+=("$object_name")
-  done <<< "$output"
-
-  export object_array
-}
-
-# test creation and deletion of bucket on versitygw
-@test "create_delete_bucket_test" {
-
-  local bucket_name="versity-gwtest-create-delete-bucket-test"
-
-  bucket_exists $bucket_name || local exists=$?
-  if [[ $exists -eq 2 ]]; then
-    fail "Bucket existence check error"
-  fi
-  if [[ $exists -eq 0 ]]; then
-    delete_bucket $bucket_name || local delete_result=$?
-    [[ $delete_result -eq 0 ]] || fail "Failed to delete bucket"
-    bucket_exists $bucket_name || local exists_two=$?
-    [[ $exists_two -eq 1 ]] || fail "Failed bucket deletion"
-  fi
-  create_bucket $bucket_name || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Failed to create bucket"
-  bucket_exists $bucket_name || local exists_three=$?
-  [[ $exists_three -eq 0 ]] || fail "Failed bucket existence check"
-  delete_bucket $bucket_name || local delete_result_two=$?
-  [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
-}
-
-# test adding and removing an object on versitygw
-@test "put_object_test" {
-
-  local bucket_name="versity-gwtest-put-object-test"
-  local object_name="test-object"
-
-  bucket_exists $bucket_name || local bucket_exists=$?
-  if [[ $bucket_exists -eq 2 ]]; then
-    fail "Bucket existence check error"
-  fi
-  local object="$bucket_name"/"$object_name"
-  if [[ $bucket_exists -eq 0 ]]; then
-    object_exists "$object" || local object_exists=$?
-    if [[ $object_exists -eq 2 ]]; then
-      fail "Object existence check error"
-    fi
-    if [[ $object_exists -eq 0 ]]; then
-      delete_object "$object" || local delete_object=$?
-      [[ $delete_object -eq 0 ]] || fail "Failed to delete object"
-    fi
-    delete_bucket $bucket_name || local delete_bucket=$?
-    [[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
-  fi
-  touch "$object_name"
-  create_bucket $bucket_name || local create_bucket=$?
-  [[ $create_bucket -eq 0 ]] || fail "Failed to create bucket"
-  put_object "$object_name" "$object" || local put_object=$?
-  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
-  object_exists "$object" || local object_exists_two=$?
-  [[ $object_exists_two -eq 0 ]] || fail "Object not added to bucket"
-  delete_object "$object" || local delete_object_two=$?
-  [[ $delete_object_two -eq 0 ]] || fail "Failed to delete object"
-  delete_bucket $bucket_name || local delete_bucket=$?
-  [[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
-  rm "$object_name"
-}
-
-# test listing buckets on versitygw
-@test "test_list_buckets" {
-
-  bucket_name_one="versity-gwtest-list-one"
-  bucket_name_two="versity-gwtest-list-two"
-
-  bucket_exists $bucket_name_one || local exists=$?
-  if [[ $exists -eq 2 ]]; then
-    fail "Bucket existence check error"
-  fi
-  if [[ $exists -eq 1 ]]; then
-    create_bucket $bucket_name_one || local bucket_create_one=$?
-    [[ $bucket_create_one -eq 0 ]] || fail "Failed to create bucket"
-  fi
-  bucket_exists $bucket_name_two || local exists_two=$?
-  if [[ $exists_two -eq 2 ]]; then
-    fail "Bucket existence check error"
-  fi
-  if [[ $exists_two -eq 1 ]]; then
-    create_bucket $bucket_name_two || local bucket_create_two=$?
-    [[ $bucket_create_two -eq 0 ]] || fail "Failed to create bucket"
-  fi
-  list_buckets
-  local bucket_one_found=false
-  local bucket_two_found=false
-  for bucket in "${bucket_array[@]}"; do
-    if [ "$bucket" == $bucket_name_one ]; then
-      bucket_one_found=true
-    elif [ "$bucket" == $bucket_name_two ]; then
-      bucket_two_found=true
-    fi
-    if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then
-      return
-    fi
-  done
-  fail "$bucket_name_one and/or $bucket_name_two not listed (all buckets: ${bucket_array[*]})"
-  delete_bucket $bucket_name_one || local deleted_one=$?
-  [[ $deleted_one -eq 0 ]] || fail "Failed to delete bucket one"
-  delete_bucket $bucket_name_two || local deleted_two=$?
-  [[ $deleted_two -eq 0 ]] || fail "Failed to delete bucket one"
-}
-
-# test listing a bucket's objects on versitygw
-@test test_list_objects {
-
-  bucket_name="versity-gwtest-list-object"
-  object_one="test-file-one"
-  object_two="test-file-two"
-
-  touch $object_one $object_two
-  check_and_create_bucket $bucket_name || local result_one=$?
-  [[ result_one -eq 0 ]] || fail "Error creating bucket"
-  put_object $object_one "$bucket_name"/"$object_one"  || local result_two=$?
-  [[ result_two -eq 0 ]] || fail "Error adding object one"
-  put_object $object_two "$bucket_name"/"$object_two" || local result_three=$?
-  [[ result_three -eq 0 ]] || fail "Error adding object two"
-  list_objects $bucket_name
-  local object_one_found=false
-  local object_two_found=false
-  for object in "${object_array[@]}"; do
-    if [ "$object" == $object_one ]; then
-      object_one_found=true
-    elif [ "$object" == $object_two ]; then
-      object_two_found=true
-    fi
-  done
-  if [ $object_one_found != true ] || [ $object_two_found != true ]; then
-    fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
-  fi
-  delete_object "$bucket_name"/"$object_one"
-  delete_object "$bucket_name"/"$object_two"
-  delete_bucket $bucket_name
-  rm $object_one $object_two
-}

tests/s3cfg.local.default

@@ -0,0 +1,8 @@
+# Setup endpoint
+host_base = 127.0.0.1:7070
+host_bucket = 127.0.0.1:7070
+bucket_location = us-east-1
+use_https = True
+
+# Enable S3 v4 signature APIs
+signature_v2 = False

tests/setup.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+source ./tests/setup_mc.sh
+source ./tests/versity.sh
+
+# bats setup function
+setup() {
+  start_versity || start_result=$?
+  if [[ $start_result -ne 0 ]]; then
+    echo "error starting versity executable"
+    return 1
+  fi
+
+  check_params || check_result=$?
+  if [[ $check_result -ne 0 ]]; then
+    echo "parameter check failed"
+    return 1
+  fi
+
+  if [[ $RUN_S3CMD == true ]]; then
+    S3CMD_OPTS=()
+    S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
+    S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
+    S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
+    export S3CMD_CONFIG S3CMD_OPTS
+  fi
+
+  if [[ $RUN_MC == true ]]; then
+    check_add_mc_alias || check_result=$?
+    if [[ $check_result -ne 0 ]]; then
+      echo "mc alias check/add failed"
+      return 1
+    fi
+  fi
+
+  export AWS_PROFILE \
+    BUCKET_ONE_NAME \
+    BUCKET_TWO_NAME
+}
+
+# make sure required environment variables for tests are defined properly
+# return 0 for yes, 1 for no
+check_params() {
+  if [ -z "$BUCKET_ONE_NAME" ]; then
+    echo "No bucket one name set"
+    return 1
+  elif [ -z "$BUCKET_TWO_NAME" ]; then
+    echo "No bucket two name set"
+    return 1
+  elif [ -z "$RECREATE_BUCKETS" ]; then
+    echo "No recreate buckets parameter set"
+    return 1
+  elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then
+    echo "RECREATE_BUCKETS must be 'true' or 'false'"
+    return 1
+  fi
+  return 0
+}
+
+# fail a test
+# param:  error message
+fail() {
+  echo "$1"
+  return 1
+}
+
+# bats teardown function
+teardown() {
+  stop_versity
+}

tests/setup_mc.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+check_for_alias() {
+  local alias_result
+  aliases=$(mc alias list)
+  if [[ $alias_result -ne 0 ]]; then
+    echo "error checking for aliases: $aliases"
+    return 2
+  fi
+  while IFS= read -r line; do
+    error=$(echo "$line" | grep -w "$MC_ALIAS ")
+    if [[ $? -eq 0 ]]; then
+      return 0
+    fi
+  done <<< "$aliases"
+  return 1
+}
+
+check_add_mc_alias() {
+  check_for_alias || alias_result=$?
+  if [[ $alias_result -eq 2 ]]; then
+    echo "error checking for aliases"
+    return 1
+  fi
+  if [[ $alias_result -eq 0 ]]; then
+    return 0
+  fi
+  local set_result
+  error=$(mc alias set --insecure "$MC_ALIAS" "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$?
+  if [[ $set_result -ne 0 ]]; then
+    echo "error setting alias: $error"
+    return 1
+  fi
+  return 0
+}

tests/setup_static.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source ./tests/setup.sh
+setup
+aws --no-verify-ssl s3 mb s3://"$BUCKET_ONE_NAME"
+aws --no-verify-ssl s3 mb s3://"$BUCKET_TWO_NAME"
+teardown

tests/teardown_static.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source ./tests/setup.sh
+setup
+aws --no-verify-ssl s3 rb s3://"$BUCKET_ONE_NAME"
+aws --no-verify-ssl s3 rb s3://"$BUCKET_TWO_NAME"
+teardown

tests/test_aws.sh

@@ -0,0 +1,317 @@
+#!/usr/bin/env bats
+
+source ./tests/setup.sh
+source ./tests/util.sh
+source ./tests/util_file.sh
+source ./tests/test_common.sh
+
+# test creation and deletion of bucket on versitygw
+@test "test_create_delete_bucket_aws" {
+  test_common_create_delete_bucket "aws"
+}
+
+# test adding and removing an object on versitygw
+@test "test_put_object-with-data" {
+  test_common_put_object_with_data "aws"
+}
+
+@test "test_put_object-no-data" {
+  test_common_put_object_no_data "aws"
+}
+
+# test listing buckets on versitygw
+@test "test_list_buckets" {
+  test_common_list_buckets "aws"
+}
+
+# test listing a bucket's objects on versitygw
+@test "test_list_objects" {
+  test_common_list_objects "aws"
+}
+
+# test ability to retrieve bucket ACLs
+@test "test_get_bucket_acl" {
+
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating bucket"
+
+  get_bucket_acl "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+
+  id=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq '.Owner.ID')
+  [[ $id == '"'"$AWS_ACCESS_KEY_ID"'"' ]] || fail "Acl mismatch"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+# test ability to retrieve object ACLs
+#@test "test_get_object_acl" {
+
+#  object_one="test-file-one"
+
+#  setup_bucket "$BUCKET_ONE_NAME" || local created=$?
+#  [[ $created -eq 0 ]] || fail "Error creating bucket"
+#  create_test_files "$object_one" || local created=$?
+#  [[ $created -eq 0 ]] || fail "Error creating test file"
+#  put_object "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one"  || local result=$?
+#  [[ result -eq 0 ]] || fail "Error adding object one"
+
+#  get_object_acl "$BUCKET_ONE_NAME" "$object_one" || local result=$?
+#  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+
+#  id=$(echo "$acl" | jq '.Owner.ID')
+#  [[ $id == '"'"$AWS_ACCESS_KEY_ID"'"' ]] || fail "Acl mismatch"
+
+#  delete_bucket_or_contents "$BUCKET_ONE_NAME"
+#}
+
+
+# test ability to delete multiple objects from bucket
+@test "test_delete_objects" {
+
+  local object_one="test-file-one"
+  local object_two="test-file-two"
+
+  create_test_files "$object_one" "$object_two" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result_one=$?
+  [[ $result_one -eq 0 ]] || fail "Error creating bucket"
+
+  put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one"  || local result_two=$?
+  [[ $result_two -eq 0 ]] || fail "Error adding object one"
+  put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
+  [[ $result_three -eq 0 ]] || fail "Error adding object two"
+
+  error=$(aws --no-verify-ssl s3api delete-objects --bucket "$BUCKET_ONE_NAME" --delete '{
+    "Objects": [
+      {"Key": "test-file-one"},
+      {"Key": "test-file-two"}
+    ]
+  }') || local result=$?
+  [[ $result -eq 0 ]] || fail "Error deleting objects: $error"
+
+  object_exists "aws" "$BUCKET_ONE_NAME"/"$object_one" || local exists_one=$?
+  [[ $exists_one -eq 1 ]] || fail "Object one not deleted"
+  object_exists "aws" "$BUCKET_ONE_NAME"/"$object_two" || local exists_two=$?
+  [[ $exists_two -eq 1 ]] || fail "Object two not deleted"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$object_one" "$object_two"
+}
+
+# test abilty to set and retrieve bucket tags
+@test "test-set-get-bucket-tags" {
+  test_common_set_get_bucket_tags "aws"
+}
+
+# test v1 s3api list objects command
+@test "test-s3api-list-objects-v1" {
+
+  local object_one="test-file-one"
+  local object_two="test-file-two"
+  local object_two_data="test data\n"
+
+  create_test_files "$object_one" "$object_two" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
+  [[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
+  put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
+  [[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME"
+  key_one=$(echo "$objects" | jq '.Contents[0].Key')
+  [[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
+  size_one=$(echo "$objects" | jq '.Contents[0].Size')
+  [[ $size_one -eq 0 ]] || fail "Object one size mismatch"
+  key_two=$(echo "$objects" | jq '.Contents[1].Key')
+  [[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
+  size_two=$(echo "$objects" | jq '.Contents[1].Size')
+  [[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$object_one" "$object_two"
+}
+
+# test v2 s3api list objects command
+@test "test-s3api-list-objects-v2" {
+
+  local object_one="test-file-one"
+  local object_two="test-file-two"
+  local object_two_data="test data\n"
+
+  create_test_files "$object_one" "$object_two" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
+  [[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
+  put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
+  [[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
+
+  list_objects_s3api_v2 "$BUCKET_ONE_NAME"
+  key_one=$(echo "$objects" | jq '.Contents[0].Key')
+  [[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
+  size_one=$(echo "$objects" | jq '.Contents[0].Size')
+  [[ $size_one -eq 0 ]] || fail "Object one size mismatch"
+  key_two=$(echo "$objects" | jq '.Contents[1].Key')
+  [[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
+  size_two=$(echo "$objects" | jq '.Contents[1].Size')
+  [[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$object_one" "$object_two"
+}
+
+# test abilty to set and retrieve object tags
+@test "test-set-get-object-tags" {
+  test_common_set_get_object_tags "aws"
+}
+
+# test multi-part upload
+@test "test-multi-part-upload" {
+
+  local bucket_file="bucket-file"
+  bucket_file_data="test file\n"
+
+  create_test_files "$bucket_file" || local created=$?
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
+  [[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
+
+  copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
+  compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || compare_result=$?
+  [[ $compare_result -eq 0 ]] || fail "Files do not match"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}
+
+# test multi-part upload abort
+@test "test-multi-part-upload-abort" {
+
+  local bucket_file="bucket-file"
+  bucket_file_data="test file\n"
+
+  create_test_files "$bucket_file" || local created=$?
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
+  [[ $abort_result -eq 0 ]] || fail "Abort failed"
+
+  object_exists "aws" "$BUCKET_ONE_NAME/$bucket_file" || exists=$?
+  [[ $exists -eq 1 ]] || fail "Upload file exists after abort"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}
+
+# test multi-part upload list parts command
+@test "test-multipart-upload-list-parts" {
+
+  local bucket_file="bucket-file"
+  local bucket_file_data="test file\n"
+
+  create_test_files "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  list_parts "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || list_result=$?
+  [[ list_result -eq 0 ]] || fail "Listing multipart upload parts failed"
+
+  declare -a parts_map
+  for ((i=0;i<$4;i++)) {
+    local part_number
+    local etag
+    part_number=$(echo "$parts" | jq ".[$i].PartNumber")
+    if [[ $part_number -eq "" ]]; then
+      echo "error:  blank part number"
+      return 1
+    fi
+    etag=$(echo "$parts" | jq ".[$i].ETag")
+    if [[ $etag == "" ]]; then
+      echo "error:  blank etag"
+      return 1
+    fi
+    parts_map[$part_number]=$etag
+  }
+
+  for ((i=0;i<$4;i++)) {
+    local part_number
+    local etag
+    part_number=$(echo "$listed_parts" | jq ".Parts[$i].PartNumber")
+    etag=$(echo "$listed_parts" | jq ".Parts[$i].ETag")
+    if [[ ${parts_map[$part_number]} != "$etag" ]]; then
+      echo "error:  etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
+      return 1
+    fi
+  }
+
+  run_abort_command "$BUCKET_ONE_NAME" "$bucket_file" $upload_id
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}
+
+# test listing of active uploads
+@test "test-multipart-upload-list-uploads" {
+
+  local bucket_file_one="bucket-file-one"
+  local bucket_file_two="bucket-file-two"
+
+  create_test_files "$bucket_file_one" "$bucket_file_two" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two"
+  [[ $? -eq 0 ]] || fail "failed to list multipart uploads"
+
+  local key_one
+  local key_two
+  key_one=$(echo "$uploads" | jq '.Uploads[0].Key')
+  key_two=$(echo "$uploads" | jq '.Uploads[1].Key')
+  key_one=${key_one//\"/}
+  key_two=${key_two//\"/}
+  if [[ "$test_file_folder/$bucket_file_one" != *"$key_one" ]]; then
+    fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)"
+  fi
+  if [[ "$test_file_folder/$bucket_file_two" != *"$key_two" ]]; then
+    fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)"
+  fi
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$bucket_file_one" "$bucket_file_two"
+}
+
+@test "test-multipart-upload-from-bucket" {
+  local bucket_file="bucket-file"
+  bucket_file_data="test file\n"
+
+  create_test_files "$bucket_file" || local created=$?
+  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
+  [[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
+
+  copy_file "s3://$BUCKET_ONE_NAME/$bucket_file-copy" "$test_file_folder/$bucket_file-copy"
+  compare_files "$test_file_folder"/$bucket_file-copy "$test_file_folder"/$bucket_file || compare_result=$?
+  [[ $compare_result -eq 0 ]] || fail "Data doesn't match"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}

tests/test_aws_posix.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bats
+
+source ./tests/setup.sh
+source ./tests/util.sh
+source ./tests/util_file.sh
+source ./tests/util_posix.sh
+
+# test that changes to local folders and files are reflected on S3
+@test "test_local_creation_deletion" {
+
+  if [[ $RECREATE_BUCKETS != "true" ]]; then
+    return
+  fi
+
+  local object_name="test-object"
+
+  if [[ -e "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME" ]]; then
+    rm -rf "${LOCAL_FOLDER:?}"/"${BUCKET_ONE_NAME:?}"
+  fi
+
+  mkdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
+  local object="$BUCKET_ONE_NAME"/"$object_name"
+  touch "$LOCAL_FOLDER"/"$object"
+
+  bucket_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_exists_two=$?
+  [[ $bucket_exists_two -eq 0 ]] || fail "Failed bucket existence check"
+  object_exists_remote_and_local "$object" || local object_exists_two=$?
+  [[ $object_exists_two -eq 0 ]] || fail "Failed object existence check"
+
+  rm "$LOCAL_FOLDER"/"$object"
+  sleep 1
+  object_not_exists_remote_and_local "$object" || local object_deleted=$?
+  [[ $object_deleted -eq 0 ]] || fail "Failed object deletion check"
+
+  rmdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
+  sleep 1
+  bucket_not_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_deleted=$?
+  [[ $bucket_deleted -eq 0 ]] || fail "Failed bucket deletion check"
+}
+
+# test head-object command
+@test "test_head_object" {
+
+  local bucket_name=$BUCKET_ONE_NAME
+  local object_name="object-one"
+
+  create_test_files $object_name
+  if [ -e "$LOCAL_FOLDER"/"$bucket_name"/$object_name ]; then
+    chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
+  fi
+  setup_bucket "aws" "$bucket_name" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating bucket"
+  put_object "aws" "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name"  || local result="$?"
+  [[ result -eq 0 ]] || fail "Error adding object one"
+
+  chmod 000 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
+  sleep 1
+  object_is_accessible "$bucket_name" $object_name || local accessible=$?
+  [[ $accessible -eq 1 ]] || fail "Object should be inaccessible"
+
+  chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
+  sleep 1
+  object_is_accessible "$bucket_name" $object_name || local accessible_two=$?
+  [[ $accessible_two -eq 0 ]] || fail "Object should be accessible"
+
+  delete_object "aws" "$bucket_name"/$object_name
+  delete_bucket_or_contents "aws" "$bucket_name"
+  delete_test_files $object_name
+}
+
+# check info, accessiblity of bucket
+@test "test_get_bucket_info" {
+
+  if [ -e "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME" ]; then
+    chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
+    sleep 1
+  else
+    setup_bucket "aws" "$BUCKET_ONE_NAME" || local created=$?
+    [[ $created -eq 0 ]] || fail "Error creating bucket"
+  fi
+
+  chmod 000 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
+  sleep 1
+  bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible=$?
+  [[ $accessible -eq 1 ]] || fail "Bucket should be inaccessible"
+
+  chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
+  sleep 1
+  bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible_two=$?
+  [[ $accessible_two -eq 0 ]] || fail "Bucket should be accessible"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}