feat: Added s3 proxy comparison to upload, download and throughput benchmark tests

.dockerignore

@@ -1,46 +0,0 @@
-# Binaries for programs and plugins
-*.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-cmd/versitygw/versitygw
-/versitygw
-
-# Test binary, built with `go test -c`
-*.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
-*.out
-
-# Go workspace file
-go.work
-
-# ignore IntelliJ directories
-.idea
-
-# auto generated VERSION file
-VERSION
-
-# build output
-/versitygw.spec
-/versitygw.spec.in
-*.tar
-*.tar.gz
-**/rand.data
-/profile.txt
-
-dist/
-
-# Release config files
-/.github
-
-# Docker configuration files
-*Dockerfile
-/docker-compose.yml
-
-# read files
-/LICENSE
-/NOTICE
-/CODE_OF_CONDUCT.md
-/README.md

.env.dev

@@ -1,8 +0,0 @@
-POSIX_PORT=7071
-PROXY_PORT=7070
-ACCESS_KEY_ID=user
-SECRET_ACCESS_KEY=pass
-IAM_DIR=.
-SETUP_DIR=.
-AZ_ACCOUNT_NAME=devstoreaccount1
-AZ_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,23 +1,27 @@
 ---
-name: Bug Report
+name: Bug report
 about: Create a report to help us improve
-title: '[Bug] - <Short Description>'
+title: ''
 labels: bug
 assignees: ''
 
 ---
 
 **Describe the bug**
-<!-- A clear and concise description of what the bug is. -->
+A clear and concise description of what the bug is.
 
 **To Reproduce**
-<!-- Steps to reproduce the behavior. -->
+Steps to reproduce the behavior.
 
 **Expected behavior**
-<!-- A clear and concise description of what you expected to happen. -->
+A clear and concise description of what you expected to happen.
 
 **Server Version**
-<!-- output of: './versitygw -version && uname -a'  -->
+ output of
+```
+./versitygw -version
+uname -a
+```
 
 **Additional context**
-<!-- Describe s3 client and version if applicable.
+Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,14 +1,14 @@
 ---
-name: Feature Request
+name: Feature request
 about: Suggest an idea for this project
-title: '[Feature] - <Short Description>'
+title: ''
 labels: enhancement
 assignees: ''
 
 ---
 
 **Describe the solution you'd like**
-<!-- A clear and concise description of what you want to happen. -->
+A clear and concise description of what you want to happen.
 
 **Additional context**
-<!-- Add any other context or screenshots about the feature request here. -->
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/test_case.md

@@ -1,33 +0,0 @@
----
-name: Test Case Request
-about: Request new test cases or additional test coverage
-title: '[Test Case] - <Short Description>'
-labels: 'testcase'
-assignees: ''
-
----
-
-## Description
-<!-- Please provide a detailed description of the test case or test coverage request. -->
-
-## Purpose
-<!-- Explain why this test case is important and what it aims to achieve. -->
-
-## Scope
-<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
-
-## Acceptance Criteria
-<!-- List the criteria that must be met for the test case to be considered complete. -->
-
-1. 
-2. 
-3. 
-
-## Additional Context
-<!-- Add any other context or screenshots about the feature request here. -->
-
-## Resources
-<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
-
-
-**Thank you for contributing to our project!**

.github/dependabot.yml

@@ -8,7 +8,3 @@ updates:
       dev-dependencies:
         patterns:
           - "*"
-    allow:
-      # Allow both direct and indirect updates for all packages
-      - dependency-type: "all"
-

.github/workflows/azurite.yml

@@ -1,37 +0,0 @@
-name: azurite functional tests
-
-on: pull_request
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-              
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 'stable'
-        id: go
-
-      - name: Set up Docker Compose
-        run: |
-          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
-
-      - name: Wait for Azurite to be ready
-        run: sleep 40
-
-      - name: Get Dependencies
-        run: |
-          go mod download
-  
-      - name: Build and Run
-        run: |
-          make
-          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
-  
-      - name: Shut down services
-        run: |
-          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/docker-bats.yml

@@ -1,28 +0,0 @@
-name: docker bats tests
-
-on: pull_request
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Build Docker Image
-        run: |
-          cp tests/.env.docker.default tests/.env.docker
-          cp tests/.secrets.default tests/.secrets
-          docker build \
-            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
-            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
-            --build-arg="MC_FOLDER=linux-amd64" \
-            --progress=plain \
-            -f tests/Dockerfile_test_bats \
-            -t bats_test .
-
-      - name: Run Docker Container
-        run: |
-          docker compose -f tests/docker-compose-bats.yml --project-directory . \
-          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/docker.yml

@@ -1,57 +0,0 @@
-name: Publish Docker image
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  push_to_registries:
-    name: Push Docker image to multiple registries
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            versity/versitygw
-            ghcr.io/${{ github.repository }}
-
-      - name: Build and push Docker images
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            VERSION=${{ github.event.release.tag_name }}
-            TIME=${{ github.event.release.published_at }}
-            BUILD=${{ github.sha }}

.github/workflows/functional.yml

@@ -1,25 +1,24 @@
 name: functional tests
-
 on: pull_request
-
 jobs:
+
   build:
     name: RunTests
     runs-on: ubuntu-latest
     steps:
 
-    - name: Checkout
-      uses: actions/checkout@v4
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
 
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
 
     - name: Get Dependencies
       run: |
-        go mod download
+        go get -v -t -d ./...
 
     - name: Build and Run
       run: |

.github/workflows/go.yml

@@ -8,10 +8,10 @@ jobs:
     steps:
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
 
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go

.github/workflows/goreleaser.yml

@@ -15,21 +15,14 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
-      - name: Fetch tags
-        run: git fetch --force --tags
-
-      - name: Setup Go
-        uses: actions/setup-go@v5
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v4
         with:
           go-version: stable
-
-      - name: Run Releaser
-        uses: goreleaser/goreleaser-action@v5
+      - uses: goreleaser/goreleaser-action@v4
         with:
           distribution: goreleaser
           version: latest

.github/workflows/shellcheck.yml

@@ -1,16 +0,0 @@
-name: shellcheck
-on: pull_request
-jobs:
-
-  build:
-    name: Run shellcheck
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Check out code
-      uses: actions/checkout@v4
-
-    - name: Run checks
-      run: |
-        shellcheck --version
-        shellcheck -e SC1091 tests/*.sh tests/*/*.sh

.github/workflows/static.yml

@@ -7,18 +7,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 1
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
 
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
  
     - name: "staticcheck"
-      uses: dominikh/staticcheck-action@v1
-      with:
-        version: "latest"
+      uses: dominikh/staticcheck-action@v1.3.0
+      with: 
+        install-go: false

.github/workflows/system.yml

@@ -1,191 +0,0 @@
-name: system tests
-on: pull_request
-jobs:
-  build:
-    name: RunTests
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - set: "mc, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "mc-non-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "mc, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "mc-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "REST, posix, non-static, all, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-non-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, user, non-static, s3 IAM"
-            IAM_TYPE: s3
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, multipart, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-multipart"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, object, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-object"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, user, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, s3, multipart|object, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
-          - set: "s3api, s3, policy|user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy,s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
-          - set: "s3cmd, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, non-user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-non-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 'stable'
-        id: go
-
-      - name: Get Dependencies
-        run: |
-          go get -v -t -d ./...
-
-      - name: Install BATS
-        run: |
-          git clone https://github.com/bats-core/bats-core.git
-          cd bats-core && ./install.sh $HOME
-          git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
-          git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
-
-      - name: Install s3cmd
-        run: |
-          sudo apt-get install s3cmd
-
-      - name: Install mc
-        run: |
-          curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
-          chmod 755 /usr/local/bin/mc
-
-      - name: Install xmllint (for rest)
-        run: |
-          sudo apt-get install libxml2-utils
-
-      - name: Build and run
-        env:
-          IAM_TYPE: ${{ matrix.IAM_TYPE }}
-          RUN_SET: ${{ matrix.RUN_SET }}
-          AWS_PROFILE: versity
-          VERSITY_EXE: ${{ github.workspace }}/versitygw
-          RUN_VERSITYGW: true
-          BACKEND: ${{ matrix.BACKEND }}
-          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
-          CERT: ${{ github.workspace }}/cert.pem
-          KEY: ${{ github.workspace }}/versitygw.pem
-          LOCAL_FOLDER: /tmp/gw
-          BUCKET_ONE_NAME: versity-gwtest-bucket-one
-          BUCKET_TWO_NAME: versity-gwtest-bucket-two
-          USERS_FOLDER: /tmp/iam
-          USERS_BUCKET: versity-gwtest-iam
-          AWS_ENDPOINT_URL: https://127.0.0.1:7070
-          PORT: 7070
-          S3CMD_CONFIG: tests/s3cfg.local.default
-          MC_ALIAS: versity
-          LOG_LEVEL: 4
-          GOCOVERDIR: ${{ github.workspace }}/cover
-          USERNAME_ONE: ABCDEFG
-          PASSWORD_ONE: 1234567
-          USERNAME_TWO: HIJKLMN
-          PASSWORD_TWO: 8901234
-          TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
-          REMOVE_TEST_FILE_FOLDER: true
-          VERSIONING_DIR: ${{ github.workspace }}/versioning
-          COMMAND_LOG: command.log
-          TIME_LOG: time.log
-        run: |
-          make testbin
-          export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
-          export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-          export AWS_REGION=us-east-1
-          export AWS_ACCESS_KEY_ID_TWO=user
-          export AWS_SECRET_ACCESS_KEY_TWO=pass
-          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
-          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
-          aws configure set aws_region $AWS_REGION --profile versity
-          mkdir $LOCAL_FOLDER
-          export WORKSPACE=$GITHUB_WORKSPACE
-          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
-          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir $GOCOVERDIR $USERS_FOLDER
-          if [[ $RECREATE_BUCKETS == "false" ]]; then
-            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
-          fi
-          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
-
-      - name: Time report
-        run: cat ${{ github.workspace }}/time.log
-
-      - name: Coverage report
-        run: |
-          go tool covdata percent -i=cover

.gitignore

@@ -25,9 +25,6 @@ go.work
 # ignore IntelliJ directories
 .idea
 
-# ignore VS code directories
-.vscode
-
 # auto generated VERSION file
 VERSION
 
@@ -39,31 +36,3 @@ VERSION
 /profile.txt
 
 dist/
-
-# secrets file for local github-actions testing
-tests/.secrets*
-
-# IAM users files often created in testing
-users.json
-users.json.backup
-
-# env files for testing
-**/.env*
-**/!.env.default
-
-# s3cmd config files (testing)
-tests/s3cfg.local*
-tests/!s3cfg.local.default
-
-# keys
-*.pem
-
-# patches
-*.patch
-
-# grafana's local database (kept on filesystem for survival between instantiations)
-metrics-exploration/grafana_data/**
-
-# bats tools
-/tests/bats-assert
-/tests/bats-support

.goreleaser.yaml

@@ -6,16 +6,12 @@ builds:
   - goos:
       - linux
       - darwin
-      - freebsd
       # windows is untested, we can start doing windows releases
       # if someone is interested in taking on testing
       # - windows
-    env:
-      # disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
-      # once we need to enable this, we will need to do per distro releases
-      - CGO_ENABLED=0
     main: ./cmd/versitygw
-    binary: versitygw
+    binary: ./cmd/versitygw
+    id: versitygw
     goarch:
       - amd64
       - arm64
@@ -26,34 +22,17 @@ archives:
   - format: tar.gz
     # this name template makes the OS and Arch compatible with the results of uname.
     name_template: >-
-      {{ .ProjectName }}_v{{ .Version }}_
+      {{ .ProjectName }}_
       {{- title .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
-
-    # Set this to true if you want all files in the archive to be in a single directory.
-    # If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
-    # you'll get a folder 'goreleaser_Linux_arm64'.
-    # If set to false, all files are extracted separately.
-    # You can also set it to a custom folder name (templating is supported).
-    wrap_in_directory: true
-
     # use zip for windows archives
     format_overrides:
     - goos: windows
       format: zip
 
-    # Additional files/globs you want to add to the archive.
-    #
-    # Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
-    # Templates: allowed
-    files:
-      - README.md
-      - LICENSE
-      - NOTICE
-
 checksum:
   name_template: 'checksums.txt'
 
@@ -68,59 +47,5 @@ changelog:
       - '^test:'
       - '^Merge '
 
-nfpms:
-  - id: packages
-    package_name: versitygw
-    vendor: Versity Software
-    homepage: https://github.com/versity/versitygw
-    maintainer: Ben McClelland <ben.mcclelland@versity.com>
-
-    description: |-
-      The Versity S3 Gateway.
-      A high-performance tool facilitating translation between AWS S3 API
-      requests and various backend storage systems, including POSIX file
-      backend storage. Its stateless architecture enables deployment in
-      clusters for increased throughput, distributing requests across gateways
-      for optimal performance. With a focus on modularity, it supports future
-      extensions for additional backend systems.
-
-    license: Apache 2.0
-
-    builds:
-      - versitygw
-
-    formats:
-      - deb
-      - rpm
-
-    umask: 0o002
-    bindir: /usr/bin
-    epoch: "1"
-    release: "1"
-
-    rpm:
-      group: "System Environment/Daemons"
-      # RPM specific scripts.
-      scripts:
-        # The pretrans script runs before all RPM package transactions / stages.
-        #pretrans: ./extra/pretrans.sh
-        # The posttrans script runs after all RPM package transactions / stages.
-        posttrans: ./extra/posttrans.sh
-
-    contents:
-      - src: extra/versitygw@.service
-        dst: /lib/systemd/system/versitygw@.service
-
-      - src: extra/example.conf
-        dst: /etc/versitygw.d/example.conf
-        type: config
-  
-      - dst: /etc/versitygw.d
-        type: dir
-        file_info:
-          mode: 0700
-
-
-
 # yaml-language-server: $schema=https://goreleaser.com/static/schema.json
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile

@@ -1,35 +0,0 @@
-FROM golang:latest
-
-# Set build arguments with default values
-ARG VERSION="none"
-ARG BUILD="none"
-ARG TIME="none"
-
-# Set environment variables
-ENV VERSION=${VERSION}
-ENV BUILD=${BUILD}
-ENV TIME=${TIME}
-
-WORKDIR /app
-
-COPY go.mod ./
-RUN go mod download
-
-COPY ./ ./
-
-WORKDIR /app/cmd/versitygw
-ENV CGO_ENABLED=0
-RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
-
-FROM alpine:latest
-
-# These arguments can be overriden when building the image
-ARG IAM_DIR=/tmp/vgw
-ARG SETUP_DIR=/tmp/vgw
-
-RUN mkdir -p $IAM_DIR
-RUN mkdir -p $SETUP_DIR
-
-COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
-
-ENTRYPOINT [ "/app/versitygw" ]

Makefile

@@ -18,10 +18,6 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 
-# docker-compose
-DCCMD=docker-compose
-DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
-
 BIN=versitygw
 
 VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -63,31 +59,18 @@ cleanall: clean
 	rm -f $(BIN)
 	rm -f versitygw-*.tar
 	rm -f versitygw-*.tar.gz
+	rm -f versitygw.spec
+
+%.spec: %.spec.in
+	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
+	mv $@+ $@
 
 TARFILE = $(BIN)-$(VERSION).tar
 
-dist:
+dist: $(BIN).spec
 	echo $(VERSION) >VERSION
 	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
+	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
 	rm -f VERSION
+	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
-
-# Creates and runs S3 gateway instance in a docker container
-.PHONY: up-posix
-up-posix:
-	$(DOCKERCOMPOSE) up posix
-
-# Creates and runs S3 gateway proxy instance in a docker container
-.PHONY: up-proxy
-up-proxy:
-	$(DOCKERCOMPOSE) up proxy
-
-# Creates and runs S3 gateway to azurite instance in a docker container
-.PHONY: up-azurite
-up-azurite:
-	$(DOCKERCOMPOSE) up azurite azuritegw
-
-# Creates and runs both S3 gateway and proxy server instances in docker containers
-.PHONY: up-app
-up-app:
-	$(DOCKERCOMPOSE) up

README.md

@@ -6,36 +6,16 @@
   <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
 </picture>
 
- [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-### Binary release builds
-Download [latest release](https://github.com/versity/versitygw/releases)
- | Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
- |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
- |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
- 
-### Use Cases
-* Turn your local filesystem into an S3 server with a single command!
-* Proxy S3 requests to S3 storage
-* Simple to deploy S3 server with a single command
-* Protocol compatibility in `posix` allows common access to files via posix or S3
-* Simplified interface for adding new storage system support
+**Current status:** Beta: Most clients functional, work in progress for more test coverage. Issue reports welcome.
 
-### News
-Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
-
-### Mailing List
-Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
-
-### Documentation
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
-### Need help?
-Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
-<br>
-Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
+* Share filesystem directory via S3 protocol
+* Simple to deploy S3 server with a single command
+* Protocol compatibility allows common access to files via posix or S3 
 
-### Overview
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
@@ -68,7 +48,7 @@ The command format is
 ```
 versitygw [global options] command [command options] [arguments...]
 ```
-The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
+The global options are specified before the backend type and the backend options are specified after.
 
 ***
 

auth/acl.go

@@ -15,19 +15,17 @@
 package auth
 
 import (
-	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 )
 
 type ACL struct {
+	ACL      types.BucketCannedACL
 	Owner    string
 	Grantees []Grantee
 }
@@ -35,7 +33,6 @@ type ACL struct {
 type Grantee struct {
 	Permission types.Permission
 	Access     string
-	Type       types.Type
 }
 
 type GetBucketAclOutput struct {
@@ -43,36 +40,12 @@ type GetBucketAclOutput struct {
 	AccessControlList AccessControlList
 }
 
-type PutBucketAclInput struct {
-	Bucket              *string
-	ACL                 types.BucketCannedACL
-	AccessControlPolicy *AccessControlPolicy
-	GrantFullControl    *string
-	GrantRead           *string
-	GrantReadACP        *string
-	GrantWrite          *string
-	GrantWriteACP       *string
+type AccessControlList struct {
+	Grants []types.Grant `xml:"Grant"`
 }
-
 type AccessControlPolicy struct {
 	AccessControlList AccessControlList `xml:"AccessControlList"`
-	Owner             *types.Owner
-}
-
-type AccessControlList struct {
-	Grants []Grant `xml:"Grant"`
-}
-
-type Grant struct {
-	Grantee    *Grt
-	Permission types.Permission
-}
-
-type Grt struct {
-	XMLNS  string     `xml:"xmlns:xsi,attr"`
-	XMLXSI types.Type `xml:"xsi:type,attr"`
-	Type   types.Type `xml:"Type"`
-	ID     string     `xml:"ID"`
+	Owner             types.Owner
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -93,19 +66,11 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
-	grants := []Grant{}
+	grants := []types.Grant{}
 
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
-		grants = append(grants, Grant{
-			Grantee: &Grt{
-				XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
-				XMLXSI: elem.Type,
-				ID:     acs,
-				Type:   elem.Type,
-			},
-			Permission: elem.Permission,
-		})
+		grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
 	}
 
 	return GetBucketAclOutput{
@@ -118,116 +83,64 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 	}, nil
 }
 
-func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
+func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
 	if input == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
-
-	defaultGrantees := []Grantee{
-		{
-			Permission: types.PermissionFullControl,
-			Access:     acl.Owner,
-			Type:       types.TypeCanonicalUser,
-		},
+	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
 	}
 
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
-		switch input.ACL {
-		case types.BucketCannedACLPublicRead:
-			defaultGrantees = append(defaultGrantees, Grantee{
-				Permission: types.PermissionRead,
-				Access:     "all-users",
-				Type:       types.TypeGroup,
-			})
-		case types.BucketCannedACLPublicReadWrite:
-			defaultGrantees = append(defaultGrantees, []Grantee{
-				{
-					Permission: types.PermissionRead,
-					Access:     "all-users",
-					Type:       types.TypeGroup,
-				},
-				{
-					Permission: types.PermissionWrite,
-					Access:     "all-users",
-					Type:       types.TypeGroup,
-				},
-			}...)
-		}
+		acl.ACL = input.ACL
+		acl.Grantees = []Grantee{}
 	} else {
+		grantees := []Grantee{}
 		accs := []string{}
 
-		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
+		if input.GrantRead != nil {
 			fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
 
-			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
+			if *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: types.PermissionFullControl,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
 				}
 			}
-			if input.GrantRead != nil && *input.GrantRead != "" {
+			if *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: types.PermissionRead,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
 				}
 			}
-			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
+			if *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: types.PermissionReadAcp,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
 				}
 			}
-			if input.GrantWrite != nil && *input.GrantWrite != "" {
+			if *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: types.PermissionWrite,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
 				}
 			}
-			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
+			if *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: types.PermissionWriteAcp,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
 				}
 			}
 
 			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
 		} else {
 			cache := make(map[string]bool)
-			for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
-				if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
-					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
-				}
-
-				access := grt.Grantee.ID
-				defaultGrantees = append(defaultGrantees, Grantee{
-					Access:     access,
-					Permission: grt.Permission,
-					Type:       types.TypeCanonicalUser,
-				})
-				if _, ok := cache[access]; !ok {
-					cache[access] = true
-					accs = append(accs, access)
+			for _, grt := range input.AccessControlPolicy.Grants {
+				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
+				if _, ok := cache[*grt.Grantee.ID]; !ok {
+					cache[*grt.Grantee.ID] = true
+					accs = append(accs, *grt.Grantee.ID)
 				}
 			}
 		}
@@ -240,9 +153,10 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 		if len(accList) > 0 {
 			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
 		}
-	}
 
-	acl.Grantees = defaultGrantees
+		acl.Grantees = grantees
+		acl.ACL = ""
+	}
 
 	result, err := json.Marshal(acl)
 	if err != nil {
@@ -286,151 +200,53 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func verifyACL(acl ACL, access string, permission types.Permission) error {
-	grantee := Grantee{
-		Access:     access,
-		Permission: permission,
-		Type:       types.TypeCanonicalUser,
-	}
-	granteeFullCtrl := Grantee{
-		Access:     access,
-		Permission: types.PermissionFullControl,
-		Type:       types.TypeCanonicalUser,
-	}
-	granteeAllUsers := Grantee{
-		Access:     "all-users",
-		Permission: permission,
-		Type:       types.TypeGroup,
-	}
-
-	isFound := false
-
-	for _, grt := range acl.Grantees {
-		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
-			isFound = true
-			break
-		}
-	}
-
-	if isFound {
-		return nil
-	}
-
-	return s3err.GetAPIError(s3err.ErrAccessDenied)
-}
-
-func MayCreateBucket(acct Account, isRoot bool) error {
+func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
 	if isRoot {
 		return nil
 	}
 
-	if acct.Role == RoleUser {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	return nil
-}
-
-func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
-	// Owner check
-	if acct.Access == acl.Owner {
+	if acl.Owner == access {
 		return nil
 	}
 
-	// Root user has access over almost everything
-	if isRoot {
-		return nil
-	}
-
-	// Admin user case
-	if acct.Role == RoleAdmin {
-		return nil
-	}
-
-	// Return access denied in all other cases
-	return s3err.GetAPIError(s3err.ErrAccessDenied)
-}
-
-type AccessOptions struct {
-	Acl           ACL
-	AclPermission types.Permission
-	IsRoot        bool
-	Acc           Account
-	Bucket        string
-	Object        string
-	Action        Action
-	Readonly      bool
-}
-
-func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
-	if opts.Readonly {
-		if opts.AclPermission == types.PermissionWrite || opts.AclPermission == types.PermissionWriteAcp {
+	if acl.ACL != "" {
+		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
 		}
-	}
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
-		return nil
-	}
-
-	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
-	if policyErr != nil {
-		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-			return policyErr
+		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
 		}
+
+		return nil
 	} else {
-		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
+		grantee := Grantee{Access: access, Permission: permission}
+		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
+
+		isFound := false
+
+		for _, grt := range acl.Grantees {
+			if grt == grantee || grt == granteeFullCtrl {
+				isFound = true
+				break
+			}
+		}
+
+		if isFound {
+			return nil
+		}
 	}
 
-	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
-		return err
-	}
-
-	return nil
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
-func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
+func IsAdmin(acct Account, isRoot bool) error {
+	if isRoot {
 		return nil
 	}
 
-	// Verify destination bucket access
-	if err := VerifyAccess(ctx, be, opts); err != nil {
-		return err
-	}
-	// Verify source bucket access
-	srcBucket, srcObject, found := strings.Cut(copySource, "/")
-	if !found {
-		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
+	if acct.Role == "admin" {
+		return nil
 	}
 
-	// Get source bucket ACL
-	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
-	if err != nil {
-		return err
-	}
-
-	var srcBucketAcl ACL
-	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
-		return err
-	}
-
-	if err := VerifyAccess(ctx, be, AccessOptions{
-		Acl:           srcBucketAcl,
-		AclPermission: types.PermissionRead,
-		IsRoot:        opts.IsRoot,
-		Acc:           opts.Acc,
-		Bucket:        srcBucket,
-		Object:        srcObject,
-		Action:        GetObjectAction,
-	}); err != nil {
-		return err
-	}
-
-	return nil
+	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }

auth/bucket_policy.go

@@ -1,152 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"errors"
-	"net/http"
-
-	"github.com/versity/versitygw/s3err"
-)
-
-var (
-	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidResource = errors.New("Policy has invalid resource")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidPrincipal = errors.New("Invalid principal in policy")
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	errInvalidAction = errors.New("Policy has invalid action")
-)
-
-type BucketPolicy struct {
-	Statement []BucketPolicyItem `json:"Statement"`
-}
-
-func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
-	for _, statement := range bp.Statement {
-		err := statement.Validate(bucket, iam)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
-	for _, statement := range bp.Statement {
-		if statement.findMatch(principal, action, resource) {
-			switch statement.Effect {
-			case BucketPolicyAccessTypeAllow:
-				return true
-			case BucketPolicyAccessTypeDeny:
-				return false
-			}
-		}
-	}
-
-	return false
-}
-
-type BucketPolicyItem struct {
-	Effect     BucketPolicyAccessType `json:"Effect"`
-	Principals Principals             `json:"Principal"`
-	Actions    Actions                `json:"Action"`
-	Resources  Resources              `json:"Resource"`
-}
-
-func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
-	if err := bpi.Effect.Validate(); err != nil {
-		return err
-	}
-	if err := bpi.Principals.Validate(iam); err != nil {
-		return err
-	}
-	if err := bpi.Resources.Validate(bucket); err != nil {
-		return err
-	}
-
-	containsObjectAction := bpi.Resources.ContainsObjectPattern()
-	containsBucketAction := bpi.Resources.ContainsBucketPattern()
-
-	for action := range bpi.Actions {
-		isObjectAction := action.IsObjectAction()
-		if isObjectAction == nil {
-			break
-		}
-		if *isObjectAction && !containsObjectAction {
-			return errResourceMismatch
-		}
-		if !*isObjectAction && !containsBucketAction {
-			return errResourceMismatch
-		}
-	}
-
-	return nil
-}
-
-func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
-	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
-		return true
-	}
-
-	return false
-}
-
-func getMalformedPolicyError(err error) error {
-	return s3err.APIError{
-		Code:           "MalformedPolicy",
-		Description:    err.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	}
-}
-
-func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
-	var policy BucketPolicy
-	if err := json.Unmarshal(policyBin, &policy); err != nil {
-		return getMalformedPolicyError(err)
-	}
-
-	if len(policy.Statement) == 0 {
-		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
-	}
-
-	if err := policy.Validate(bucket, iam); err != nil {
-		return getMalformedPolicyError(err)
-	}
-
-	return nil
-}
-
-func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
-	var bucketPolicy BucketPolicy
-	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
-		return err
-	}
-
-	resource := bucket
-	if object != "" {
-		resource += "/" + object
-	}
-
-	if !bucketPolicy.isAllowed(access, action, resource) {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	return nil
-}

auth/bucket_policy_actions.go

@@ -1,250 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"strings"
-)
-
-type Action string
-
-const (
-	GetBucketAclAction                     Action = "s3:GetBucketAcl"
-	CreateBucketAction                     Action = "s3:CreateBucket"
-	PutBucketAclAction                     Action = "s3:PutBucketAcl"
-	DeleteBucketAction                     Action = "s3:DeleteBucket"
-	PutBucketVersioningAction              Action = "s3:PutBucketVersioning"
-	GetBucketVersioningAction              Action = "s3:GetBucketVersioning"
-	PutBucketPolicyAction                  Action = "s3:PutBucketPolicy"
-	GetBucketPolicyAction                  Action = "s3:GetBucketPolicy"
-	DeleteBucketPolicyAction               Action = "s3:DeleteBucketPolicy"
-	AbortMultipartUploadAction             Action = "s3:AbortMultipartUpload"
-	ListMultipartUploadPartsAction         Action = "s3:ListMultipartUploadParts"
-	ListBucketMultipartUploadsAction       Action = "s3:ListBucketMultipartUploads"
-	PutObjectAction                        Action = "s3:PutObject"
-	GetObjectAction                        Action = "s3:GetObject"
-	GetObjectVersionAction                 Action = "s3:GetObjectVersion"
-	DeleteObjectAction                     Action = "s3:DeleteObject"
-	GetObjectAclAction                     Action = "s3:GetObjectAcl"
-	GetObjectAttributesAction              Action = "s3:GetObjectAttributes"
-	PutObjectAclAction                     Action = "s3:PutObjectAcl"
-	RestoreObjectAction                    Action = "s3:RestoreObject"
-	GetBucketTaggingAction                 Action = "s3:GetBucketTagging"
-	PutBucketTaggingAction                 Action = "s3:PutBucketTagging"
-	GetObjectTaggingAction                 Action = "s3:GetObjectTagging"
-	PutObjectTaggingAction                 Action = "s3:PutObjectTagging"
-	DeleteObjectTaggingAction              Action = "s3:DeleteObjectTagging"
-	ListBucketVersionsAction               Action = "s3:ListBucketVersions"
-	ListBucketAction                       Action = "s3:ListBucket"
-	GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
-	PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
-	GetObjectLegalHoldAction               Action = "s3:GetObjectLegalHold"
-	PutObjectLegalHoldAction               Action = "s3:PutObjectLegalHold"
-	GetObjectRetentionAction               Action = "s3:GetObjectRetention"
-	PutObjectRetentionAction               Action = "s3:PutObjectRetention"
-	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
-	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
-	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
-	AllActions                             Action = "s3:*"
-)
-
-var supportedActionList = map[Action]struct{}{
-	GetBucketAclAction:                     {},
-	CreateBucketAction:                     {},
-	PutBucketAclAction:                     {},
-	DeleteBucketAction:                     {},
-	PutBucketVersioningAction:              {},
-	GetBucketVersioningAction:              {},
-	PutBucketPolicyAction:                  {},
-	GetBucketPolicyAction:                  {},
-	DeleteBucketPolicyAction:               {},
-	AbortMultipartUploadAction:             {},
-	ListMultipartUploadPartsAction:         {},
-	ListBucketMultipartUploadsAction:       {},
-	PutObjectAction:                        {},
-	GetObjectAction:                        {},
-	GetObjectVersionAction:                 {},
-	DeleteObjectAction:                     {},
-	GetObjectAclAction:                     {},
-	GetObjectAttributesAction:              {},
-	PutObjectAclAction:                     {},
-	RestoreObjectAction:                    {},
-	GetBucketTaggingAction:                 {},
-	PutBucketTaggingAction:                 {},
-	GetObjectTaggingAction:                 {},
-	PutObjectTaggingAction:                 {},
-	DeleteObjectTaggingAction:              {},
-	ListBucketVersionsAction:               {},
-	ListBucketAction:                       {},
-	PutBucketObjectLockConfigurationAction: {},
-	GetObjectLegalHoldAction:               {},
-	PutObjectLegalHoldAction:               {},
-	GetObjectRetentionAction:               {},
-	PutObjectRetentionAction:               {},
-	BypassGovernanceRetentionAction:        {},
-	PutBucketOwnershipControlsAction:       {},
-	GetBucketOwnershipControlsAction:       {},
-	AllActions:                             {},
-}
-
-var supportedObjectActionList = map[Action]struct{}{
-	AbortMultipartUploadAction:      {},
-	ListMultipartUploadPartsAction:  {},
-	PutObjectAction:                 {},
-	GetObjectAction:                 {},
-	GetObjectVersionAction:          {},
-	DeleteObjectAction:              {},
-	GetObjectAclAction:              {},
-	GetObjectAttributesAction:       {},
-	PutObjectAclAction:              {},
-	RestoreObjectAction:             {},
-	GetObjectTaggingAction:          {},
-	PutObjectTaggingAction:          {},
-	DeleteObjectTaggingAction:       {},
-	GetObjectLegalHoldAction:        {},
-	PutObjectLegalHoldAction:        {},
-	GetObjectRetentionAction:        {},
-	PutObjectRetentionAction:        {},
-	BypassGovernanceRetentionAction: {},
-	AllActions:                      {},
-}
-
-// Validates Action: it should either wildcard match with supported actions list or be in it
-func (a Action) IsValid() error {
-	if !strings.HasPrefix(string(a), "s3:") {
-		return errInvalidAction
-	}
-
-	if a == AllActions {
-		return nil
-	}
-
-	if a[len(a)-1] == '*' {
-		pattern := strings.TrimSuffix(string(a), "*")
-		for act := range supportedActionList {
-			if strings.HasPrefix(string(act), pattern) {
-				return nil
-			}
-		}
-
-		return errInvalidAction
-	}
-
-	_, found := supportedActionList[a]
-	if !found {
-		return errInvalidAction
-	}
-	return nil
-}
-
-func getBoolPtr(bl bool) *bool {
-	return &bl
-}
-
-// Checks if the action is object action
-// nil points to 's3:*'
-func (a Action) IsObjectAction() *bool {
-	if a == AllActions {
-		return nil
-	}
-	if a[len(a)-1] == '*' {
-		pattern := strings.TrimSuffix(string(a), "*")
-		for act := range supportedObjectActionList {
-			if strings.HasPrefix(string(act), pattern) {
-				return getBoolPtr(true)
-			}
-		}
-
-		return getBoolPtr(false)
-	}
-
-	_, found := supportedObjectActionList[a]
-	return &found
-}
-
-func (a Action) WildCardMatch(act Action) bool {
-	if strings.HasSuffix(string(a), "*") {
-		pattern := strings.TrimSuffix(string(a), "*")
-		return strings.HasPrefix(string(act), pattern)
-	}
-	return false
-}
-
-type Actions map[Action]struct{}
-
-// Override UnmarshalJSON method to decode both []string and string properties
-func (a *Actions) UnmarshalJSON(data []byte) error {
-	ss := []string{}
-	var err error
-	if err = json.Unmarshal(data, &ss); err == nil {
-		if len(ss) == 0 {
-			return errInvalidAction
-		}
-		*a = make(Actions)
-		for _, s := range ss {
-			err = a.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	} else {
-		var s string
-		if err = json.Unmarshal(data, &s); err == nil {
-			if s == "" {
-				return errInvalidAction
-			}
-			*a = make(Actions)
-			err = a.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return err
-}
-
-// Validates and adds a new Action to Actions map
-func (a Actions) Add(str string) error {
-	action := Action(str)
-	err := action.IsValid()
-	if err != nil {
-		return err
-	}
-
-	a[action] = struct{}{}
-	return nil
-}
-
-func (a Actions) FindMatch(action Action) bool {
-	_, ok := a[AllActions]
-	if ok {
-		return true
-	}
-	// First O(1) check for non wildcard actions
-	_, found := a[action]
-	if found {
-		return true
-	}
-
-	for act := range a {
-		if strings.HasSuffix(string(act), "*") && act.WildCardMatch(action) {
-			return true
-		}
-	}
-
-	return false
-}

auth/bucket_policy_effect.go

@@ -1,35 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import "fmt"
-
-type BucketPolicyAccessType string
-
-const (
-	BucketPolicyAccessTypeDeny  BucketPolicyAccessType = "Deny"
-	BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
-)
-
-// Checks policy statement Effect to be valid ("Deny", "Allow")
-func (bpat BucketPolicyAccessType) Validate() error {
-	switch bpat {
-	case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
-		return nil
-	}
-
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	return fmt.Errorf("Invalid effect: %v", bpat)
-}

auth/bucket_policy_principals.go

@@ -1,123 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-)
-
-type Principals map[string]struct{}
-
-func (p Principals) Add(key string) {
-	p[key] = struct{}{}
-}
-
-// Override UnmarshalJSON method to decode both []string and string properties
-func (p *Principals) UnmarshalJSON(data []byte) error {
-	ss := []string{}
-	var s string
-	var k struct {
-		AWS string
-	}
-
-	var err error
-
-	if err = json.Unmarshal(data, &ss); err == nil {
-		if len(ss) == 0 {
-			return errInvalidPrincipal
-		}
-		*p = make(Principals)
-		for _, s := range ss {
-			p.Add(s)
-		}
-		return nil
-	} else if err = json.Unmarshal(data, &s); err == nil {
-		if s == "" {
-			return errInvalidPrincipal
-		}
-		*p = make(Principals)
-		p.Add(s)
-
-		return nil
-	} else if err = json.Unmarshal(data, &k); err == nil {
-		if k.AWS == "" {
-			return errInvalidPrincipal
-		}
-		*p = make(Principals)
-		p.Add(k.AWS)
-
-		return nil
-	} else {
-		var sk struct {
-			AWS []string
-		}
-		if err = json.Unmarshal(data, &sk); err == nil {
-			if len(sk.AWS) == 0 {
-				return errInvalidPrincipal
-			}
-			*p = make(Principals)
-			for _, s := range sk.AWS {
-				p.Add(s)
-			}
-		}
-	}
-
-	return err
-}
-
-// Converts Principals map to a slice, by omitting "*"
-func (p Principals) ToSlice() []string {
-	principals := []string{}
-	for p := range p {
-		if p == "*" {
-			continue
-		}
-		principals = append(principals, p)
-	}
-
-	return principals
-}
-
-// Validates Principals by checking user account access keys existence
-func (p Principals) Validate(iam IAMService) error {
-	_, containsWildCard := p["*"]
-	if containsWildCard {
-		if len(p) == 1 {
-			return nil
-		}
-		return errInvalidPrincipal
-	}
-
-	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
-	if err != nil {
-		return err
-	}
-	if len(accs) > 0 {
-		return errInvalidPrincipal
-	}
-
-	return nil
-}
-
-func (p Principals) Contains(userAccess string) bool {
-	// "*" means it matches for any user account
-	_, ok := p["*"]
-	if ok {
-		return true
-	}
-
-	_, found := p[userAccess]
-	return found
-}

auth/bucket_policy_resources.go

@@ -1,136 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"strings"
-)
-
-type Resources map[string]struct{}
-
-const ResourceArnPrefix = "arn:aws:s3:::"
-
-// Override UnmarshalJSON method to decode both []string and string properties
-func (r *Resources) UnmarshalJSON(data []byte) error {
-	ss := []string{}
-	var err error
-	if err = json.Unmarshal(data, &ss); err == nil {
-		if len(ss) == 0 {
-			return errInvalidResource
-		}
-		*r = make(Resources)
-		for _, s := range ss {
-			err = r.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	} else {
-		var s string
-		if err = json.Unmarshal(data, &s); err == nil {
-			if s == "" {
-				return errInvalidResource
-			}
-			*r = make(Resources)
-			err = r.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return err
-}
-
-// Adds and validates a new resource to Resources map
-func (r Resources) Add(rc string) error {
-	ok, pattern := isValidResource(rc)
-	if !ok {
-		return errInvalidResource
-	}
-
-	r[pattern] = struct{}{}
-
-	return nil
-}
-
-// Checks if the resources contain object pattern
-func (r Resources) ContainsObjectPattern() bool {
-	for resource := range r {
-		if resource == "*" || strings.Contains(resource, "/") {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Checks if the resources contain bucket pattern
-func (r Resources) ContainsBucketPattern() bool {
-	for resource := range r {
-		if resource == "*" || !strings.Contains(resource, "/") {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
-func (r Resources) Validate(bucket string) error {
-	for resource := range r {
-		if !strings.HasPrefix(resource, bucket) {
-			return errInvalidResource
-		}
-	}
-
-	return nil
-}
-
-func (r Resources) FindMatch(resource string) bool {
-	for res := range r {
-		if strings.HasSuffix(res, "*") {
-			pattern := strings.TrimSuffix(res, "*")
-			if strings.HasPrefix(resource, pattern) {
-				return true
-			}
-		} else {
-			if res == resource {
-				return true
-			}
-		}
-	}
-
-	return false
-}
-
-// Checks the resource to have arn prefix and not starting with /
-func isValidResource(rc string) (isValid bool, pattern string) {
-	if !strings.HasPrefix(rc, ResourceArnPrefix) {
-		return false, ""
-	}
-
-	res := strings.TrimPrefix(rc, ResourceArnPrefix)
-	if res == "" {
-		return false, ""
-	}
-	// The resource can't start with / (bucket name comes first)
-	if strings.HasPrefix(res, "/") {
-		return false, ""
-	}
-
-	return true, res
-}

auth/iam.go

@@ -16,61 +16,17 @@ package auth
 
 import (
 	"errors"
-	"fmt"
 	"time"
 )
 
-type Role string
-
-const (
-	RoleUser     Role = "user"
-	RoleAdmin    Role = "admin"
-	RoleUserPlus Role = "userplus"
-)
-
-func (r Role) IsValid() bool {
-	switch r {
-	case RoleAdmin:
-		return true
-	case RoleUser:
-		return true
-	case RoleUserPlus:
-		return true
-	default:
-		return false
-	}
-}
-
 // Account is a gateway IAM account
 type Account struct {
-	Access  string `json:"access"`
-	Secret  string `json:"secret"`
-	Role    Role   `json:"role"`
-	UserID  int    `json:"userID"`
-	GroupID int    `json:"groupID"`
-}
-
-type ListUserAccountsResult struct {
-	Accounts []Account
-}
-
-// Mutable props, which could be changed when updating an IAM account
-type MutableProps struct {
-	Secret  *string `json:"secret"`
-	UserID  *int    `json:"userID"`
-	GroupID *int    `json:"groupID"`
-}
-
-func updateAcc(acc *Account, props MutableProps) {
-	if props.Secret != nil {
-		acc.Secret = *props.Secret
-	}
-	if props.GroupID != nil {
-		acc.GroupID = *props.GroupID
-	}
-	if props.UserID != nil {
-		acc.UserID = *props.UserID
-	}
+	Access    string `json:"access"`
+	Secret    string `json:"secret"`
+	Role      string `json:"role"`
+	UserID    int    `json:"userID"`
+	GroupID   int    `json:"groupID"`
+	ProjectID int    `json:"projectID"`
 }
 
 // IAMService is the interface for all IAM service implementations
@@ -79,51 +35,26 @@ func updateAcc(acc *Account, props MutableProps) {
 type IAMService interface {
 	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
-	UpdateUserAccount(access string, props MutableProps) error
 	DeleteUserAccount(access string) error
 	ListUserAccounts() ([]Account, error)
 	Shutdown() error
 }
 
-var (
-	// ErrUserExists is returned when the user already exists
-	ErrUserExists = errors.New("user already exists")
-	// ErrNoSuchUser is returned when the user does not exist
-	ErrNoSuchUser = errors.New("user not found")
-)
+var ErrNoSuchUser = errors.New("user not found")
 
 type Opts struct {
-	RootAccount            Account
-	Dir                    string
-	LDAPServerURL          string
-	LDAPBindDN             string
-	LDAPPassword           string
-	LDAPQueryBase          string
-	LDAPObjClasses         string
-	LDAPAccessAtr          string
-	LDAPSecretAtr          string
-	LDAPRoleAtr            string
-	LDAPUserIdAtr          string
-	LDAPGroupIdAtr         string
-	VaultEndpointURL       string
-	VaultSecretStoragePath string
-	VaultMountPath         string
-	VaultRootToken         string
-	VaultRoleId            string
-	VaultRoleSecret        string
-	VaultServerCert        string
-	VaultClientCert        string
-	VaultClientCertKey     string
-	S3Access               string
-	S3Secret               string
-	S3Region               string
-	S3Bucket               string
-	S3Endpoint             string
-	S3DisableSSlVerfiy     bool
-	S3Debug                bool
-	CacheDisable           bool
-	CacheTTL               int
-	CachePrune             int
+	Dir            string
+	LDAPServerURL  string
+	LDAPBindDN     string
+	LDAPPassword   string
+	LDAPQueryBase  string
+	LDAPObjClasses string
+	LDAPAccessAtr  string
+	LDAPSecretAtr  string
+	LDAPRoleAtr    string
+	CacheDisable   bool
+	CacheTTL       int
+	CachePrune     int
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -132,26 +63,13 @@ func New(o *Opts) (IAMService, error) {
 
 	switch {
 	case o.Dir != "":
-		svc, err = NewInternal(o.RootAccount, o.Dir)
-		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
+		svc, err = NewInternal(o.Dir)
 	case o.LDAPServerURL != "":
-		svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
-			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
-			o.LDAPGroupIdAtr, o.LDAPObjClasses)
-		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
-	case o.S3Endpoint != "":
-		svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
-			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
-		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
-			o.S3Endpoint, o.S3Bucket)
-	case o.VaultEndpointURL != "":
-		svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
-			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
-			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
-		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
+		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
+			o.LDAPObjClasses)
 	default:
 		// if no iam options selected, default to the single user mode
-		fmt.Println("No IAM service configured, enabling single account mode")
 		return IAMServiceSingle{}, nil
 	}
 

auth/iam_cache.go

@@ -66,21 +66,6 @@ func (i *icache) get(k string) (Account, bool) {
 	return v.value, true
 }
 
-func (i *icache) update(k string, props MutableProps) {
-	i.Lock()
-	defer i.Unlock()
-
-	item, found := i.items[k]
-	if found {
-		updateAcc(&item.value, props)
-
-		// refresh the expiration date
-		item.exp = time.Now().Add(i.expire)
-
-		i.items[k] = item
-	}
-}
-
 func (i *icache) Delete(k string) {
 	i.Lock()
 	delete(i.items, k)
@@ -145,7 +130,7 @@ func (c *IAMCache) CreateAccount(account Account) error {
 	acct := Account{
 		Access: strings.Clone(account.Access),
 		Secret: strings.Clone(account.Secret),
-		Role:   Role(strings.Clone(string(account.Role))),
+		Role:   strings.Clone(account.Role),
 	}
 
 	c.iamcache.set(acct.Access, acct)
@@ -181,16 +166,6 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
 	return nil
 }
 
-func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
-	err := c.service.UpdateUserAccount(access, props)
-	if err != nil {
-		return err
-	}
-
-	c.iamcache.update(access, props)
-	return nil
-}
-
 // ListUserAccounts is a passthrough to the underlying service and
 // does not make use of the cache
 func (c *IAMCache) ListUserAccounts() ([]Account, error) {

auth/iam_internal.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"sync"
 	"time"
 )
 
@@ -33,15 +32,7 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	// This mutex will help with racing updates to the IAM data
-	// from multiple requests to this gateway instance, but
-	// will not help with racing updates to multiple load balanced
-	// gateway instances. This is a limitation of the internal
-	// IAM service. All account updates should be sent to a single
-	// gateway instance if possible.
-	sync.RWMutex
-	dir     string
-	rootAcc Account
+	dir string
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -55,10 +46,9 @@ type iAMConfig struct {
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
+func NewInternal(dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		dir:     dir,
-		rootAcc: rootAcc,
+		dir: dir,
 	}
 
 	err := i.initIAM()
@@ -72,13 +62,6 @@ func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
-	if account.Access == s.rootAcc.Access {
-		return ErrUserExists
-	}
-
-	s.Lock()
-	defer s.Unlock()
-
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -87,7 +70,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 
 		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
-			return nil, ErrUserExists
+			return nil, fmt.Errorf("account already exists")
 		}
 		conf.AccessAccounts[account.Access] = account
 
@@ -103,13 +86,6 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
-	if access == s.rootAcc.Access {
-		return s.rootAcc, nil
-	}
-
-	s.RLock()
-	defer s.RUnlock()
-
 	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -123,41 +99,9 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
-// UpdateUserAccount updates the specified user account fields. Returns
-// ErrNoSuchUser if the account does not exist.
-func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
-	s.Lock()
-	defer s.Unlock()
-
-	return s.storeIAM(func(data []byte) ([]byte, error) {
-		conf, err := parseIAM(data)
-		if err != nil {
-			return nil, fmt.Errorf("get iam data: %w", err)
-		}
-
-		acc, found := conf.AccessAccounts[access]
-		if !found {
-			return nil, ErrNoSuchUser
-		}
-
-		updateAcc(&acc, props)
-		conf.AccessAccounts[access] = acc
-
-		b, err := json.Marshal(conf)
-		if err != nil {
-			return nil, fmt.Errorf("failed to serialize iam: %w", err)
-		}
-
-		return b, nil
-	})
-}
-
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
-	s.Lock()
-	defer s.Unlock()
-
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -177,9 +121,6 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 
 // ListUserAccounts lists all the user accounts stored.
 func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
-	s.RLock()
-	defer s.RUnlock()
-
 	conf, err := s.getIAM()
 	if err != nil {
 		return []Account{}, fmt.Errorf("get iam data: %w", err)
@@ -194,11 +135,12 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
 	var accs []Account
 	for _, k := range keys {
 		accs = append(accs, Account{
-			Access:  k,
-			Secret:  conf.AccessAccounts[k].Secret,
-			Role:    conf.AccessAccounts[k].Role,
-			UserID:  conf.AccessAccounts[k].UserID,
-			GroupID: conf.AccessAccounts[k].GroupID,
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
 		})
 	}
 
@@ -247,10 +189,6 @@ func parseIAM(b []byte) (iAMConfig, error) {
 		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
 	}
 
-	if conf.AccessAccounts == nil {
-		conf.AccessAccounts = make(map[string]Account)
-	}
-
 	return conf, nil
 }
 
@@ -332,7 +270,7 @@ func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
 		// reset retries on successful read
 		retries = 0
 
-		err = os.Remove(fname)
+		err = os.Remove(iamFile)
 		if errors.Is(err, fs.ErrNotExist) {
 			// racing with someone else updating
 			// keep retrying after backoff

auth/iam_ldap.go

@@ -1,22 +1,7 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package auth
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/go-ldap/ldap/v3"
@@ -29,19 +14,15 @@ type LdapIAMService struct {
 	accessAtr  string
 	secretAtr  string
 	roleAtr    string
-	groupIdAtr string
-	userIdAtr  string
-	rootAcc    Account
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
-	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
-		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
-	conn, err := ldap.DialURL(url)
+	conn, err := ldap.Dial("tcp", url)
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
 	}
@@ -57,23 +38,15 @@ func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAt
 		accessAtr:  accAtr,
 		secretAtr:  secAtr,
 		roleAtr:    roleAtr,
-		userIdAtr:  userIdAtr,
-		groupIdAtr: groupIdAtr,
-		rootAcc:    rootAcc,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
-	if ld.rootAcc.Access == account.Access {
-		return ErrUserExists
-	}
-	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
-	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
-	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
-	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
+	userEntry.Attribute(ld.roleAtr, []string{account.Role})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -84,9 +57,6 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
 }
 
 func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
-	if access == ld.rootAcc.Access {
-		return ld.rootAcc, nil
-	}
 	searchRequest := ldap.NewSearchRequest(
 		ld.queryBase,
 		ldap.ScopeWholeSubtree,
@@ -95,7 +65,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
 		nil,
 	)
 
@@ -104,48 +74,14 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		return Account{}, err
 	}
 
-	if len(result.Entries) == 0 {
-		return Account{}, ErrNoSuchUser
-	}
-
 	entry := result.Entries[0]
-	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
-	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
-	}
-	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
-	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
-	}
 	return Account{
-		Access:  entry.GetAttributeValue(ld.accessAtr),
-		Secret:  entry.GetAttributeValue(ld.secretAtr),
-		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
-		GroupID: groupId,
-		UserID:  userId,
+		Access: entry.GetAttributeValue(ld.accessAtr),
+		Secret: entry.GetAttributeValue(ld.secretAtr),
+		Role:   entry.GetAttributeValue(ld.roleAtr),
 	}, nil
 }
 
-func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
-	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
-	if props.Secret != nil {
-		req.Replace(ld.secretAtr, []string{*props.Secret})
-	}
-	if props.GroupID != nil {
-		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
-	}
-	if props.UserID != nil {
-		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
-	}
-
-	err := ld.conn.Modify(req)
-	//TODO: Handle non existing user case
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 func (ld *LdapIAMService) DeleteUserAccount(access string) error {
 	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
 
@@ -170,7 +106,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(&%v)", searchFilter),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
 		nil,
 	)
 
@@ -181,20 +117,10 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 
 	result := []Account{}
 	for _, el := range resp.Entries {
-		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
-		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
-		}
-		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
-		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
-		}
 		result = append(result, Account{
-			Access:  el.GetAttributeValue(ld.accessAtr),
-			Secret:  el.GetAttributeValue(ld.secretAtr),
-			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
-			GroupID: groupId,
-			UserID:  userId,
+			Access: el.GetAttributeValue(ld.accessAtr),
+			Secret: el.GetAttributeValue(ld.secretAtr),
+			Role:   el.GetAttributeValue(ld.roleAtr),
 		})
 	}
 

auth/iam_s3_object.go

@@ -1,305 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"sort"
-	"sync"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/aws/smithy-go"
-)
-
-// IAMServiceS3 stores user accounts in an S3 object
-// The endpoint, credentials, bucket, and region are provided
-// from cli configuration.
-// The object format and name is the same as the internal IAM service:
-// coming from iAMConfig and iamFile in iam_internal.
-
-type IAMServiceS3 struct {
-	// This mutex will help with racing updates to the IAM data
-	// from multiple requests to this gateway instance, but
-	// will not help with racing updates to multiple load balanced
-	// gateway instances. This is a limitation of the internal
-	// IAM service. All account updates should be sent to a single
-	// gateway instance if possible.
-	sync.RWMutex
-
-	access        string
-	secret        string
-	region        string
-	bucket        string
-	endpoint      string
-	sslSkipVerify bool
-	debug         bool
-	rootAcc       Account
-	client        *s3.Client
-}
-
-var _ IAMService = &IAMServiceS3{}
-
-func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
-	if access == "" {
-		return nil, fmt.Errorf("must provide s3 IAM service access key")
-	}
-	if secret == "" {
-		return nil, fmt.Errorf("must provide s3 IAM service secret key")
-	}
-	if region == "" {
-		return nil, fmt.Errorf("must provide s3 IAM service region")
-	}
-	if bucket == "" {
-		return nil, fmt.Errorf("must provide s3 IAM service bucket")
-	}
-	if endpoint == "" {
-		return nil, fmt.Errorf("must provide s3 IAM service endpoint")
-	}
-
-	i := &IAMServiceS3{
-		access:        access,
-		secret:        secret,
-		region:        region,
-		bucket:        bucket,
-		endpoint:      endpoint,
-		sslSkipVerify: sslSkipVerify,
-		debug:         debug,
-		rootAcc:       rootAcc,
-	}
-
-	cfg, err := i.getConfig()
-	if err != nil {
-		return nil, fmt.Errorf("init s3 IAM: %v", err)
-	}
-
-	if endpoint != "" {
-		i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
-			o.BaseEndpoint = &endpoint
-		})
-		return i, nil
-	}
-
-	i.client = s3.NewFromConfig(cfg)
-	return i, nil
-}
-
-func (s *IAMServiceS3) CreateAccount(account Account) error {
-	if s.rootAcc.Access == account.Access {
-		return ErrUserExists
-	}
-
-	s.Lock()
-	defer s.Unlock()
-
-	conf, err := s.getAccounts()
-	if err != nil {
-		return err
-	}
-
-	_, ok := conf.AccessAccounts[account.Access]
-	if ok {
-		return ErrUserExists
-	}
-	conf.AccessAccounts[account.Access] = account
-
-	return s.storeAccts(conf)
-}
-
-func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
-	if access == s.rootAcc.Access {
-		return s.rootAcc, nil
-	}
-
-	s.RLock()
-	defer s.RUnlock()
-
-	conf, err := s.getAccounts()
-	if err != nil {
-		return Account{}, err
-	}
-
-	acct, ok := conf.AccessAccounts[access]
-	if !ok {
-		return Account{}, ErrNoSuchUser
-	}
-
-	return acct, nil
-}
-
-func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
-	s.Lock()
-	defer s.Unlock()
-
-	conf, err := s.getAccounts()
-	if err != nil {
-		return err
-	}
-
-	acc, ok := conf.AccessAccounts[access]
-	if !ok {
-		return ErrNoSuchUser
-	}
-
-	updateAcc(&acc, props)
-	conf.AccessAccounts[access] = acc
-
-	return s.storeAccts(conf)
-}
-
-func (s *IAMServiceS3) DeleteUserAccount(access string) error {
-	s.Lock()
-	defer s.Unlock()
-
-	conf, err := s.getAccounts()
-	if err != nil {
-		return err
-	}
-
-	_, ok := conf.AccessAccounts[access]
-	if !ok {
-		return fmt.Errorf("account does not exist")
-	}
-	delete(conf.AccessAccounts, access)
-
-	return s.storeAccts(conf)
-}
-
-func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
-	s.RLock()
-	defer s.RUnlock()
-
-	conf, err := s.getAccounts()
-	if err != nil {
-		return nil, err
-	}
-
-	keys := make([]string, 0, len(conf.AccessAccounts))
-	for k := range conf.AccessAccounts {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	var accs []Account
-	for _, k := range keys {
-		accs = append(accs, Account{
-			Access:  k,
-			Secret:  conf.AccessAccounts[k].Secret,
-			Role:    conf.AccessAccounts[k].Role,
-			UserID:  conf.AccessAccounts[k].UserID,
-			GroupID: conf.AccessAccounts[k].GroupID,
-		})
-	}
-
-	return accs, nil
-}
-
-func (s *IAMServiceS3) Shutdown() error {
-	return nil
-}
-
-func (s *IAMServiceS3) getConfig() (aws.Config, error) {
-	creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
-
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
-	}
-	client := &http.Client{Transport: tr}
-
-	opts := []func(*config.LoadOptions) error{
-		config.WithRegion(s.region),
-		config.WithCredentialsProvider(creds),
-		config.WithHTTPClient(client),
-	}
-
-	if s.debug {
-		opts = append(opts,
-			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
-	}
-
-	return config.LoadDefaultConfig(context.Background(), opts...)
-}
-
-func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
-	obj := iamFile
-
-	out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
-		Bucket: &s.bucket,
-		Key:    &obj,
-	})
-	if err != nil {
-		// if the error is object not exists,
-		// init empty accounts struct and return that
-		var nsk *types.NoSuchKey
-		if errors.As(err, &nsk) {
-			return iAMConfig{AccessAccounts: map[string]Account{}}, nil
-		}
-		var apiErr smithy.APIError
-		if errors.As(err, &apiErr) {
-			if apiErr.ErrorCode() == "NotFound" {
-				return iAMConfig{AccessAccounts: map[string]Account{}}, nil
-			}
-		}
-
-		// all other errors, return the error
-		return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
-	}
-
-	defer out.Body.Close()
-
-	b, err := io.ReadAll(out.Body)
-	if err != nil {
-		return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
-	}
-
-	conf, err := parseIAM(b)
-	if err != nil {
-		return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
-	}
-
-	return conf, nil
-}
-
-func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
-	b, err := json.Marshal(conf)
-	if err != nil {
-		return fmt.Errorf("failed to serialize iam: %w", err)
-	}
-
-	obj := iamFile
-	uploader := manager.NewUploader(s.client)
-	upinfo := &s3.PutObjectInput{
-		Body:   bytes.NewReader(b),
-		Bucket: &s.bucket,
-		Key:    &obj,
-	}
-	_, err = uploader.Upload(context.Background(), upinfo)
-	if err != nil {
-		return fmt.Errorf("store accounts in %v: %w", iamFile, err)
-	}
-
-	return nil
-}

auth/iam_single.go

@@ -32,12 +32,7 @@ func (IAMServiceSingle) CreateAccount(account Account) error {
 
 // GetUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	return Account{}, ErrNoSuchUser
-}
-
-// UpdateUserAccount no accounts in single tenant mode
-func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
-	return ErrNotSupported
+	return Account{}, ErrNotSupported
 }
 
 // DeleteUserAccount no accounts in single tenant mode

auth/iam_vault.go

@@ -1,256 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strings"
-	"time"
-
-	vault "github.com/hashicorp/vault-client-go"
-	"github.com/hashicorp/vault-client-go/schema"
-)
-
-type VaultIAMService struct {
-	client            *vault.Client
-	reqOpts           []vault.RequestOption
-	secretStoragePath string
-	rootAcc           Account
-}
-
-var _ IAMService = &VaultIAMService{}
-
-func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
-	opts := []vault.ClientOption{
-		vault.WithAddress(endpoint),
-		// set request timeout to 10 secs
-		vault.WithRequestTimeout(10 * time.Second),
-	}
-	if serverCert != "" {
-		tls := vault.TLSConfiguration{}
-
-		tls.ServerCertificate.FromBytes = []byte(serverCert)
-		if clientCert != "" {
-			if clientCertKey == "" {
-				return nil, fmt.Errorf("client certificate and client certificate should both be specified")
-			}
-
-			tls.ClientCertificate.FromBytes = []byte(clientCert)
-			tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
-		}
-
-		opts = append(opts, vault.WithTLS(tls))
-	}
-
-	client, err := vault.New(opts...)
-	if err != nil {
-		return nil, fmt.Errorf("init vault client: %w", err)
-	}
-
-	reqOpts := []vault.RequestOption{}
-	// if mount path is not specified, it defaults to "approle"
-	if mountPath != "" {
-		reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
-	}
-
-	// Authentication
-	switch {
-	case rootToken != "":
-		err := client.SetToken(rootToken)
-		if err != nil {
-			return nil, fmt.Errorf("root token authentication failure: %w", err)
-		}
-	case roleID != "":
-		if roleSecret == "" {
-			return nil, fmt.Errorf("role id and role secret must both be specified")
-		}
-
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
-			RoleId:   roleID,
-			SecretId: roleSecret,
-		}, reqOpts...)
-		cancel()
-		if err != nil {
-			return nil, fmt.Errorf("approle authentication failure: %w", err)
-		}
-
-		if err := client.SetToken(resp.Auth.ClientToken); err != nil {
-			return nil, fmt.Errorf("approle authentication set token failure: %w", err)
-		}
-	default:
-		return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
-	}
-
-	return &VaultIAMService{
-		client:            client,
-		reqOpts:           reqOpts,
-		secretStoragePath: secretStoragePath,
-		rootAcc:           rootAcc,
-	}, nil
-}
-
-func (vt *VaultIAMService) CreateAccount(account Account) error {
-	if vt.rootAcc.Access == account.Access {
-		return ErrUserExists
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
-		Data: map[string]any{
-			account.Access: account,
-		},
-		Options: map[string]interface{}{
-			"cas": 0,
-		},
-	}, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		if strings.Contains(err.Error(), "check-and-set") {
-			return ErrUserExists
-		}
-		return err
-	}
-
-	return nil
-}
-
-func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
-	if vt.rootAcc.Access == access {
-		return vt.rootAcc, nil
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		return Account{}, err
-	}
-
-	acc, err := parseVaultUserAccount(resp.Data.Data, access)
-	if err != nil {
-		return Account{}, err
-	}
-
-	return acc, nil
-}
-
-func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
-	//TODO: We need something like a transaction here ?
-	acc, err := vt.GetUserAccount(access)
-	if err != nil {
-		return err
-	}
-
-	updateAcc(&acc, props)
-
-	err = vt.DeleteUserAccount(access)
-	if err != nil {
-		return err
-	}
-
-	err = vt.CreateAccount(acc)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (vt *VaultIAMService) DeleteUserAccount(access string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		if vault.IsErrorStatus(err, 404) {
-			return []Account{}, nil
-		}
-		return nil, err
-	}
-
-	accs := []Account{}
-
-	for _, acss := range resp.Data.Keys {
-		acc, err := vt.GetUserAccount(acss)
-		if err != nil {
-			return nil, err
-		}
-		accs = append(accs, acc)
-	}
-
-	return accs, nil
-}
-
-// the client doesn't have explicit shutdown, as it uses http.Client
-func (vt *VaultIAMService) Shutdown() error {
-	return nil
-}
-
-var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
-
-func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
-	usrAcc, ok := data[access].(map[string]interface{})
-	if !ok {
-		return acc, errInvalidUser
-	}
-
-	acss, ok := usrAcc["access"].(string)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	secret, ok := usrAcc["secret"].(string)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	role, ok := usrAcc["role"].(string)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	userIdJson, ok := usrAcc["userID"].(json.Number)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	userId, err := userIdJson.Int64()
-	if err != nil {
-		return acc, errInvalidUser
-	}
-	groupIdJson, ok := usrAcc["groupID"].(json.Number)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	groupId, err := groupIdJson.Int64()
-	if err != nil {
-		return acc, errInvalidUser
-	}
-
-	return Account{
-		Access:  acss,
-		Secret:  secret,
-		Role:    Role(role),
-		UserID:  int(userId),
-		GroupID: int(groupId),
-	}, nil
-}

auth/object_lock.go

@@ -1,269 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"context"
-	"encoding/json"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3response"
-)
-
-type BucketLockConfig struct {
-	Enabled          bool
-	DefaultRetention *types.DefaultRetention
-	CreatedAt        *time.Time
-}
-
-func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
-	var lockConfig types.ObjectLockConfiguration
-	if err := xml.Unmarshal(input, &lockConfig); err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-	}
-
-	if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
-		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-	}
-
-	config := BucketLockConfig{
-		Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
-	}
-
-	if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
-		retention := lockConfig.Rule.DefaultRetention
-
-		if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
-			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-		}
-		if retention.Years != nil && retention.Days != nil {
-			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-		}
-
-		if retention.Days != nil && *retention.Days <= 0 {
-			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
-		}
-		if retention.Years != nil && *retention.Years <= 0 {
-			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
-		}
-
-		config.DefaultRetention = retention
-		now := time.Now()
-		config.CreatedAt = &now
-	}
-
-	return json.Marshal(config)
-}
-
-func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
-	var config BucketLockConfig
-	if err := json.Unmarshal(input, &config); err != nil {
-		return nil, fmt.Errorf("parse object lock config: %w", err)
-	}
-
-	result := &types.ObjectLockConfiguration{
-		Rule: &types.ObjectLockRule{
-			DefaultRetention: config.DefaultRetention,
-		},
-	}
-
-	if config.Enabled {
-		result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
-	}
-
-	return result, nil
-}
-
-func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
-	var retention s3response.PutObjectRetentionInput
-	if err := xml.Unmarshal(input, &retention); err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
-	}
-
-	if retention.RetainUntilDate.Before(time.Now()) {
-		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
-	}
-	switch retention.Mode {
-	case types.ObjectLockRetentionModeCompliance:
-	case types.ObjectLockRetentionModeGovernance:
-	default:
-		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-	}
-
-	return json.Marshal(retention)
-}
-
-func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
-	var retention types.ObjectLockRetention
-	if err := json.Unmarshal(input, &retention); err != nil {
-		return nil, fmt.Errorf("parse object lock retention: %w", err)
-	}
-
-	return &retention, nil
-}
-
-func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
-	if status == nil {
-		return nil
-	}
-
-	if *status {
-		return &types.ObjectLockLegalHold{
-			Status: types.ObjectLockLegalHoldStatusOn,
-		}
-	}
-
-	return &types.ObjectLockLegalHold{
-		Status: types.ObjectLockLegalHoldStatusOff,
-	}
-}
-
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
-	data, err := be.GetObjectLockConfiguration(ctx, bucket)
-	if err != nil {
-		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
-			return nil
-		}
-
-		return err
-	}
-
-	var bucketLockConfig BucketLockConfig
-	if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse object lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return nil
-	}
-
-	checkDefaultRetention := false
-
-	if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
-		expirationDate := *bucketLockConfig.CreatedAt
-		if bucketLockConfig.DefaultRetention.Days != nil {
-			expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
-		}
-		if bucketLockConfig.DefaultRetention.Years != nil {
-			expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
-		}
-
-		if expirationDate.After(time.Now()) {
-			checkDefaultRetention = true
-		}
-	}
-
-	for _, obj := range objects {
-		var key, versionId string
-		if obj.Key != nil {
-			key = *obj.Key
-		}
-		if obj.VersionId != nil {
-			versionId = *obj.VersionId
-		}
-		checkRetention := true
-		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
-		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
-			continue
-		}
-		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
-			checkRetention = false
-		}
-		if err != nil && checkRetention {
-			return err
-		}
-
-		if checkRetention {
-			retention, err := ParseObjectLockRetentionOutput(retentionData)
-			if err != nil {
-				return err
-			}
-
-			if retention.Mode != "" && retention.RetainUntilDate != nil {
-				if retention.RetainUntilDate.After(time.Now()) {
-					switch retention.Mode {
-					case types.ObjectLockRetentionModeGovernance:
-						if !bypass {
-							return s3err.GetAPIError(s3err.ErrObjectLocked)
-						} else {
-							policy, err := be.GetBucketPolicy(ctx, bucket)
-							if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-								return s3err.GetAPIError(s3err.ErrObjectLocked)
-							}
-							if err != nil {
-								return err
-							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
-							if err != nil {
-								return s3err.GetAPIError(s3err.ErrObjectLocked)
-							}
-						}
-					case types.ObjectLockRetentionModeCompliance:
-						return s3err.GetAPIError(s3err.ErrObjectLocked)
-					}
-				}
-			}
-		}
-
-		checkLegalHold := true
-
-		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
-		if err != nil {
-			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
-				continue
-			}
-			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
-				checkLegalHold = false
-			} else {
-				return err
-			}
-		}
-
-		if checkLegalHold && *status {
-			return s3err.GetAPIError(s3err.ErrObjectLocked)
-		}
-
-		if checkDefaultRetention {
-			switch bucketLockConfig.DefaultRetention.Mode {
-			case types.ObjectLockRetentionModeGovernance:
-				if !bypass {
-					return s3err.GetAPIError(s3err.ErrObjectLocked)
-				} else {
-					policy, err := be.GetBucketPolicy(ctx, bucket)
-					if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-						return s3err.GetAPIError(s3err.ErrObjectLocked)
-					}
-					if err != nil {
-						return err
-					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
-					if err != nil {
-						return s3err.GetAPIError(s3err.ErrObjectLocked)
-					}
-				}
-			case types.ObjectLockRetentionModeCompliance:
-				return s3err.GetAPIError(s3err.ErrObjectLocked)
-			}
-		}
-	}
-
-	return nil
-}

aws/LICENSE.txt

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

aws/NOTICE.txt

@@ -1,4 +0,0 @@
-AWS SDK for Go
-Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-Copyright 2014-2015 Stripe, Inc.
-Copyright 2024 Versity Software

aws/README.md

@@ -1,11 +0,0 @@
-# AWS SDK Go v2
-
-This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License.
-
-## Description
-
-The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements.
-
-## License
-
-The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information.

aws/internal/awstesting/unit/unit.go

@@ -1,61 +0,0 @@
-// Package unit performs initialization and validation for unit tests
-package unit
-
-import (
-	"context"
-	"crypto/rsa"
-	"math/big"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-)
-
-func init() {
-	config = aws.Config{}
-	config.Region = "mock-region"
-	config.Credentials = StubCredentialsProvider{}
-}
-
-// StubCredentialsProvider provides a stub credential provider that returns
-// static credentials that never expire.
-type StubCredentialsProvider struct{}
-
-// Retrieve satisfies the CredentialsProvider interface. Returns stub
-// credential value, and never error.
-func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) {
-	return aws.Credentials{
-		AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION",
-		Source: "unit test credentials",
-	}, nil
-}
-
-var config aws.Config
-
-// Config returns a copy of the mock configuration for unit tests.
-func Config() aws.Config { return config.Copy() }
-
-// RSAPrivateKey is used for testing functionality that requires some
-// sort of private key. Taken from crypto/rsa/rsa_test.go
-//
-// Credit to golang 1.11
-var RSAPrivateKey = &rsa.PrivateKey{
-	PublicKey: rsa.PublicKey{
-		N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"),
-		E: 3,
-	},
-	D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"),
-	Primes: []*big.Int{
-		fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"),
-		fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"),
-	},
-}
-
-// Taken from crypto/rsa/rsa_test.go
-//
-// Credit to golang 1.11
-func fromBase10(base10 string) *big.Int {
-	i, ok := new(big.Int).SetString(base10, 10)
-	if !ok {
-		panic("bad number: " + base10)
-	}
-	return i
-}

aws/signer/internal/v4/cache.go

@@ -1,115 +0,0 @@
-package v4
-
-import (
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-)
-
-func lookupKey(service, region string) string {
-	var s strings.Builder
-	s.Grow(len(region) + len(service) + 3)
-	s.WriteString(region)
-	s.WriteRune('/')
-	s.WriteString(service)
-	return s.String()
-}
-
-type derivedKey struct {
-	AccessKey  string
-	Date       time.Time
-	Credential []byte
-}
-
-type derivedKeyCache struct {
-	values map[string]derivedKey
-	mutex  sync.RWMutex
-}
-
-func newDerivedKeyCache() derivedKeyCache {
-	return derivedKeyCache{
-		values: make(map[string]derivedKey),
-	}
-}
-
-func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte {
-	key := lookupKey(service, region)
-	s.mutex.RLock()
-	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
-		s.mutex.RUnlock()
-		return cred
-	}
-	s.mutex.RUnlock()
-
-	s.mutex.Lock()
-	if cred, ok := s.get(key, credentials, signingTime.Time); ok {
-		s.mutex.Unlock()
-		return cred
-	}
-	cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime)
-	entry := derivedKey{
-		AccessKey:  credentials.AccessKeyID,
-		Date:       signingTime.Time,
-		Credential: cred,
-	}
-	s.values[key] = entry
-	s.mutex.Unlock()
-
-	return cred
-}
-
-func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) {
-	cacheEntry, ok := s.retrieveFromCache(key)
-	if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) {
-		return cacheEntry.Credential, true
-	}
-	return nil, false
-}
-
-func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) {
-	if v, ok := s.values[key]; ok {
-		return v, true
-	}
-	return derivedKey{}, false
-}
-
-// SigningKeyDeriver derives a signing key from a set of credentials
-type SigningKeyDeriver struct {
-	cache derivedKeyCache
-}
-
-// NewSigningKeyDeriver returns a new SigningKeyDeriver
-func NewSigningKeyDeriver() *SigningKeyDeriver {
-	return &SigningKeyDeriver{
-		cache: newDerivedKeyCache(),
-	}
-}
-
-// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing.
-func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte {
-	return k.cache.Get(credential, service, region, signingTime)
-}
-
-func deriveKey(secret, service, region string, t SigningTime) []byte {
-	hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat()))
-	hmacRegion := HMACSHA256(hmacDate, []byte(region))
-	hmacService := HMACSHA256(hmacRegion, []byte(service))
-	return HMACSHA256(hmacService, []byte("aws4_request"))
-}
-
-func isSameDay(x, y time.Time) bool {
-	xYear, xMonth, xDay := x.Date()
-	yYear, yMonth, yDay := y.Date()
-
-	if xYear != yYear {
-		return false
-	}
-
-	if xMonth != yMonth {
-		return false
-	}
-
-	return xDay == yDay
-}

aws/signer/internal/v4/const.go

@@ -1,40 +0,0 @@
-package v4
-
-// Signature Version 4 (SigV4) Constants
-const (
-	// EmptyStringSHA256 is the hex encoded sha256 value of an empty string
-	EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
-
-	// UnsignedPayload indicates that the request payload body is unsigned
-	UnsignedPayload = "UNSIGNED-PAYLOAD"
-
-	// AmzAlgorithmKey indicates the signing algorithm
-	AmzAlgorithmKey = "X-Amz-Algorithm"
-
-	// AmzSecurityTokenKey indicates the security token to be used with temporary credentials
-	AmzSecurityTokenKey = "X-Amz-Security-Token"
-
-	// AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z'
-	AmzDateKey = "X-Amz-Date"
-
-	// AmzCredentialKey is the access key ID and credential scope
-	AmzCredentialKey = "X-Amz-Credential"
-
-	// AmzSignedHeadersKey is the set of headers signed for the request
-	AmzSignedHeadersKey = "X-Amz-SignedHeaders"
-
-	// AmzSignatureKey is the query parameter to store the SigV4 signature
-	AmzSignatureKey = "X-Amz-Signature"
-
-	// TimeFormat is the time format to be used in the X-Amz-Date header or query parameter
-	TimeFormat = "20060102T150405Z"
-
-	// ShortTimeFormat is the shorten time format used in the credential scope
-	ShortTimeFormat = "20060102"
-
-	// ContentSHAKey is the SHA256 of request body
-	ContentSHAKey = "X-Amz-Content-Sha256"
-
-	// StreamingEventsPayload indicates that the request payload body is a signed event stream.
-	StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS"
-)

aws/signer/internal/v4/header_rules.go

@@ -1,88 +0,0 @@
-package v4
-
-import (
-	"strings"
-)
-
-// Rules houses a set of Rule needed for validation of a
-// string value
-type Rules []Rule
-
-// Rule interface allows for more flexible rules and just simply
-// checks whether or not a value adheres to that Rule
-type Rule interface {
-	IsValid(value string) bool
-}
-
-// IsValid will iterate through all rules and see if any rules
-// apply to the value and supports nested rules
-func (r Rules) IsValid(value string) bool {
-	for _, rule := range r {
-		if rule.IsValid(value) {
-			return true
-		}
-	}
-	return false
-}
-
-// MapRule generic Rule for maps
-type MapRule map[string]struct{}
-
-// IsValid for the map Rule satisfies whether it exists in the map
-func (m MapRule) IsValid(value string) bool {
-	_, ok := m[value]
-	return ok
-}
-
-// AllowList is a generic Rule for include listing
-type AllowList struct {
-	Rule
-}
-
-// IsValid for AllowList checks if the value is within the AllowList
-func (w AllowList) IsValid(value string) bool {
-	return w.Rule.IsValid(value)
-}
-
-// ExcludeList is a generic Rule for exclude listing
-type ExcludeList struct {
-	Rule
-}
-
-// IsValid for AllowList checks if the value is within the AllowList
-func (b ExcludeList) IsValid(value string) bool {
-	return !b.Rule.IsValid(value)
-}
-
-// Patterns is a list of strings to match against
-type Patterns []string
-
-// IsValid for Patterns checks each pattern and returns if a match has
-// been found
-func (p Patterns) IsValid(value string) bool {
-	for _, pattern := range p {
-		if hasPrefixFold(value, pattern) {
-			return true
-		}
-	}
-	return false
-}
-
-// InclusiveRules rules allow for rules to depend on one another
-type InclusiveRules []Rule
-
-// IsValid will return true if all rules are true
-func (r InclusiveRules) IsValid(value string) bool {
-	for _, rule := range r {
-		if !rule.IsValid(value) {
-			return false
-		}
-	}
-	return true
-}
-
-// hasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
-// under Unicode case-folding.
-func hasPrefixFold(s, prefix string) bool {
-	return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
-}

aws/signer/internal/v4/headers.go

@@ -1,72 +0,0 @@
-package v4
-
-// IgnoredHeaders is a list of headers that are ignored during signing
-var IgnoredHeaders = Rules{
-	ExcludeList{
-		MapRule{
-			"Authorization": struct{}{},
-			// some clients use user-agent in signed headers
-			// "User-Agent":      struct{}{},
-			"X-Amzn-Trace-Id": struct{}{},
-			"Expect":          struct{}{},
-		},
-	},
-}
-
-// RequiredSignedHeaders is a allow list for Build canonical headers.
-var RequiredSignedHeaders = Rules{
-	AllowList{
-		MapRule{
-			"Cache-Control":                         struct{}{},
-			"Content-Disposition":                   struct{}{},
-			"Content-Encoding":                      struct{}{},
-			"Content-Language":                      struct{}{},
-			"Content-Md5":                           struct{}{},
-			"Content-Type":                          struct{}{},
-			"Expires":                               struct{}{},
-			"If-Match":                              struct{}{},
-			"If-Modified-Since":                     struct{}{},
-			"If-None-Match":                         struct{}{},
-			"If-Unmodified-Since":                   struct{}{},
-			"Range":                                 struct{}{},
-			"X-Amz-Acl":                             struct{}{},
-			"X-Amz-Copy-Source":                     struct{}{},
-			"X-Amz-Copy-Source-If-Match":            struct{}{},
-			"X-Amz-Copy-Source-If-Modified-Since":   struct{}{},
-			"X-Amz-Copy-Source-If-None-Match":       struct{}{},
-			"X-Amz-Copy-Source-If-Unmodified-Since": struct{}{},
-			"X-Amz-Copy-Source-Range":               struct{}{},
-			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
-			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       struct{}{},
-			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   struct{}{},
-			"X-Amz-Expected-Bucket-Owner":                                 struct{}{},
-			"X-Amz-Grant-Full-control":                                    struct{}{},
-			"X-Amz-Grant-Read":                                            struct{}{},
-			"X-Amz-Grant-Read-Acp":                                        struct{}{},
-			"X-Amz-Grant-Write":                                           struct{}{},
-			"X-Amz-Grant-Write-Acp":                                       struct{}{},
-			"X-Amz-Metadata-Directive":                                    struct{}{},
-			"X-Amz-Mfa":                                                   struct{}{},
-			"X-Amz-Request-Payer":                                         struct{}{},
-			"X-Amz-Server-Side-Encryption":                                struct{}{},
-			"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id":                 struct{}{},
-			"X-Amz-Server-Side-Encryption-Context":                        struct{}{},
-			"X-Amz-Server-Side-Encryption-Customer-Algorithm":             struct{}{},
-			"X-Amz-Server-Side-Encryption-Customer-Key":                   struct{}{},
-			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":               struct{}{},
-			"X-Amz-Storage-Class":                                         struct{}{},
-			"X-Amz-Website-Redirect-Location":                             struct{}{},
-			"X-Amz-Content-Sha256":                                        struct{}{},
-			"X-Amz-Tagging":                                               struct{}{},
-		},
-	},
-	Patterns{"X-Amz-Object-Lock-"},
-	Patterns{"X-Amz-Meta-"},
-}
-
-// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value
-// represents whether or not it is a pattern.
-var AllowedQueryHoisting = InclusiveRules{
-	ExcludeList{RequiredSignedHeaders},
-	Patterns{"X-Amz-"},
-}

aws/signer/internal/v4/headers_test.go

@@ -1,63 +0,0 @@
-package v4
-
-import "testing"
-
-func TestAllowedQueryHoisting(t *testing.T) {
-	cases := map[string]struct {
-		Header      string
-		ExpectHoist bool
-	}{
-		"object-lock": {
-			Header:      "X-Amz-Object-Lock-Mode",
-			ExpectHoist: false,
-		},
-		"s3 metadata": {
-			Header:      "X-Amz-Meta-SomeName",
-			ExpectHoist: false,
-		},
-		"another header": {
-			Header:      "X-Amz-SomeOtherHeader",
-			ExpectHoist: true,
-		},
-		"non X-AMZ header": {
-			Header:      "X-SomeOtherHeader",
-			ExpectHoist: false,
-		},
-	}
-
-	for name, c := range cases {
-		t.Run(name, func(t *testing.T) {
-			if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a {
-				t.Errorf("expect hoist %v, was %v", e, a)
-			}
-		})
-	}
-}
-
-func TestIgnoredHeaders(t *testing.T) {
-	cases := map[string]struct {
-		Header        string
-		ExpectIgnored bool
-	}{
-		"expect": {
-			Header:        "Expect",
-			ExpectIgnored: true,
-		},
-		"authorization": {
-			Header:        "Authorization",
-			ExpectIgnored: true,
-		},
-		"X-AMZ header": {
-			Header:        "X-Amz-Content-Sha256",
-			ExpectIgnored: false,
-		},
-	}
-
-	for name, c := range cases {
-		t.Run(name, func(t *testing.T) {
-			if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a {
-				t.Errorf("expect ignored %v, was %v", e, a)
-			}
-		})
-	}
-}

aws/signer/internal/v4/hmac.go

@@ -1,13 +0,0 @@
-package v4
-
-import (
-	"crypto/hmac"
-	"crypto/sha256"
-)
-
-// HMACSHA256 computes a HMAC-SHA256 of data given the provided key.
-func HMACSHA256(key []byte, data []byte) []byte {
-	hash := hmac.New(sha256.New, key)
-	hash.Write(data)
-	return hash.Sum(nil)
-}

aws/signer/internal/v4/host.go

@@ -1,75 +0,0 @@
-package v4
-
-import (
-	"net/http"
-	"strings"
-)
-
-// SanitizeHostForHeader removes default port from host and updates request.Host
-func SanitizeHostForHeader(r *http.Request) {
-	host := getHost(r)
-	port := portOnly(host)
-	if port != "" && isDefaultPort(r.URL.Scheme, port) {
-		r.Host = stripPort(host)
-	}
-}
-
-// Returns host from request
-func getHost(r *http.Request) string {
-	if r.Host != "" {
-		return r.Host
-	}
-
-	return r.URL.Host
-}
-
-// Hostname returns u.Host, without any port number.
-//
-// If Host is an IPv6 literal with a port number, Hostname returns the
-// IPv6 literal without the square brackets. IPv6 literals may include
-// a zone identifier.
-//
-// Copied from the Go 1.8 standard library (net/url)
-func stripPort(hostport string) string {
-	colon := strings.IndexByte(hostport, ':')
-	if colon == -1 {
-		return hostport
-	}
-	if i := strings.IndexByte(hostport, ']'); i != -1 {
-		return strings.TrimPrefix(hostport[:i], "[")
-	}
-	return hostport[:colon]
-}
-
-// Port returns the port part of u.Host, without the leading colon.
-// If u.Host doesn't contain a port, Port returns an empty string.
-//
-// Copied from the Go 1.8 standard library (net/url)
-func portOnly(hostport string) string {
-	colon := strings.IndexByte(hostport, ':')
-	if colon == -1 {
-		return ""
-	}
-	if i := strings.Index(hostport, "]:"); i != -1 {
-		return hostport[i+len("]:"):]
-	}
-	if strings.Contains(hostport, "]") {
-		return ""
-	}
-	return hostport[colon+len(":"):]
-}
-
-// Returns true if the specified URI is using the standard port
-// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs)
-func isDefaultPort(scheme, port string) bool {
-	if port == "" {
-		return true
-	}
-
-	lowerCaseScheme := strings.ToLower(scheme)
-	if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") {
-		return true
-	}
-
-	return false
-}

aws/signer/internal/v4/scope.go

@@ -1,13 +0,0 @@
-package v4
-
-import "strings"
-
-// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope
-func BuildCredentialScope(signingTime SigningTime, region, service string) string {
-	return strings.Join([]string{
-		signingTime.ShortTimeFormat(),
-		region,
-		service,
-		"aws4_request",
-	}, "/")
-}

aws/signer/internal/v4/time.go

@@ -1,36 +0,0 @@
-package v4
-
-import "time"
-
-// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing.
-type SigningTime struct {
-	time.Time
-	timeFormat      string
-	shortTimeFormat string
-}
-
-// NewSigningTime creates a new SigningTime given a time.Time
-func NewSigningTime(t time.Time) SigningTime {
-	return SigningTime{
-		Time: t,
-	}
-}
-
-// TimeFormat provides a time formatted in the X-Amz-Date format.
-func (m *SigningTime) TimeFormat() string {
-	return m.format(&m.timeFormat, TimeFormat)
-}
-
-// ShortTimeFormat provides a time formatted of 20060102.
-func (m *SigningTime) ShortTimeFormat() string {
-	return m.format(&m.shortTimeFormat, ShortTimeFormat)
-}
-
-func (m *SigningTime) format(target *string, format string) string {
-	if len(*target) > 0 {
-		return *target
-	}
-	v := m.Time.Format(format)
-	*target = v
-	return v
-}

aws/signer/internal/v4/util.go

@@ -1,80 +0,0 @@
-package v4
-
-import (
-	"net/url"
-	"strings"
-)
-
-const doubleSpace = "  "
-
-// StripExcessSpaces will rewrite the passed in slice's string values to not
-// contain multiple side-by-side spaces.
-func StripExcessSpaces(str string) string {
-	var j, k, l, m, spaces int
-	// Trim trailing spaces
-	for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- {
-	}
-
-	// Trim leading spaces
-	for k = 0; k < j && str[k] == ' '; k++ {
-	}
-	str = str[k : j+1]
-
-	// Strip multiple spaces.
-	j = strings.Index(str, doubleSpace)
-	if j < 0 {
-		return str
-	}
-
-	buf := []byte(str)
-	for k, m, l = j, j, len(buf); k < l; k++ {
-		if buf[k] == ' ' {
-			if spaces == 0 {
-				// First space.
-				buf[m] = buf[k]
-				m++
-			}
-			spaces++
-		} else {
-			// End of multiple spaces.
-			spaces = 0
-			buf[m] = buf[k]
-			m++
-		}
-	}
-
-	return string(buf[:m])
-}
-
-// GetURIPath returns the escaped URI component from the provided URL.
-func GetURIPath(u *url.URL) string {
-	var uriPath string
-
-	if len(u.Opaque) > 0 {
-		const schemeSep, pathSep, queryStart = "//", "/", "?"
-
-		opaque := u.Opaque
-		// Cut off the query string if present.
-		if idx := strings.Index(opaque, queryStart); idx >= 0 {
-			opaque = opaque[:idx]
-		}
-
-		// Cutout the scheme separator if present.
-		if strings.Index(opaque, schemeSep) == 0 {
-			opaque = opaque[len(schemeSep):]
-		}
-
-		// capture URI path starting with first path separator.
-		if idx := strings.Index(opaque, pathSep); idx >= 0 {
-			uriPath = opaque[idx:]
-		}
-	} else {
-		uriPath = u.EscapedPath()
-	}
-
-	if len(uriPath) == 0 {
-		uriPath = "/"
-	}
-
-	return uriPath
-}

aws/signer/internal/v4/util_test.go

@@ -1,158 +0,0 @@
-package v4
-
-import (
-	"net/http"
-	"net/url"
-	"testing"
-)
-
-func lazyURLParse(v string) func() (*url.URL, error) {
-	return func() (*url.URL, error) {
-		return url.Parse(v)
-	}
-}
-
-func TestGetURIPath(t *testing.T) {
-	cases := map[string]struct {
-		getURL func() (*url.URL, error)
-		expect string
-	}{
-		// Cases
-		"with scheme": {
-			getURL: lazyURLParse("https://localhost:9000"),
-			expect: "/",
-		},
-		"no port, with scheme": {
-			getURL: lazyURLParse("https://localhost"),
-			expect: "/",
-		},
-		"without scheme": {
-			getURL: lazyURLParse("localhost:9000"),
-			expect: "/",
-		},
-		"without scheme, with path": {
-			getURL: lazyURLParse("localhost:9000/abc123"),
-			expect: "/abc123",
-		},
-		"without scheme, with separator": {
-			getURL: lazyURLParse("//localhost:9000"),
-			expect: "/",
-		},
-		"no port, without scheme, with separator": {
-			getURL: lazyURLParse("//localhost"),
-			expect: "/",
-		},
-		"without scheme, with separator, with path": {
-			getURL: lazyURLParse("//localhost:9000/abc123"),
-			expect: "/abc123",
-		},
-		"no port, without scheme, with separator, with path": {
-			getURL: lazyURLParse("//localhost/abc123"),
-			expect: "/abc123",
-		},
-		"opaque with query string": {
-			getURL: lazyURLParse("localhost:9000/abc123?efg=456"),
-			expect: "/abc123",
-		},
-		"failing test": {
-			getURL: func() (*url.URL, error) {
-				endpoint := "https://service.region.amazonaws.com"
-				req, _ := http.NewRequest("POST", endpoint, nil)
-				u := req.URL
-
-				u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
-
-				query := u.Query()
-				query.Set("some-query-key", "value")
-				u.RawQuery = query.Encode()
-
-				return u, nil
-			},
-			expect: "/bucket/key-._~,!@#$%^&*()",
-		},
-	}
-
-	for name, c := range cases {
-		t.Run(name, func(t *testing.T) {
-			u, err := c.getURL()
-			if err != nil {
-				t.Fatalf("failed to get URL, %v", err)
-			}
-
-			actual := GetURIPath(u)
-			if e, a := c.expect, actual; e != a {
-				t.Errorf("expect %v path, got %v", e, a)
-			}
-		})
-	}
-}
-
-func TestStripExcessHeaders(t *testing.T) {
-	vals := []string{
-		"",
-		"123",
-		"1 2 3",
-		"1 2 3 ",
-		"  1 2 3",
-		"1  2 3",
-		"1  23",
-		"1  2  3",
-		"1  2  ",
-		" 1  2  ",
-		"12   3",
-		"12   3   1",
-		"12           3     1",
-		"12     3       1abc123",
-	}
-
-	expected := []string{
-		"",
-		"123",
-		"1 2 3",
-		"1 2 3",
-		"1 2 3",
-		"1 2 3",
-		"1 23",
-		"1 2 3",
-		"1 2",
-		"1 2",
-		"12 3",
-		"12 3 1",
-		"12 3 1",
-		"12 3 1abc123",
-	}
-
-	for i := 0; i < len(vals); i++ {
-		r := StripExcessSpaces(vals[i])
-		if e, a := expected[i], r; e != a {
-			t.Errorf("%d, expect %v, got %v", i, e, a)
-		}
-	}
-}
-
-var stripExcessSpaceCases = []string{
-	`AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`,
-	`123   321   123   321`,
-	`   123   321   123   321   `,
-	`   123    321    123          321   `,
-	"123",
-	"1 2 3",
-	"  1 2 3",
-	"1  2 3",
-	"1  23",
-	"1  2  3",
-	"1  2  ",
-	" 1  2  ",
-	"12   3",
-	"12   3   1",
-	"12           3     1",
-	"12     3       1abc123",
-}
-
-func BenchmarkStripExcessSpaces(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		for _, v := range stripExcessSpaceCases {
-			StripExcessSpaces(v)
-		}
-	}
-}

aws/signer/v4/functional_test.go

@@ -1,139 +0,0 @@
-package v4_test
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"testing"
-	"time"
-
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/versity/versitygw/aws/internal/awstesting/unit"
-	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
-)
-
-var standaloneSignCases = []struct {
-	OrigURI                    string
-	OrigQuery                  string
-	Region, Service, SubDomain string
-	ExpSig                     string
-	EscapedURI                 string
-}{
-	{
-		OrigURI:   `/logs-*/_search`,
-		OrigQuery: `pretty=true`,
-		Region:    "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
-		EscapedURI: `/logs-%2A/_search`,
-		ExpSig:     `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
-	},
-}
-
-func TestStandaloneSign_CustomURIEscape(t *testing.T) {
-	var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967`
-
-	creds, err := unit.Config().Credentials.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-	signer := v4.NewSigner(func(signer *v4.SignerOptions) {
-		signer.DisableURIPathEscaping = true
-	})
-
-	host := "https://subdomain.us-east-1.es.amazonaws.com"
-	req, err := http.NewRequest("GET", host, nil)
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-
-	req.URL.Path = `/log-*/_search`
-	req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search"
-
-	err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0))
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-
-	actual := req.Header.Get("Authorization")
-	if e, a := expectSig, actual; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-}
-
-func TestStandaloneSign(t *testing.T) {
-	creds, err := unit.Config().Credentials.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-	signer := v4.NewSigner()
-
-	for _, c := range standaloneSignCases {
-		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
-			c.SubDomain, c.Region, c.Service)
-
-		req, err := http.NewRequest("GET", host, nil)
-		if err != nil {
-			t.Errorf("expected no error, but received %v", err)
-		}
-
-		// URL.EscapedPath() will be used by the signer to get the
-		// escaped form of the request's URI path.
-		req.URL.Path = c.OrigURI
-		req.URL.RawQuery = c.OrigQuery
-
-		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
-		if err != nil {
-			t.Errorf("expected no error, but received %v", err)
-		}
-
-		actual := req.Header.Get("Authorization")
-		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
-		}
-		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
-		}
-		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but received %v", e, a)
-		}
-	}
-}
-
-func TestStandaloneSign_RawPath(t *testing.T) {
-	creds, err := unit.Config().Credentials.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-	signer := v4.NewSigner()
-
-	for _, c := range standaloneSignCases {
-		host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
-			c.SubDomain, c.Region, c.Service)
-
-		req, err := http.NewRequest("GET", host, nil)
-		if err != nil {
-			t.Errorf("expected no error, but received %v", err)
-		}
-
-		// URL.EscapedPath() will be used by the signer to get the
-		// escaped form of the request's URI path.
-		req.URL.Path = c.OrigURI
-		req.URL.RawPath = c.EscapedURI
-		req.URL.RawQuery = c.OrigQuery
-
-		err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
-		if err != nil {
-			t.Errorf("expected no error, but received %v", err)
-		}
-
-		actual := req.Header.Get("Authorization")
-		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
-		}
-		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
-		}
-		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but received %v", e, a)
-		}
-	}
-}

aws/signer/v4/v4.go

@@ -1,565 +0,0 @@
-// Package v4 implements signing for AWS V4 signer
-//
-// Provides request signing for request that need to be signed with
-// AWS V4 Signatures.
-//
-// # Standalone Signer
-//
-// Generally using the signer outside of the SDK should not require any additional
-//
-//	The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires
-//
-// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent
-// to the service as.
-//
-// The signer will first check the URL.Opaque field, and use its value if set.
-// The signer does require the URL.Opaque field to be set in the form of:
-//
-//	"//<hostname>/<path>"
-//
-//	// e.g.
-//	"//example.com/some/path"
-//
-// The leading "//" and hostname are required or the URL.Opaque escaping will
-// not work correctly.
-//
-// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath()
-// method and using the returned value.
-//
-// AWS v4 signature validation requires that the canonical string's URI path
-// element must be the URI escaped form of the HTTP request's path.
-// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-//
-// The Go HTTP client will perform escaping automatically on the request. Some
-// of these escaping may cause signature validation errors because the HTTP
-// request differs from the URI path or query that the signature was generated.
-// https://golang.org/pkg/net/url/#URL.EscapedPath
-//
-// Because of this, it is recommended that when using the signer outside of the
-// SDK that explicitly escaping the request prior to being signed is preferable,
-// and will help prevent signature validation errors. This can be done by setting
-// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then
-// call URL.EscapedPath() if Opaque is not set.
-//
-// Test `TestStandaloneSign` provides a complete example of using the signer
-// outside of the SDK and pre-escaping the URI path.
-package v4
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"hash"
-	"net/http"
-	"net/textproto"
-	"net/url"
-	"slices"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/smithy-go/encoding/httpbinding"
-	"github.com/aws/smithy-go/logging"
-	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
-)
-
-const (
-	signingAlgorithm    = "AWS4-HMAC-SHA256"
-	authorizationHeader = "Authorization"
-
-	// Version of signing v4
-	Version = "SigV4"
-)
-
-// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests
-type HTTPSigner interface {
-	SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
-}
-
-type keyDerivator interface {
-	DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte
-}
-
-// SignerOptions is the SigV4 Signer options.
-type SignerOptions struct {
-	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
-	// request header to the request's query string. This is most commonly used
-	// with pre-signed requests preventing headers from being added to the
-	// request's query string.
-	DisableHeaderHoisting bool
-
-	// Disables the automatic escaping of the URI path of the request for the
-	// siganture's canonical string's path. For services that do not need additional
-	// escaping then use this to disable the signer escaping the path.
-	//
-	// S3 is an example of a service that does not need additional escaping.
-	//
-	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-	DisableURIPathEscaping bool
-
-	// The logger to send log messages to.
-	Logger logging.Logger
-
-	// Enable logging of signed requests.
-	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
-	// presigned URL.
-	LogSigning bool
-
-	// Disables setting the session token on the request as part of signing
-	// through X-Amz-Security-Token. This is needed for variations of v4 that
-	// present the token elsewhere.
-	DisableSessionToken bool
-}
-
-// Signer applies AWS v4 signing to given request. Use this to sign requests
-// that need to be signed with AWS V4 Signatures.
-type Signer struct {
-	options      SignerOptions
-	keyDerivator keyDerivator
-}
-
-// NewSigner returns a new SigV4 Signer
-func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
-	options := SignerOptions{}
-
-	for _, fn := range optFns {
-		fn(&options)
-	}
-
-	return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()}
-}
-
-type httpSigner struct {
-	Request      *http.Request
-	ServiceName  string
-	Region       string
-	Time         v4Internal.SigningTime
-	Credentials  aws.Credentials
-	KeyDerivator keyDerivator
-	IsPreSign    bool
-	SignedHdrs   []string
-
-	PayloadHash string
-
-	DisableHeaderHoisting  bool
-	DisableURIPathEscaping bool
-	DisableSessionToken    bool
-}
-
-func (s *httpSigner) Build() (signedRequest, error) {
-	req := s.Request
-
-	query := req.URL.Query()
-	headers := req.Header
-
-	s.setRequiredSigningFields(headers, query)
-
-	// Sort Each Query Key's Values
-	for key := range query {
-		sort.Strings(query[key])
-	}
-
-	v4Internal.SanitizeHostForHeader(req)
-
-	credentialScope := s.buildCredentialScope()
-	credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope
-	if s.IsPreSign {
-		query.Set(v4Internal.AmzCredentialKey, credentialStr)
-	}
-
-	unsignedHeaders := headers
-	if s.IsPreSign && !s.DisableHeaderHoisting {
-		var urlValues url.Values
-		urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers)
-		for k := range urlValues {
-			query[k] = urlValues[k]
-		}
-	}
-
-	host := req.URL.Host
-	if len(req.Host) > 0 {
-		host = req.Host
-	}
-
-	signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength)
-
-	if s.IsPreSign {
-		query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr)
-	}
-
-	var rawQuery strings.Builder
-	rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1))
-
-	canonicalURI := v4Internal.GetURIPath(req.URL)
-	if !s.DisableURIPathEscaping {
-		canonicalURI = httpbinding.EscapePath(canonicalURI, false)
-	}
-
-	canonicalString := s.buildCanonicalString(
-		req.Method,
-		canonicalURI,
-		rawQuery.String(),
-		signedHeadersStr,
-		canonicalHeaderStr,
-	)
-
-	strToSign := s.buildStringToSign(credentialScope, canonicalString)
-	signingSignature, err := s.buildSignature(strToSign)
-	if err != nil {
-		return signedRequest{}, err
-	}
-
-	if s.IsPreSign {
-		rawQuery.WriteString("&X-Amz-Signature=")
-		rawQuery.WriteString(signingSignature)
-	} else {
-		headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature))
-	}
-
-	req.URL.RawQuery = rawQuery.String()
-
-	return signedRequest{
-		Request:         req,
-		SignedHeaders:   signedHeaders,
-		CanonicalString: canonicalString,
-		StringToSign:    strToSign,
-		PreSigned:       s.IsPreSign,
-	}, nil
-}
-
-func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string {
-	const credential = "Credential="
-	const signedHeaders = "SignedHeaders="
-	const signature = "Signature="
-	const commaSpace = ", "
-
-	var parts strings.Builder
-	parts.Grow(len(signingAlgorithm) + 1 +
-		len(credential) + len(credentialStr) + 2 +
-		len(signedHeaders) + len(signedHeadersStr) + 2 +
-		len(signature) + len(signingSignature),
-	)
-	parts.WriteString(signingAlgorithm)
-	parts.WriteRune(' ')
-	parts.WriteString(credential)
-	parts.WriteString(credentialStr)
-	parts.WriteString(commaSpace)
-	parts.WriteString(signedHeaders)
-	parts.WriteString(signedHeadersStr)
-	parts.WriteString(commaSpace)
-	parts.WriteString(signature)
-	parts.WriteString(signingSignature)
-	return parts.String()
-}
-
-// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the
-// request is made to, and time the request is signed at. The signTime allows
-// you to specify that a request is signed for the future, and cannot be
-// used until then.
-//
-// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
-// must be provided. Even if the request has no payload (aka body). If the
-// request has no payload you should use the hex encoded SHA-256 of an empty
-// string as the payloadHash value.
-//
-//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-//
-// Some services such as Amazon S3 accept alternative values for the payload
-// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
-// included in the request signature.
-//
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
-//
-// Sign differs from Presign in that it will sign the request using HTTP
-// header values. This type of signing is intended for http.Request values that
-// will not be shared, or are shared in a way the header values on the request
-// will not be lost.
-//
-// The passed in request will be modified in place.
-func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) error {
-	options := s.options
-
-	for _, fn := range optFns {
-		fn(&options)
-	}
-
-	signer := &httpSigner{
-		Request:                r,
-		PayloadHash:            payloadHash,
-		ServiceName:            service,
-		Region:                 region,
-		Credentials:            credentials,
-		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
-		DisableHeaderHoisting:  options.DisableHeaderHoisting,
-		DisableURIPathEscaping: options.DisableURIPathEscaping,
-		DisableSessionToken:    options.DisableSessionToken,
-		KeyDerivator:           s.keyDerivator,
-		SignedHdrs:             signedHdrs,
-	}
-
-	signedRequest, err := signer.Build()
-	if err != nil {
-		return err
-	}
-
-	logSigningInfo(ctx, options, &signedRequest, false)
-
-	return nil
-}
-
-// PresignHTTP signs AWS v4 requests with the payload hash, service name, region
-// the request is made to, and time the request is signed at. The signTime
-// allows you to specify that a request is signed for the future, and cannot
-// be used until then.
-//
-// Returns the signed URL and the map of HTTP headers that were included in the
-// signature or an error if signing the request failed. For presigned requests
-// these headers and their values must be included on the HTTP request when it
-// is made. This is helpful to know what header values need to be shared with
-// the party the presigned request will be distributed to.
-//
-// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
-// must be provided. Even if the request has no payload (aka body). If the
-// request has no payload you should use the hex encoded SHA-256 of an empty
-// string as the payloadHash value.
-//
-//	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-//
-// Some services such as Amazon S3 accept alternative values for the payload
-// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
-// included in the request signature.
-//
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
-//
-// PresignHTTP differs from SignHTTP in that it will sign the request using
-// query string instead of header values. This allows you to share the
-// Presigned Request's URL with third parties, or distribute it throughout your
-// system with minimal dependencies.
-//
-// PresignHTTP will not set the expires time of the presigned request
-// automatically. To specify the expire duration for a request add the
-// "X-Amz-Expires" query parameter on the request with the value as the
-// duration in seconds the presigned URL should be considered valid for. This
-// parameter is not used by all AWS services, and is most notable used by
-// Amazon S3 APIs.
-//
-//	expires := 20 * time.Minute
-//	query := req.URL.Query()
-//	query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10))
-//	req.URL.RawQuery = query.Encode()
-//
-// This method does not modify the provided request.
-func (s *Signer) PresignHTTP(
-	ctx context.Context, credentials aws.Credentials, r *http.Request,
-	payloadHash string, service string, region string, signingTime time.Time,
-	signedHdrs []string,
-	optFns ...func(*SignerOptions),
-) (signedURI string, signedHeaders http.Header, err error) {
-	options := s.options
-
-	for _, fn := range optFns {
-		fn(&options)
-	}
-
-	signer := &httpSigner{
-		Request:                r.Clone(r.Context()),
-		PayloadHash:            payloadHash,
-		ServiceName:            service,
-		Region:                 region,
-		Credentials:            credentials,
-		Time:                   v4Internal.NewSigningTime(signingTime.UTC()),
-		IsPreSign:              true,
-		DisableHeaderHoisting:  options.DisableHeaderHoisting,
-		DisableURIPathEscaping: options.DisableURIPathEscaping,
-		DisableSessionToken:    options.DisableSessionToken,
-		KeyDerivator:           s.keyDerivator,
-		SignedHdrs:             signedHdrs,
-	}
-
-	signedRequest, err := signer.Build()
-	if err != nil {
-		return "", nil, err
-	}
-
-	logSigningInfo(ctx, options, &signedRequest, true)
-
-	signedHeaders = make(http.Header)
-
-	// For the signed headers we canonicalize the header keys in the returned map.
-	// This avoids situations where can standard library double headers like host header. For example the standard
-	// library will set the Host header, even if it is present in lower-case form.
-	for k, v := range signedRequest.SignedHeaders {
-		key := textproto.CanonicalMIMEHeaderKey(k)
-		signedHeaders[key] = append(signedHeaders[key], v...)
-	}
-
-	return signedRequest.Request.URL.String(), signedHeaders, nil
-}
-
-func (s *httpSigner) buildCredentialScope() string {
-	return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName)
-}
-
-func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) {
-	query := url.Values{}
-	unsignedHeaders := http.Header{}
-	for k, h := range header {
-		if r.IsValid(k) {
-			query[k] = h
-		} else {
-			unsignedHeaders[k] = h
-		}
-	}
-
-	return query, unsignedHeaders
-}
-
-func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) {
-	signed = make(http.Header)
-
-	var headers []string
-	const hostHeader = "host"
-	headers = append(headers, hostHeader)
-	signed[hostHeader] = append(signed[hostHeader], host)
-
-	const contentLengthHeader = "content-length"
-	if slices.Contains(s.SignedHdrs, contentLengthHeader) {
-		headers = append(headers, contentLengthHeader)
-		signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10))
-	}
-
-	for k, v := range header {
-		if !rule.IsValid(k) {
-			continue // ignored header
-		}
-		if strings.EqualFold(k, contentLengthHeader) {
-			// prevent signing already handled content-length header.
-			continue
-		}
-
-		lowerCaseKey := strings.ToLower(k)
-		if _, ok := signed[lowerCaseKey]; ok {
-			// include additional values
-			signed[lowerCaseKey] = append(signed[lowerCaseKey], v...)
-			continue
-		}
-
-		headers = append(headers, lowerCaseKey)
-		signed[lowerCaseKey] = v
-	}
-	sort.Strings(headers)
-
-	signedHeaders = strings.Join(headers, ";")
-
-	var canonicalHeaders strings.Builder
-	n := len(headers)
-	const colon = ':'
-	for i := 0; i < n; i++ {
-		if headers[i] == hostHeader {
-			canonicalHeaders.WriteString(hostHeader)
-			canonicalHeaders.WriteRune(colon)
-			canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host))
-		} else {
-			canonicalHeaders.WriteString(headers[i])
-			canonicalHeaders.WriteRune(colon)
-			// Trim out leading, trailing, and dedup inner spaces from signed header values.
-			values := signed[headers[i]]
-			for j, v := range values {
-				cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v))
-				canonicalHeaders.WriteString(cleanedValue)
-				if j < len(values)-1 {
-					canonicalHeaders.WriteRune(',')
-				}
-			}
-		}
-		canonicalHeaders.WriteRune('\n')
-	}
-	canonicalHeadersStr = canonicalHeaders.String()
-
-	return signed, signedHeaders, canonicalHeadersStr
-}
-
-func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string {
-	return strings.Join([]string{
-		method,
-		uri,
-		query,
-		canonicalHeaders,
-		signedHeaders,
-		s.PayloadHash,
-	}, "\n")
-}
-
-func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string {
-	return strings.Join([]string{
-		signingAlgorithm,
-		s.Time.TimeFormat(),
-		credentialScope,
-		hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))),
-	}, "\n")
-}
-
-func makeHash(hash hash.Hash, b []byte) []byte {
-	hash.Reset()
-	hash.Write(b)
-	return hash.Sum(nil)
-}
-
-func (s *httpSigner) buildSignature(strToSign string) (string, error) {
-	key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time)
-	return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil
-}
-
-func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) {
-	amzDate := s.Time.TimeFormat()
-
-	if s.IsPreSign {
-		query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm)
-		sessionToken := s.Credentials.SessionToken
-		if !s.DisableSessionToken && len(sessionToken) > 0 {
-			query.Set("X-Amz-Security-Token", sessionToken)
-		}
-
-		query.Set(v4Internal.AmzDateKey, amzDate)
-		return
-	}
-
-	headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate)
-
-	if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 {
-		headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken)
-	}
-}
-
-func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) {
-	if !options.LogSigning {
-		return
-	}
-	signedURLMsg := ""
-	if isPresign {
-		signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String())
-	}
-	logger := logging.WithContext(ctx, options.Logger)
-	logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg)
-}
-
-type signedRequest struct {
-	Request         *http.Request
-	SignedHeaders   http.Header
-	CanonicalString string
-	StringToSign    string
-	PreSigned       bool
-}
-
-const logSignInfoMsg = `Request Signature:
----[ CANONICAL STRING  ]-----------------------------
-%s
----[ STRING TO SIGN ]--------------------------------
-%s%s
------------------------------------------------------`
-const logSignedURLMsg = `
----[ SIGNED URL ]------------------------------------
-%s`

aws/signer/v4/v4_test.go

@@ -1,358 +0,0 @@
-package v4
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/google/go-cmp/cmp"
-	v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
-)
-
-var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"}
-
-func buildRequest(serviceName, region, body string) (*http.Request, string) {
-	reader := strings.NewReader(body)
-	return buildRequestWithBodyReader(serviceName, region, reader)
-}
-
-func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) {
-	var bodyLen int
-
-	type lenner interface {
-		Len() int
-	}
-	if lr, ok := body.(lenner); ok {
-		bodyLen = lr.Len()
-	}
-
-	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
-	req, _ := http.NewRequest("POST", endpoint, body)
-	req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
-	req.Header.Set("X-Amz-Target", "prefix.Operation")
-	req.Header.Set("Content-Type", "application/x-amz-json-1.0")
-
-	if bodyLen > 0 {
-		req.ContentLength = int64(bodyLen)
-	}
-
-	req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)")
-	req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
-	req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
-
-	h := sha256.New()
-	_, _ = io.Copy(h, body)
-	payloadHash := hex.EncodeToString(h.Sum(nil))
-
-	return req, payloadHash
-}
-
-func TestPresignRequest(t *testing.T) {
-	req, body := buildRequest("dynamodb", "us-east-1", "{}")
-
-	query := req.URL.Query()
-	query.Set("X-Amz-Expires", "300")
-	req.URL.RawQuery = query.Encode()
-
-	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
-	signer := NewSigner()
-	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
-	if err != nil {
-		t.Fatalf("expected no error, got %v", err)
-	}
-
-	expectedDate := "19700101T000000Z"
-	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
-	expectedSig := "122f0b9e091e4ba84286097e2b3404a1f1f4c4aad479adda95b7dff0ccbe5581"
-	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
-	expectedTarget := "prefix.Operation"
-
-	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
-	if err != nil {
-		t.Errorf("expect no error, got %v", err)
-	}
-
-	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
-		t.Errorf("expect %v to be empty", a)
-	}
-	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-
-	for _, h := range strings.Split(expectedHeaders, ";") {
-		v := headers.Get(h)
-		if len(v) == 0 {
-			t.Errorf("expect %v, to be present in header map", h)
-		}
-	}
-}
-
-func TestPresignBodyWithArrayRequest(t *testing.T) {
-	req, body := buildRequest("dynamodb", "us-east-1", "{}")
-	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
-
-	query := req.URL.Query()
-	query.Set("X-Amz-Expires", "300")
-	req.URL.RawQuery = query.Encode()
-
-	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
-	signer := NewSigner()
-	signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-
-	q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
-	if err != nil {
-		t.Errorf("expect no error, got %v", err)
-	}
-
-	expectedDate := "19700101T000000Z"
-	expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
-	expectedSig := "e3ac55addee8711b76c6d608d762cff285fe8b627a057f8b5ec9268cf82c08b1"
-	expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
-	expectedTarget := "prefix.Operation"
-
-	if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
-		t.Errorf("expect %v to be empty, was not", a)
-	}
-	if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-
-	for _, h := range strings.Split(expectedHeaders, ";") {
-		v := headers.Get(h)
-		if len(v) == 0 {
-			t.Errorf("expect %v, to be present in header map", h)
-		}
-	}
-}
-
-func TestSignRequest(t *testing.T) {
-	req, body := buildRequest("dynamodb", "us-east-1", "{}")
-	signer := NewSigner()
-	signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
-	err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-
-	expectedDate := "19700101T000000Z"
-	expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9"
-
-	q := req.Header
-	if e, a := expectedSig, q.Get("Authorization"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-	if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-}
-
-func TestBuildCanonicalRequest(t *testing.T) {
-	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
-	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
-
-	ctx := &httpSigner{
-		ServiceName:  "dynamodb",
-		Region:       "us-east-1",
-		Request:      req,
-		Time:         v4Internal.NewSigningTime(time.Now()),
-		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
-	}
-
-	build, err := ctx.Build()
-	if err != nil {
-		t.Fatalf("expected no error, got %v", err)
-	}
-
-	expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z"
-	if e, a := expected, build.Request.URL.String(); e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
-}
-
-func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
-	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
-	req.Body = io.NopCloser(bytes.NewReader([]byte{}))
-
-	s := NewSigner()
-
-	origBody := req.Body
-
-	err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
-	if err != nil {
-		t.Fatalf("expect no error, got %v", err)
-	}
-
-	if req.Body != origBody {
-		t.Errorf("expect request body to not be chagned")
-	}
-}
-
-func TestRequestHost(t *testing.T) {
-	req, _ := buildRequest("dynamodb", "us-east-1", "{}")
-	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
-	req.Host = "myhost"
-
-	query := req.URL.Query()
-	query.Set("X-Amz-Expires", "5")
-	req.URL.RawQuery = query.Encode()
-
-	ctx := &httpSigner{
-		ServiceName:  "dynamodb",
-		Region:       "us-east-1",
-		Request:      req,
-		Time:         v4Internal.NewSigningTime(time.Now()),
-		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
-	}
-
-	build, err := ctx.Build()
-	if err != nil {
-		t.Fatalf("expected no error, got %v", err)
-	}
-
-	if !strings.Contains(build.CanonicalString, "host:"+req.Host) {
-		t.Errorf("canonical host header invalid")
-	}
-}
-
-func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) {
-	body := `{"description": "this is a test"}`
-	req, _ := buildRequest("dynamodb", "us-east-1", body)
-	req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
-	req.Host = "myhost"
-
-	contentLength := fmt.Sprintf("%d", len([]byte(body)))
-	req.Header.Add("Content-Length", contentLength)
-
-	query := req.URL.Query()
-	query.Set("X-Amz-Expires", "5")
-	req.URL.RawQuery = query.Encode()
-
-	ctx := &httpSigner{
-		ServiceName:  "dynamodb",
-		Region:       "us-east-1",
-		Request:      req,
-		Time:         v4Internal.NewSigningTime(time.Now()),
-		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
-	}
-
-	_, err := ctx.Build()
-	if err != nil {
-		t.Fatalf("expected no error, got %v", err)
-	}
-
-	//if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") {
-	//	t.Errorf("canonical header content-length invalid")
-	//}
-}
-
-func TestSign_buildCanonicalHeaders(t *testing.T) {
-	serviceName := "mockAPI"
-	region := "mock-region"
-	endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
-
-	req, err := http.NewRequest("POST", endpoint, nil)
-	if err != nil {
-		t.Fatalf("failed to create request, %v", err)
-	}
-
-	req.Header.Set("FooInnerSpace", "   inner      space    ")
-	req.Header.Set("FooLeadingSpace", "    leading-space")
-	req.Header.Add("FooMultipleSpace", "no-space")
-	req.Header.Add("FooMultipleSpace", "\ttab-space")
-	req.Header.Add("FooMultipleSpace", "trailing-space    ")
-	req.Header.Set("FooNoSpace", "no-space")
-	req.Header.Set("FooTabSpace", "\ttab-space\t")
-	req.Header.Set("FooTrailingSpace", "trailing-space    ")
-	req.Header.Set("FooWrappedSpace", "   wrapped-space    ")
-
-	ctx := &httpSigner{
-		ServiceName:  serviceName,
-		Region:       region,
-		Request:      req,
-		Time:         v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)),
-		KeyDerivator: v4Internal.NewSigningKeyDeriver(),
-	}
-
-	build, err := ctx.Build()
-	if err != nil {
-		t.Fatalf("expected no error, got %v", err)
-	}
-
-	expectCanonicalString := strings.Join([]string{
-		`POST`,
-		`/`,
-		``,
-		`fooinnerspace:inner space`,
-		`fooleadingspace:leading-space`,
-		`foomultiplespace:no-space,tab-space,trailing-space`,
-		`foonospace:no-space`,
-		`footabspace:tab-space`,
-		`footrailingspace:trailing-space`,
-		`foowrappedspace:wrapped-space`,
-		`host:mockAPI.mock-region.amazonaws.com`,
-		`x-amz-date:20211020T124200Z`,
-		``,
-		`fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`,
-		``,
-	}, "\n")
-	if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" {
-		t.Errorf("expect match, got\n%s", diff)
-	}
-}
-
-func BenchmarkPresignRequest(b *testing.B) {
-	signer := NewSigner()
-	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
-
-	query := req.URL.Query()
-	query.Set("X-Amz-Expires", "5")
-	req.URL.RawQuery = query.Encode()
-
-	for i := 0; i < b.N; i++ {
-		signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
-	}
-}
-
-func BenchmarkSignRequest(b *testing.B) {
-	signer := NewSigner()
-	req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
-	for i := 0; i < b.N; i++ {
-		signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
-	}
-}

backend/azure/err.go

@@ -1,63 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package azure
-
-import (
-	"errors"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/versity/versitygw/s3err"
-)
-
-// Parses azure ResponseError into AWS APIError
-func azureErrToS3Err(apiErr error) error {
-	var azErr *azcore.ResponseError
-	if !errors.As(apiErr, &azErr) {
-		return apiErr
-	}
-
-	return azErrToS3err(azErr)
-}
-
-func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
-	switch azErr.ErrorCode {
-	case "ContainerAlreadyExists":
-		return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
-	case "InvalidResourceName", "ContainerNotFound":
-		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
-	case "BlobNotFound":
-		return s3err.GetAPIError(s3err.ErrNoSuchKey)
-	case "TagsTooLarge":
-		return s3err.GetAPIError(s3err.ErrInvalidTag)
-	case "Requested Range Not Satisfiable":
-		return s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-	return s3err.APIError{
-		Code:           azErr.ErrorCode,
-		Description:    azErr.RawResponse.Status,
-		HTTPStatusCode: azErr.StatusCode,
-	}
-}
-
-func parseMpError(mpErr error) error {
-	err := azureErrToS3Err(mpErr)
-
-	serr, ok := err.(s3err.APIError)
-	if !ok || serr.Code != "NoSuchKey" {
-		return mpErr
-	}
-
-	return s3err.GetAPIError(s3err.ErrNoSuchUpload)
-}

backend/backend.go

@@ -15,15 +15,13 @@
 package backend
 
 import (
-	"bufio"
 	"context"
 	"fmt"
+	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
-	"github.com/versity/versitygw/s3select"
 )
 
 //go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
@@ -32,23 +30,15 @@ type Backend interface {
 	Shutdown()
 
 	// bucket operations
-	ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
 	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
-	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
+	CreateBucket(context.Context, *s3.CreateBucketInput) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
-	DeleteBucket(_ context.Context, bucket string) error
-	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
-	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
-	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
-	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
-	DeleteBucketPolicy(_ context.Context, bucket string) error
-	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
-	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
-	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
+	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
 
 	// multipart operations
-	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
@@ -57,43 +47,29 @@ type Backend interface {
 	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
 
 	// standard object operations
-	PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
+	PutObject(context.Context, *s3.PutObjectInput) (string, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
 	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
-	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
-	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
-	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
-	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
-	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
 
 	// special case object operations
 	RestoreObject(context.Context, *s3.RestoreObjectInput) error
-	SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
+	SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
 
-	// bucket tagging operations
-	GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
-	PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
-	DeleteBucketTagging(_ context.Context, bucket string) error
-
-	// object tagging operations
+	// object tags operations
 	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
 	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
 	DeleteObjectTagging(_ context.Context, bucket, object string) error
 
-	// object lock operations
-	PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
-	GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
-	PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
-	GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
-	PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
-	GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
-
 	// non AWS actions
-	ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
+	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
 	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
@@ -108,7 +84,7 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -117,42 +93,18 @@ func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.
 func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, []byte) error {
+func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
-	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
-	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
-	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -173,35 +125,35 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu
 	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
-	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
-	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
-	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
-	return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
-	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -210,33 +162,8 @@ func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) e
 func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
-	return func(w *bufio.Writer) {
-		var getProgress s3select.GetProgress
-		progress := input.RequestProgress
-		if progress != nil && *progress.Enabled {
-			getProgress = func() (bytesScanned int64, bytesProcessed int64) {
-				return -1, -1
-			}
-		}
-		mh := s3select.NewMessageHandler(ctx, w, getProgress)
-		apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
-		mh.FinishWithError(apiErr.Code, apiErr.Description)
-	}
-}
-
-func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
-	return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-
-func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
+	return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
@@ -249,26 +176,7 @@ func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-
-func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {

backend/common.go

@@ -18,9 +18,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"io"
-	"net/http"
-	"os"
+	"io/fs"
 	"strconv"
 	"strings"
 	"time"
@@ -30,10 +28,9 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
-const (
-	// this is the media type for directories in AWS and Nextcloud
-	DirContentType     = "application/x-directory"
-	DefaultContentType = "binary/octet-stream"
+var (
+	// RFC3339TimeFormat RFC3339 time format
+	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -50,18 +47,8 @@ func (d ByObjectName) Len() int           { return len(d) }
 func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
 
-func GetPtrFromString(str string) *string {
-	if str == "" {
-		return nil
-	}
-	return &str
-}
-
-func GetStringFromPtr(str *string) string {
-	if str == nil {
-		return ""
-	}
-	return *str
+func GetStringPtr(s string) *string {
+	return &s
 }
 
 func GetTimePtr(t time.Time) *time.Time {
@@ -74,9 +61,9 @@ var (
 
 // ParseRange parses input range header and returns startoffset, length, and
 // error. If no endoffset specified, then length is set to -1.
-func ParseRange(size int64, acceptRange string) (int64, int64, error) {
+func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, size, nil
+		return 0, fi.Size(), nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
@@ -112,38 +99,6 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
-// ParseCopySource parses x-amz-copy-source header and returns source bucket,
-// source object, versionId, error respectively
-func ParseCopySource(copySourceHeader string) (string, string, string, error) {
-	if copySourceHeader[0] == '/' {
-		copySourceHeader = copySourceHeader[1:]
-	}
-
-	var copySource, versionId string
-	i := strings.LastIndex(copySourceHeader, "?versionId=")
-	if i == -1 {
-		copySource = copySourceHeader
-	} else {
-		copySource = copySourceHeader[:i]
-		versionId = copySourceHeader[i+11:]
-	}
-
-	srcBucket, srcObject, ok := strings.Cut(copySource, "/")
-	if !ok {
-		return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
-	}
-
-	return srcBucket, srcObject, versionId, nil
-}
-
-func CreateExceedingRangeErr(objSize int64) s3err.APIError {
-	return s3err.APIError{
-		Code:           "InvalidArgument",
-		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
-		HTTPStatusCode: http.StatusBadRequest,
-	}
-}
-
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
@@ -165,16 +120,3 @@ func md5String(data []byte) string {
 	sum := md5.Sum(data)
 	return hex.EncodeToString(sum[:])
 }
-
-type FileSectionReadCloser struct {
-	R io.Reader
-	F *os.File
-}
-
-func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
-	return f.R.Read(p)
-}
-
-func (f *FileSectionReadCloser) Close() error {
-	return f.F.Close()
-}

backend/meta/meta.go

@@ -1,42 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package meta
-
-import "os"
-
-// MetadataStorer defines the interface for managing metadata.
-// When object == "", the operation is on the bucket.
-type MetadataStorer interface {
-	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
-	// Returns the value of the attribute, or an error if the attribute does not exist.
-	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
-
-	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
-	// If attribute already exists, new attribute should replace existing.
-	// Returns an error if the operation fails.
-	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
-
-	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
-	// Returns an error if the operation fails.
-	DeleteAttribute(bucket, object, attribute string) error
-
-	// ListAttributes lists all attributes for an object or a bucket.
-	// Returns list of attribute names, or an error if the operation fails.
-	ListAttributes(bucket, object string) ([]string, error)
-
-	// DeleteAttributes removes all attributes for an object or a bucket.
-	// Returns an error if the operation fails.
-	DeleteAttributes(bucket, object string) error
-}

backend/meta/xattr.go

@@ -1,114 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package meta
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"syscall"
-
-	"github.com/pkg/xattr"
-)
-
-const (
-	xattrPrefix = "user."
-)
-
-var (
-	// ErrNoSuchKey is returned when the key does not exist.
-	ErrNoSuchKey = errors.New("no such key")
-)
-
-type XattrMeta struct{}
-
-// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
-	if f != nil {
-		b, err := xattr.FGet(f, xattrPrefix+attribute)
-		if errors.Is(err, xattr.ENOATTR) {
-			return nil, ErrNoSuchKey
-		}
-		return b, err
-	}
-
-	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
-	if errors.Is(err, xattr.ENOATTR) {
-		return nil, ErrNoSuchKey
-	}
-	return b, err
-}
-
-// StoreAttribute stores the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
-	if f != nil {
-		return xattr.FSet(f, xattrPrefix+attribute, value)
-	}
-
-	return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
-}
-
-// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
-	err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
-	if errors.Is(err, xattr.ENOATTR) {
-		return ErrNoSuchKey
-	}
-	return err
-}
-
-// DeleteAttributes is not implemented for xattr since xattrs
-// are automatically removed when the file is deleted.
-func (x XattrMeta) DeleteAttributes(bucket, object string) error {
-	return nil
-}
-
-// ListAttributes lists all attributes for an object in a bucket.
-func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
-	attrs, err := xattr.List(filepath.Join(bucket, object))
-	if err != nil {
-		return nil, err
-	}
-	attributes := make([]string, 0, len(attrs))
-	for _, attr := range attrs {
-		if !isUserAttr(attr) {
-			continue
-		}
-		attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
-	}
-	return attributes, nil
-}
-
-func isUserAttr(attr string) bool {
-	return strings.HasPrefix(attr, xattrPrefix)
-}
-
-// Test is a helper function to test if xattrs are supported.
-func (x XattrMeta) Test(path string) error {
-	// check for platform support
-	if !xattr.XATTR_SUPPORTED {
-		return fmt.Errorf("xattrs are not supported on this platform")
-	}
-
-	// check if the filesystem supports xattrs
-	_, err := xattr.Get(path, "user.test")
-	if errors.Is(err, syscall.ENOTSUP) {
-		return fmt.Errorf("xattrs are not supported on this filesystem")
-	}
-
-	return nil
-}

backend/mkdir.go

@@ -1,77 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-// Copyright 2024 Versity Software
-
-// MkdirAll borrowed from stdlib to add ability to set ownership
-// as directories are created
-
-package backend
-
-import (
-	"io/fs"
-	"os"
-
-	"github.com/versity/versitygw/s3err"
-)
-
-// MkdirAll is similar to os.MkdirAll but it will return
-// ErrObjectParentIsFile when appropriate
-// MkdirAll creates a directory named path,
-// along with any necessary parents, and returns nil,
-// or else returns an error.
-// The permission bits perm (before umask) are used for all
-// directories that MkdirAll creates.
-// Any newly created directory is set to provided uid/gid ownership.
-// If path is already a directory, MkdirAll does nothing
-// and returns nil.
-// Any directory created will be set to provided uid/gid ownership
-// if doChown is true.
-func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, dirPerm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return err
-	}
-	if doChown {
-		err = os.Chown(path, uid, gid)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

backend/posix/posix_darwin.go

@@ -12,9 +12,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build !linux
-// +build !linux
-
 package posix
 
 import (
@@ -24,9 +21,6 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/backend"
 )
 
 type tmpfile struct {
@@ -36,36 +30,20 @@ type tmpfile struct {
 	size    int64
 }
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
-	uid, gid, doChown := p.getChownIDs(acct)
-
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// Create a temp file for upload while in progress (see link comments below).
-	var err error
-	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	err := os.MkdirAll(dir, 0700)
 	if err != nil {
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
-		return nil, fmt.Errorf("create temp file: %w", err)
+		return nil, err
 	}
-
-	if doChown {
-		err := f.Chown(uid, gid)
-		if err != nil {
-			return nil, fmt.Errorf("set temp file ownership: %w", err)
-		}
-	}
-
 	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
 }
 
-var (
-	// TODO: make this configurable
-	defaultFilePerm fs.FileMode = 0644
-)
-
 func (tmp *tmpfile) link() error {
 	tempname := tmp.f.Name()
 	// cleanup in case anything goes wrong, if rename succeeds then
@@ -83,9 +61,6 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	// reset default file mode because CreateTemp uses 0600
-	tmp.f.Chmod(defaultFilePerm)
-
 	err = tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)
@@ -112,7 +87,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
-
-func (tmp *tmpfile) File() *os.File {
-	return tmp.f
-}

backend/posix/posix_linux.go

@@ -12,9 +12,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build linux
-// +build linux
-
 package posix
 
 import (
@@ -27,43 +24,30 @@ import (
 	"strconv"
 	"syscall"
 
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/backend"
 	"golang.org/x/sys/unix"
 )
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f          *os.File
-	bucket     string
-	objname    string
-	isOTmp     bool
-	size       int64
-	needsChown bool
-	uid        int
-	gid        int
-	newDirPerm fs.FileMode
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
 }
 
-var (
-	// TODO: make this configurable
-	defaultFilePerm uint32 = 0644
-)
-
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
-	uid, gid, doChown := p.getChownIDs(acct)
-
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
 	// Not all filesystems support this, so fallback to CreateTemp for when
 	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
 	if err != nil {
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+		err := os.MkdirAll(dir, 0700)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -72,27 +56,11 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		if err != nil {
 			return nil, err
 		}
-		tmp := &tmpfile{
-			f:          f,
-			bucket:     bucket,
-			objname:    obj,
-			size:       size,
-			needsChown: doChown,
-			uid:        uid,
-			gid:        gid,
-		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
 		// falloc is best effort, its fine if this fails
-		if size > 0 && dofalloc {
+		if size > 0 {
 			tmp.falloc()
 		}
-
-		if doChown {
-			err := f.Chown(uid, gid)
-			if err != nil {
-				return nil, fmt.Errorf("set temp file ownership: %w", err)
-			}
-		}
-
 		return tmp, nil
 	}
 
@@ -100,30 +68,11 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{
-		f:          f,
-		bucket:     bucket,
-		objname:    obj,
-		isOTmp:     true,
-		size:       size,
-		needsChown: doChown,
-		uid:        uid,
-		gid:        gid,
-		newDirPerm: p.newDirPerm,
-	}
-
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
 	// falloc is best effort, its fine if this fails
-	if size > 0 && dofalloc {
+	if size > 0 {
 		tmp.falloc()
 	}
-
-	if doChown {
-		err := f.Chown(uid, gid)
-		if err != nil {
-			return nil, fmt.Errorf("set temp file ownership: %w", err)
-		}
-	}
-
 	return tmp, nil
 }
 
@@ -136,9 +85,6 @@ func (tmp *tmpfile) falloc() error {
 }
 
 func (tmp *tmpfile) link() error {
-	// make sure this is cleaned up in all error cases
-	defer tmp.f.Close()
-
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
 	// with any other simultaneous uploads. The final operation is to move the
@@ -151,13 +97,6 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	dir := filepath.Dir(objPath)
-
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
-	if err != nil {
-		return fmt.Errorf("make parent dir: %w", err)
-	}
-
 	if !tmp.isOTmp {
 		// O_TMPFILE not suported, use fallback
 		return tmp.fallbackLink()
@@ -169,27 +108,17 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dirf, err := os.Open(dir)
+	dir, err := os.Open(filepath.Dir(objPath))
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dirf.Close()
+	defer dir.Close()
 
-	for {
-		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-		if errors.Is(err, syscall.EEXIST) {
-			err := os.Remove(objPath)
-			if err != nil && !errors.Is(err, fs.ErrNotExist) {
-				return fmt.Errorf("remove stale path: %w", err)
-			}
-			continue
-		}
-		if err != nil {
-			return fmt.Errorf("link tmpfile (fd %q as %q): %w",
-				filepath.Base(tmp.f.Name()), objPath, err)
-		}
-		break
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if err != nil {
+		return fmt.Errorf("link tmpfile (%q in %q): %w",
+			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
 	}
 
 	err = tmp.f.Close()
@@ -206,9 +135,6 @@ func (tmp *tmpfile) fallbackLink() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
-	// reset default file mode because CreateTemp uses 0600
-	tmp.f.Chmod(fs.FileMode(defaultFilePerm))
-
 	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)
@@ -236,7 +162,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
-
-func (tmp *tmpfile) File() *os.File {
-	return tmp.f
-}

backend/posix/posix_windows.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length")
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/s3proxy/client.go

@@ -17,6 +17,7 @@ package s3proxy
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net/http"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -25,24 +26,24 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go/middleware"
+	"github.com/versity/versitygw/auth"
 )
 
-func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
-	cfg, err := s.getConfig(ctx, s.access, s.secret)
-	if err != nil {
-		return nil, err
+func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		return nil, fmt.Errorf("invalid account in context")
 	}
 
-	if s.endpoint != "" {
-		return s3.NewFromConfig(cfg, func(o *s3.Options) {
-			o.BaseEndpoint = &s.endpoint
-		}), nil
+	cfg, err := s.getConfig(ctx, acct.Access, acct.Secret)
+	if err != nil {
+		return nil, err
 	}
 
 	return s3.NewFromConfig(cfg), nil
 }
 
-func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
 	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
 
 	tr := &http.Transport{
@@ -56,6 +57,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 		config.WithHTTPClient(client),
 	}
 
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
 	if s.disableChecksum {
 		opts = append(opts,
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -68,3 +74,13 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 
 	return config.LoadDefaultConfig(ctx, opts...)
 }
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *S3be) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}

backend/scoutfs/scoutfs.go

@@ -26,35 +26,20 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/xattr"
-	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3response"
 )
 
-type ScoutfsOpts struct {
-	ChownUID    bool
-	ChownGID    bool
-	GlacierMode bool
-	BucketLinks bool
-	NewDirPerm  fs.FileMode
-}
-
 type ScoutFS struct {
 	*posix.Posix
 	rootfd  *os.File
 	rootdir string
 
-	// bucket/object metadata storage facility
-	meta meta.MetadataStorer
-
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
 	// HEAD object: if file offline, set obj storage class to GLACIER
@@ -65,19 +50,6 @@ type ScoutFS struct {
 	// ListObjects: if file offline, set obj storage class to GLACIER
 	// RestoreObject: add batch stage request to file
 	glaciermode bool
-
-	// chownuid/gid enable chowning of files to the account uid/gid
-	// when objects are uploaded
-	chownuid bool
-	chowngid bool
-
-	// euid/egid are the effective uid/gid of the running versitygw process
-	// used to determine if chowning is needed
-	euid int
-	egid int
-
-	// newDirPerm is the permissions to use when creating new directories
-	newDirPerm fs.FileMode
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -86,13 +58,8 @@ const (
 	metaTmpDir          = ".sgwtmp"
 	metaTmpMultipartDir = metaTmpDir + "/multipart"
 	tagHdr              = "X-Amz-Tagging"
-	metaHdr             = "X-Amz-Meta"
-	contentTypeHdr      = "content-type"
-	contentEncHdr       = "content-encoding"
 	emptyMD5            = "d41d8cd98f00b204e9800998ecf8427e"
-	etagkey             = "etag"
-	objectRetentionKey  = "object-retention"
-	objectLegalHoldKey  = "object-legal-hold"
+	etagkey             = "user.etag"
 )
 
 var (
@@ -103,12 +70,11 @@ var (
 
 const (
 	// ScoutFS special xattr types
+
 	systemPrefix = "scoutfs.hide."
 	onameAttr    = systemPrefix + "objname"
 	flagskey     = systemPrefix + "sam_flags"
 	stagecopykey = systemPrefix + "sam_stagereq"
-
-	fsBlocksize = 4096
 )
 
 const (
@@ -127,6 +93,14 @@ const (
 	ExtCacheDone
 )
 
+// Option sets various options for scoutfs
+type Option func(s *ScoutFS)
+
+// WithGlacierEmulation sets glacier mode emulation
+func WithGlacierEmulation() Option {
+	return func(s *ScoutFS) { s.glaciermode = true }
+}
+
 func (s *ScoutFS) Shutdown() {
 	s.Posix.Shutdown()
 	s.rootfd.Close()
@@ -137,47 +111,10 @@ func (*ScoutFS) String() string {
 	return "ScoutFS Gateway"
 }
 
-// getChownIDs returns the uid and gid that should be used for chowning
-// the object to the account uid/gid. It also returns a boolean indicating
-// if chowning is needed.
-func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
-	uid := s.euid
-	gid := s.egid
-	var needsChown bool
-	if s.chownuid && acct.UserID != s.euid {
-		uid = acct.UserID
-		needsChown = true
-	}
-	if s.chowngid && acct.GroupID != s.egid {
-		gid = acct.GroupID
-		needsChown = true
-	}
-
-	return uid, gid, needsChown
-}
-
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
-func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	acct, ok := ctx.Value("account").(auth.Account)
-	if !ok {
-		acct = auth.Account{}
-	}
-
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if input.UploadId == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
-	}
-	if input.MultipartUpload == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
-	}
-
+func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
@@ -196,20 +133,15 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		return nil, err
 	}
 
-	objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
+	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
 
 	// check all parts ok
 	last := len(parts) - 1
 	partsize := int64(0)
 	var totalsize int64
-	for i, part := range parts {
-		if part.PartNumber == nil || *part.PartNumber < 1 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
-
-		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
-		fullPartPath := filepath.Join(bucket, partObjPath)
-		fi, err := os.Lstat(fullPartPath)
+	for i, p := range parts {
+		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
+		fi, err := os.Lstat(partPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
@@ -217,140 +149,88 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
 		if i == 0 {
 			partsize = fi.Size()
 		}
-
-		// partsize must be a multiple of the filesystem blocksize
-		// except for last part
-		if i < last && partsize%fsBlocksize != 0 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
-
 		totalsize += fi.Size()
 		// all parts except the last need to be the same size
 		if i < last && partsize != fi.Size() {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
+		// non-last part sizes need to be multiples of 4k for move blocks
+		// TODO: fallback to no move blocks if not 4k aligned?
+		if i == 0 && i < last && fi.Size()%4096 != 0 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		}
 
-		b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
+		b, err := xattr.Get(partPath, "user.etag")
 		etag := string(b)
 		if err != nil {
 			etag = ""
 		}
-		if parts[i].ETag == nil || etag != *parts[i].ETag {
+		if etag != *parts[i].ETag {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 	}
 
 	// use totalsize=0 because we wont be writing to the file, only moving
 	// extents around.  so we dont want to fallocate this.
-	f, err := s.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0, acct)
+	f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
 	if err != nil {
-		if errors.Is(err, syscall.EDQUOT) {
-			return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
-		}
 		return nil, fmt.Errorf("open temp file: %w", err)
 	}
 	defer f.cleanup()
 
-	for _, part := range parts {
-		if part.PartNumber == nil || *part.PartNumber < 1 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
-
-		partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
-		fullPartPath := filepath.Join(bucket, partObjPath)
-		pf, err := os.Open(fullPartPath)
+	for _, p := range parts {
+		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
 		if err != nil {
-			return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
+			return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
 		}
 
 		// scoutfs move data is a metadata only operation that moves the data
 		// extent references from the source, appeding to the destination.
 		// this needs to be 4k aligned.
-		err = moveData(pf, f.File())
+		err = moveData(pf, f.f)
 		pf.Close()
 		if err != nil {
-			return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
+			return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
 		}
 	}
 
 	userMetaData := make(map[string]string)
 	upiddir := filepath.Join(objdir, uploadID)
-	cType, _ := s.loadUserMetaData(bucket, upiddir, userMetaData)
+	loadUserMetaData(upiddir, userMetaData)
 
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
 	if dir != "" {
-		uid, gid, doChown := s.getChownIDs(acct)
-		err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
-		if err != nil {
-			return nil, err
+		if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
+			if err != nil {
+				return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
+			}
 		}
 	}
+	err = f.link()
+	if err != nil {
+		return nil, fmt.Errorf("link object in namespace: %w", err)
+	}
 
 	for k, v := range userMetaData {
-		err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
+		err = xattr.Set(objname, "user."+k, []byte(v))
 		if err != nil {
+			// cleanup object if returning error
+			os.Remove(objname)
 			return nil, fmt.Errorf("set user attr %q: %w", k, err)
 		}
 	}
 
-	// load and set tagging
-	tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
-	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object tagging: %w", err)
-	}
-	if err == nil {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
-		if err != nil {
-			return nil, fmt.Errorf("set object tagging: %w", err)
-		}
-	}
-
-	// set content-type
-	if cType != "" {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
-		if err != nil {
-			return nil, fmt.Errorf("set object content type: %w", err)
-		}
-	}
-
-	// load and set legal hold
-	lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
-	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object legal hold: %w", err)
-	}
-	if err == nil {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
-		if err != nil {
-			return nil, fmt.Errorf("set object legal hold: %w", err)
-		}
-	}
-
-	// load and set retention
-	ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
-	if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-		return nil, fmt.Errorf("get object retention: %w", err)
-	}
-	if err == nil {
-		err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
-		if err != nil {
-			return nil, fmt.Errorf("set object retention: %w", err)
-		}
-	}
-
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := backend.GetMultipartMD5(parts)
 
-	err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
+	err = xattr.Set(objname, "user.etag", []byte(s3MD5))
 	if err != nil {
+		// cleanup object if returning error
+		os.Remove(objname)
 		return nil, fmt.Errorf("set etag attr: %w", err)
 	}
 
-	err = f.link()
-	if err != nil {
-		return nil, fmt.Errorf("link object in namespace: %w", err)
-	}
-
 	// cleanup tmp dirs
 	os.RemoveAll(upiddir)
 	// use Remove for objdir in case there are still other uploads
@@ -378,107 +258,106 @@ func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte
 	return sum, nil
 }
 
-// fll out the user metadata map with the metadata for the object
-// and return the content type and encoding
-func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
-	ents, err := s.meta.ListAttributes(bucket, object)
+func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
+	ents, err := xattr.List(path)
 	if err != nil || len(ents) == 0 {
-		return "", ""
+		return
 	}
 	for _, e := range ents {
 		if !isValidMeta(e) {
 			continue
 		}
-		b, err := s.meta.RetrieveAttribute(nil, bucket, object, e)
+		b, err := xattr.Get(path, e)
+		if err == syscall.ENODATA {
+			m[strings.TrimPrefix(e, "user.")] = ""
+			continue
+		}
 		if err != nil {
 			continue
 		}
-		if b == nil {
-			m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = ""
-			continue
-		}
-		m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
+		m[strings.TrimPrefix(e, "user.")] = string(b)
 	}
 
-	var contentType, contentEncoding string
-	b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
+	b, err := xattr.Get(path, "user.content-type")
 	contentType = string(b)
+	if err != nil {
+		contentType = ""
+	}
 	if contentType != "" {
-		m[contentTypeHdr] = contentType
+		m["content-type"] = contentType
 	}
 
-	b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
+	b, err = xattr.Get(path, "user.content-encoding")
 	contentEncoding = string(b)
+	if err != nil {
+		contentEncoding = ""
+	}
 	if contentEncoding != "" {
-		m[contentEncHdr] = contentEncoding
+		m["content-encoding"] = contentEncoding
 	}
 
-	return contentType, contentEncoding
+	return
 }
 
 func isValidMeta(val string) bool {
-	if strings.HasPrefix(val, metaHdr) {
+	if strings.HasPrefix(val, "user.X-Amz-Meta") {
 		return true
 	}
-	if strings.EqualFold(val, "Expires") {
+	if strings.EqualFold(val, "user.Expires") {
 		return true
 	}
 	return false
 }
 
-func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
+// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
+// when appropriate
+func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
 	}
-	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
 	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		err = mkdirAll(path[:j-1], perm, bucket, object)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, perm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
+	}
+	return nil
+}
+
+func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 
-	if input.PartNumber != nil {
-		uploadId, sum, err := s.retrieveUploadId(bucket, object)
-		if err != nil {
-			return nil, err
-		}
-
-		ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("read parts: %w", err)
-		}
-
-		partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
-
-		part, err := os.Stat(filepath.Join(bucket, partPath))
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
-		}
-		if errors.Is(err, syscall.ENAMETOOLONG) {
-			return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("stat part: %w", err)
-		}
-
-		b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
-		etag := string(b)
-		if err != nil {
-			etag = ""
-		}
-		partsCount := int32(len(ents))
-		size := part.Size()
-
-		return &s3.HeadObjectOutput{
-			LastModified:  backend.GetTimePtr(part.ModTime()),
-			ETag:          &etag,
-			PartsCount:    &partsCount,
-			ContentLength: &size,
-		}, nil
-	}
-
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -488,30 +367,18 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 	}
 
 	objPath := filepath.Join(bucket, object)
-
 	fi, err := os.Stat(objPath)
-	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
+	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
-	if errors.Is(err, syscall.ENAMETOOLONG) {
-		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
-	}
 	if err != nil {
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
-	if strings.HasSuffix(object, "/") && !fi.IsDir() {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
 
 	userMetaData := make(map[string]string)
-	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
+	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
 
-	if fi.IsDir() {
-		// this is the media type for directories in AWS and Nextcloud
-		contentType = "application/x-directory"
-	}
-
-	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
+	b, err := xattr.Get(objPath, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -548,57 +415,19 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
 		}
 	}
 
-	contentLength := fi.Size()
-
-	var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
-	status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
-	if err == nil {
-		if *status {
-			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
-		} else {
-			objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
-		}
-	}
-
-	var objectLockMode types.ObjectLockMode
-	var objectLockRetainUntilDate *time.Time
-	retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
-	if err == nil {
-		var config types.ObjectLockRetention
-		if err := json.Unmarshal(retention, &config); err == nil {
-			objectLockMode = types.ObjectLockMode(config.Mode)
-			objectLockRetainUntilDate = config.RetainUntilDate
-		}
-	}
-
 	return &s3.HeadObjectOutput{
-		ContentLength:             &contentLength,
-		ContentType:               &contentType,
-		ContentEncoding:           &contentEncoding,
-		ETag:                      &etag,
-		LastModified:              backend.GetTimePtr(fi.ModTime()),
-		Metadata:                  userMetaData,
-		StorageClass:              stclass,
-		Restore:                   &requestOngoing,
-		ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
-		ObjectLockMode:            objectLockMode,
-		ObjectLockRetainUntilDate: objectLockRetainUntilDate,
+		ContentLength:   fi.Size(),
+		ContentType:     &contentType,
+		ContentEncoding: &contentEncoding,
+		ETag:            &etag,
+		LastModified:    backend.GetTimePtr(fi.ModTime()),
+		Metadata:        userMetaData,
+		StorageClass:    stclass,
+		Restore:         &requestOngoing,
 	}, nil
 }
 
-func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
-	sum := sha256.Sum256([]byte(object))
-	objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
-
-	entries, err := os.ReadDir(objdir)
-	if err != nil || len(entries) == 0 {
-		return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
-	return entries[0].Name(), sum, nil
-}
-
-func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
 	bucket := *input.Bucket
 	object := *input.Key
 	acceptRange := *input.Range
@@ -612,34 +441,19 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 	}
 
 	objPath := filepath.Join(bucket, object)
-
 	fi, err := os.Stat(objPath)
-	if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
+	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 	}
-	if errors.Is(err, syscall.ENAMETOOLONG) {
-		return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
-	}
 	if err != nil {
 		return nil, fmt.Errorf("stat object: %w", err)
 	}
 
-	if strings.HasSuffix(object, "/") && !fi.IsDir() {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
-	startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
+	startOffset, length, err := backend.ParseRange(fi, acceptRange)
 	if err != nil {
 		return nil, err
 	}
 
-	objSize := fi.Size()
-	if fi.IsDir() {
-		// directory objects are always 0 len
-		objSize = 0
-		length = 0
-	}
-
 	if length == -1 {
 		length = fi.Size() - startOffset + 1
 	}
@@ -648,11 +462,6 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
-	var contentRange string
-	if acceptRange != "" {
-		contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
-	}
-
 	if s.glaciermode {
 		// Check if there are any offline exents associated with this file.
 		// If so, we will return the InvalidObjectState error.
@@ -675,14 +484,19 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 	if err != nil {
 		return nil, fmt.Errorf("open object: %w", err)
 	}
+	defer f.Close()
 
 	rdr := io.NewSectionReader(f, startOffset, length)
+	_, err = io.Copy(writer, rdr)
+	if err != nil {
+		return nil, fmt.Errorf("copy data: %w", err)
+	}
 
 	userMetaData := make(map[string]string)
 
-	contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
+	contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
 
-	b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
+	b, err := xattr.Get(objPath, etagkey)
 	etag := string(b)
 	if err != nil {
 		etag = ""
@@ -693,20 +507,16 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
 		return nil, fmt.Errorf("get object tags: %w", err)
 	}
 
-	tagCount := int32(len(tags))
-
 	return &s3.GetObjectOutput{
 		AcceptRanges:    &acceptRange,
-		ContentLength:   &length,
+		ContentLength:   length,
 		ContentEncoding: &contentEncoding,
 		ContentType:     &contentType,
 		ETag:            &etag,
 		LastModified:    backend.GetTimePtr(fi.ModTime()),
 		Metadata:        userMetaData,
-		TagCount:        &tagCount,
+		TagCount:        int32(len(tags)),
 		StorageClass:    types.StorageClassStandard,
-		ContentRange:    &contentRange,
-		Body:            &backend.FileSectionReadCloser{R: rdr, F: f},
 	}, nil
 }
 
@@ -731,100 +541,70 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
 	return tags, nil
 }
 
-func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
-	if input.Bucket == nil {
-		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
+func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
 	bucket := *input.Bucket
-	prefix := ""
-	if input.Prefix != nil {
-		prefix = *input.Prefix
-	}
-	marker := ""
-	if input.Marker != nil {
-		marker = *input.Marker
-	}
-	delim := ""
-	if input.Delimiter != nil {
-		delim = *input.Delimiter
-	}
-	maxkeys := int32(0)
-	if input.MaxKeys != nil {
-		maxkeys = *input.MaxKeys
-	}
+	prefix := *input.Prefix
+	marker := *input.Marker
+	delim := *input.Delimiter
+	maxkeys := input.MaxKeys
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
 	if err != nil {
-		return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err)
+		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
 		s.fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
-		return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
+		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
-	return s3response.ListObjectsResult{
+	return &s3.ListObjectsOutput{
 		CommonPrefixes: results.CommonPrefixes,
 		Contents:       results.Objects,
 		Delimiter:      &delim,
-		IsTruncated:    &results.Truncated,
+		IsTruncated:    results.Truncated,
 		Marker:         &marker,
-		MaxKeys:        &maxkeys,
+		MaxKeys:        maxkeys,
 		Name:           &bucket,
 		NextMarker:     &results.NextMarker,
 		Prefix:         &prefix,
 	}, nil
 }
 
-func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
-	if input.Bucket == nil {
-		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
+func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
 	bucket := *input.Bucket
-	prefix := ""
-	if input.Prefix != nil {
-		prefix = *input.Prefix
-	}
-	marker := ""
-	if input.ContinuationToken != nil {
-		marker = *input.ContinuationToken
-	}
-	delim := ""
-	if input.Delimiter != nil {
-		delim = *input.Delimiter
-	}
-	maxkeys := int32(0)
-	if input.MaxKeys != nil {
-		maxkeys = *input.MaxKeys
-	}
+	prefix := *input.Prefix
+	marker := *input.ContinuationToken
+	delim := *input.Delimiter
+	maxkeys := input.MaxKeys
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
-		return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
+		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
 	if err != nil {
-		return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err)
+		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys),
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
 		s.fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
-		return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
+		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
-	return s3response.ListObjectsV2Result{
+	return &s3.ListObjectsV2Output{
 		CommonPrefixes:        results.CommonPrefixes,
 		Contents:              results.Objects,
 		Delimiter:             &delim,
-		IsTruncated:           &results.Truncated,
+		IsTruncated:           results.Truncated,
 		ContinuationToken:     &marker,
-		MaxKeys:               &maxkeys,
+		MaxKeys:               int32(maxkeys),
 		Name:                  &bucket,
 		NextContinuationToken: &results.NextMarker,
 		Prefix:                &prefix,
@@ -832,58 +612,53 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
 }
 
 func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
-	return func(path string, d fs.DirEntry) (s3response.Object, error) {
+	return func(path string, d fs.DirEntry) (types.Object, error) {
 		objPath := filepath.Join(bucket, path)
 		if d.IsDir() {
 			// directory object only happens if directory empty
 			// check to see if this is a directory object by checking etag
-			etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
-			if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
-				return s3response.Object{}, backend.ErrSkipObj
+			etagBytes, err := xattr.Get(objPath, etagkey)
+			if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return s3response.Object{}, fmt.Errorf("get etag: %w", err)
+				return types.Object{}, fmt.Errorf("get etag: %w", err)
 			}
 			etag := string(etagBytes)
 
 			fi, err := d.Info()
 			if errors.Is(err, fs.ErrNotExist) {
-				return s3response.Object{}, backend.ErrSkipObj
+				return types.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
+				return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 			}
 
 			key := path + "/"
-			mtime := fi.ModTime()
 
-			return s3response.Object{
+			return types.Object{
 				ETag:         &etag,
 				Key:          &key,
-				LastModified: &mtime,
-				StorageClass: types.ObjectStorageClassStandard,
+				LastModified: backend.GetTimePtr(fi.ModTime()),
 			}, nil
 		}
 
 		// file object, get object info and fill out object data
-		b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
+		etagBytes, err := xattr.Get(objPath, etagkey)
 		if errors.Is(err, fs.ErrNotExist) {
-			return s3response.Object{}, backend.ErrSkipObj
+			return types.Object{}, backend.ErrSkipObj
 		}
-		if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
-			return s3response.Object{}, fmt.Errorf("get etag: %w", err)
+		if err != nil && !isNoAttr(err) {
+			return types.Object{}, fmt.Errorf("get etag: %w", err)
 		}
-		// note: meta.ErrNoSuchKey will return etagBytes = []byte{}
-		// so this will just set etag to "" if its not already set
-
-		etag := string(b)
+		etag := string(etagBytes)
 
 		fi, err := d.Info()
 		if errors.Is(err, fs.ErrNotExist) {
-			return s3response.Object{}, backend.ErrSkipObj
+			return types.Object{}, backend.ErrSkipObj
 		}
 		if err != nil {
-			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
 
 		sc := types.ObjectStorageClassStandard
@@ -892,24 +667,21 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 			// If so, we will return the InvalidObjectState error.
 			st, err := statMore(objPath)
 			if errors.Is(err, fs.ErrNotExist) {
-				return s3response.Object{}, backend.ErrSkipObj
+				return types.Object{}, backend.ErrSkipObj
 			}
 			if err != nil {
-				return s3response.Object{}, fmt.Errorf("stat more: %w", err)
+				return types.Object{}, fmt.Errorf("stat more: %w", err)
 			}
 			if st.Offline_blocks != 0 {
 				sc = types.ObjectStorageClassGlacier
 			}
 		}
 
-		size := fi.Size()
-		mtime := fi.ModTime()
-
-		return s3response.Object{
+		return types.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: &mtime,
-			Size:         &size,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
+			Size:         fi.Size(),
 			StorageClass: sc,
 		}, nil
 	}
@@ -991,9 +763,15 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 }
 
 func isNoAttr(err error) bool {
+	if err == nil {
+		return false
+	}
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
+	if err == syscall.ENODATA {
+		return true
+	}
 	return false
 }

backend/scoutfs/scoutfs_compat.go

@@ -17,31 +17,23 @@
 package scoutfs
 
 import (
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 
 	"golang.org/x/sys/unix"
 
 	"github.com/versity/scoutfs-go"
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
-func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
-	metastore := meta.XattrMeta{}
-
-	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
-		ChownUID:    opts.ChownUID,
-		ChownGID:    opts.ChownGID,
-		BucketLinks: opts.BucketLinks,
-		NewDirPerm:  opts.NewDirPerm,
-	})
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+	p, err := posix.New(rootdir)
 	if err != nil {
 		return nil, err
 	}
@@ -51,72 +43,71 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
-	return &ScoutFS{
-		Posix:       p,
-		rootfd:      f,
-		rootdir:     rootdir,
-		meta:        metastore,
-		chownuid:    opts.ChownUID,
-		chowngid:    opts.ChownGID,
-		glaciermode: opts.GlacierMode,
-		newDirPerm:  opts.NewDirPerm,
-	}, nil
+	s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
+	for _, opt := range opts {
+		opt(s)
+	}
+
+	return s, nil
 }
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f          *os.File
-	bucket     string
-	objname    string
-	size       int64
-	needsChown bool
-	uid        int
-	gid        int
-	newDirPerm fs.FileMode
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
 }
 
-var (
-	defaultFilePerm uint32 = 0644
-)
-
-func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
-	uid, gid, doChown := s.getChownIDs(acct)
-
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
 	if err != nil {
-		return nil, err
+		// O_TMPFILE not supported, try fallback
+		err := os.MkdirAll(dir, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("make temp dir: %w", err)
+		}
+		f, err := os.CreateTemp(dir,
+			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+		if err != nil {
+			return nil, err
+		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		// falloc is best effort, its fine if this fails
+		if size > 0 {
+			tmp.falloc()
+		}
+		return tmp, nil
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{
-		f:          f,
-		bucket:     bucket,
-		objname:    obj,
-		size:       size,
-		needsChown: doChown,
-		uid:        uid,
-		gid:        gid,
-		newDirPerm: s.newDirPerm,
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	// falloc is best effort, its fine if this fails
+	if size > 0 {
+		tmp.falloc()
 	}
-
-	if doChown {
-		err := f.Chown(uid, gid)
-		if err != nil {
-			return nil, fmt.Errorf("set temp file ownership: %w", err)
-		}
-	}
-
 	return tmp, nil
 }
 
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
 func (tmp *tmpfile) link() error {
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
@@ -130,11 +121,9 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	dir := filepath.Dir(objPath)
-
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
-	if err != nil {
-		return fmt.Errorf("make parent dir: %w", err)
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
 	}
 
 	procdir, err := os.Open(procfddir)
@@ -143,14 +132,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dirf, err := os.Open(dir)
+	dir, err := os.Open(filepath.Dir(objPath))
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dirf.Close()
+	defer dir.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile: %w", err)
 	}
@@ -163,6 +152,26 @@ func (tmp *tmpfile) link() error {
 	return nil
 }
 
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
 func (tmp *tmpfile) Write(b []byte) (int, error) {
 	if int64(len(b)) > tmp.size {
 		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
@@ -177,10 +186,6 @@ func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
 
-func (tmp *tmpfile) File() *os.File {
-	return tmp.f
-}
-
 func moveData(from *os.File, to *os.File) error {
 	return scoutfs.MoveData(from, to)
 }

backend/scoutfs/scoutfs_incompat.go

@@ -20,26 +20,21 @@ import (
 	"errors"
 	"fmt"
 	"os"
-
-	"github.com/versity/versitygw/auth"
 )
 
-func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
-type tmpfile struct{}
+type tmpfile struct {
+	f *os.File
+}
 
 var (
 	errNotSupported = errors.New("not supported")
 )
 
-func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile, error) {
-	// make these look used for static check
-	_ = s.chownuid
-	_ = s.chowngid
-	_ = s.euid
-	_ = s.egid
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	return nil, errNotSupported
 }
 
@@ -54,14 +49,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
-func (tmp *tmpfile) File() *os.File {
-	return nil
-}
-
-func moveData(_, _ *os.File) error {
+func moveData(from *os.File, to *os.File) error {
 	return errNotSupported
 }
 
-func statMore(_ string) (stat, error) {
+func statMore(path string) (stat, error) {
 	return stat{}, errNotSupported
 }

backend/walk.go

@@ -15,34 +15,32 @@
 package backend
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"io/fs"
+	"os"
 	"sort"
 	"strings"
-	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/s3response"
 )
 
 type WalkResults struct {
 	CommonPrefixes []types.CommonPrefix
-	Objects        []s3response.Object
+	Objects        []types.Object
 	Truncated      bool
 	NextMarker     string
 }
 
-type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
+type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
 
 var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
-	var objects []s3response.Object
+	var objects []types.Object
 
 	var pastMarker bool
 	if marker == "" {
@@ -53,21 +51,10 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 	var newMarker string
 	var truncated bool
 
-	root := "."
-	if strings.Contains(prefix, "/") {
-		idx := strings.LastIndex(prefix, "/")
-		if idx > 0 {
-			root = prefix[:idx]
-		}
-	}
-
-	err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
 		// Ignore the root directory
 		if path == "." {
 			return nil
@@ -84,9 +71,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipAll
 		}
 
-		// After this point, return skipflag instead of nil
-		// so we can skip a directory without an early return
-		var skipflag error
 		if d.IsDir() {
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
@@ -96,57 +80,44 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// building to match. So only skip if path isn't a prefix of prefix
 			// and prefix isn't a prefix of path.
 			if prefix != "" &&
-				!strings.HasPrefix(path+"/", prefix) &&
-				!strings.HasPrefix(prefix, path+"/") {
+				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
+				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
 				return fs.SkipDir
 			}
 
-			// Don't recurse into subdirectories which contain the delimiter
-			// after reaching the prefix
-			if delimiter != "" &&
-				strings.HasPrefix(path+"/", prefix) &&
-				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
-				skipflag = fs.SkipDir
-			} else {
-				// TODO: can we do better here rather than a second readdir
-				// per directory?
-				ents, err := fs.ReadDir(fileSystem, path)
-				if err != nil {
-					return fmt.Errorf("readdir %q: %w", path, err)
-				}
-				if len(ents) == 0 && delimiter == "" {
-					dirobj, err := getObj(path+"/", d)
-					if err == ErrSkipObj {
-						return skipflag
-					}
-					if err != nil {
-						return fmt.Errorf("directory to object %q: %w", path, err)
-					}
-					objects = append(objects, dirobj)
-
-					return skipflag
-				}
-
-				if len(ents) != 0 {
-					return skipflag
-				}
+			// TODO: can we do better here rather than a second readdir
+			// per directory?
+			ents, err := fs.ReadDir(fileSystem, path)
+			if err != nil {
+				return fmt.Errorf("readdir %q: %w", path, err)
 			}
-			path += "/"
+			if len(ents) == 0 {
+				dirobj, err := getObj(path, d)
+				if err == ErrSkipObj {
+					return nil
+				}
+				if err != nil {
+					return fmt.Errorf("directory to object %q: %w", path, err)
+				}
+				objects = append(objects, dirobj)
+			}
+
+			return nil
 		}
 
 		if !pastMarker {
 			if path == marker {
 				pastMarker = true
-				return skipflag
+				return nil
 			}
 			if path < marker {
-				return skipflag
+				return nil
 			}
 		}
 
 		// If object doesn't have prefix, don't include in results.
 		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return skipflag
+			return nil
 		}
 
 		if delimiter == "" {
@@ -154,7 +125,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// prefix are included in results
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return skipflag
+				return nil
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -165,7 +136,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 				pastMax = true
 			}
 
-			return skipflag
+			return nil
 		}
 
 		// Since delimiter is specified, we only want results that
@@ -194,7 +165,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		if !found {
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return skipflag
+				return nil
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
@@ -203,38 +174,20 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			if (len(objects) + len(cpmap)) == int(max) {
 				pastMax = true
 			}
-			return skipflag
+			return nil
 		}
 
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
-		// delimiter at the end when we add to the map.
-		cprefNoDelim := prefix + before
-		cpref := prefix + before + delimiter
-		if cpref == marker {
-			pastMarker = true
-			return skipflag
-		}
-
-		if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
-			// skip common prefixes that are before the marker
-			return skipflag
-		}
-
-		cpmap[cpref] = struct{}{}
+		// delimiter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
 		if (len(objects) + len(cpmap)) == int(max) {
-			newMarker = cpref
-			truncated = true
-			return fs.SkipAll
+			pastMax = true
 		}
 
-		return skipflag
+		return nil
 	})
 	if err != nil {
-		// suppress file not found caused by user's prefix
-		if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
-			return WalkResults{}, nil
-		}
 		return WalkResults{}, err
 	}
 
@@ -267,213 +220,3 @@ func contains(a string, strs []string) bool {
 	}
 	return false
 }
-
-type WalkVersioningResults struct {
-	CommonPrefixes      []types.CommonPrefix
-	ObjectVersions      []types.ObjectVersion
-	DelMarkers          []types.DeleteMarkerEntry
-	Truncated           bool
-	NextMarker          string
-	NextVersionIdMarker string
-}
-
-type ObjVersionFuncResult struct {
-	ObjectVersions      []types.ObjectVersion
-	DelMarkers          []types.DeleteMarkerEntry
-	NextVersionIdMarker string
-	Truncated           bool
-}
-
-type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
-
-// WalkVersions walks the supplied fs.FS and returns results compatible with
-// ListObjectVersions action response
-func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
-	cpmap := make(map[string]struct{})
-	var objects []types.ObjectVersion
-	var delMarkers []types.DeleteMarkerEntry
-
-	var pastMarker bool
-	if keyMarker == "" {
-		pastMarker = true
-	}
-	var nextMarker string
-	var nextVersionIdMarker string
-	var truncated bool
-
-	pastVersionIdMarker := versionIdMarker == ""
-
-	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
-		// Ignore the root directory
-		if path == "." {
-			return nil
-		}
-		if contains(d.Name(), skipdirs) {
-			return fs.SkipDir
-		}
-
-		if !pastMarker {
-			if path == keyMarker {
-				pastMarker = true
-			}
-			if path < keyMarker {
-				return nil
-			}
-		}
-
-		if d.IsDir() {
-			// If prefix is defined and the directory does not match prefix,
-			// do not descend into the directory because nothing will
-			// match this prefix. Make sure to append the / at the end of
-			// directories since this is implied as a directory path name.
-			// If path is a prefix of prefix, then path could still be
-			// building to match. So only skip if path isn't a prefix of prefix
-			// and prefix isn't a prefix of path.
-			if prefix != "" &&
-				!strings.HasPrefix(path+"/", prefix) &&
-				!strings.HasPrefix(prefix, path+"/") {
-				return fs.SkipDir
-			}
-
-			// Don't recurse into subdirectories when listing with delimiter.
-			if delimiter == "/" &&
-				prefix != path+"/" &&
-				strings.HasPrefix(path+"/", prefix) {
-				cpmap[path+"/"] = struct{}{}
-				return fs.SkipDir
-			}
-
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
-			if err == ErrSkipObj {
-				return nil
-			}
-			if err != nil {
-				return fmt.Errorf("directory to object %q: %w", path, err)
-			}
-			objects = append(objects, res.ObjectVersions...)
-			delMarkers = append(delMarkers, res.DelMarkers...)
-			if res.Truncated {
-				truncated = true
-				nextMarker = path
-				nextVersionIdMarker = res.NextVersionIdMarker
-				return fs.SkipAll
-			}
-
-			return nil
-		}
-
-		// If object doesn't have prefix, don't include in results.
-		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return nil
-		}
-
-		if delimiter == "" {
-			// If no delimiter specified, then all files with matching
-			// prefix are included in results
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
-			if err == ErrSkipObj {
-				return nil
-			}
-			if err != nil {
-				return fmt.Errorf("file to object %q: %w", path, err)
-			}
-			objects = append(objects, res.ObjectVersions...)
-			delMarkers = append(delMarkers, res.DelMarkers...)
-			if res.Truncated {
-				truncated = true
-				nextMarker = path
-				nextVersionIdMarker = res.NextVersionIdMarker
-				return fs.SkipAll
-			}
-
-			return nil
-		}
-
-		// Since delimiter is specified, we only want results that
-		// do not contain the delimiter beyond the prefix.  If the
-		// delimiter exists past the prefix, then the substring
-		// between the prefix and delimiter is part of common prefixes.
-		//
-		// For example:
-		// prefix = A/
-		// delimiter = /
-		// and objects:
-		// A/file
-		// A/B/file
-		// B/C
-		// would return:
-		// objects: A/file
-		// common prefix: A/B/
-		//
-		// Note: No objects are included past the common prefix since
-		// these are all rolled up into the common prefix.
-		// Note: The delimiter can be anything, so we have to operate on
-		// the full path without any assumptions on posix directory hierarchy
-		// here.  Usually the delimiter will be "/", but thats not required.
-		suffix := strings.TrimPrefix(path, prefix)
-		before, _, found := strings.Cut(suffix, delimiter)
-		if !found {
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
-			if err == ErrSkipObj {
-				return nil
-			}
-			if err != nil {
-				return fmt.Errorf("file to object %q: %w", path, err)
-			}
-			objects = append(objects, res.ObjectVersions...)
-			delMarkers = append(delMarkers, res.DelMarkers...)
-
-			if res.Truncated {
-				truncated = true
-				nextMarker = path
-				nextVersionIdMarker = res.NextVersionIdMarker
-				return fs.SkipAll
-			}
-			return nil
-		}
-
-		// Common prefixes are a set, so should not have duplicates.
-		// These are abstractly a "directory", so need to include the
-		// delimiter at the end.
-		cpmap[prefix+before+delimiter] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
-			nextMarker = path
-			truncated = true
-
-			return fs.SkipAll
-		}
-
-		return nil
-	})
-	if err != nil {
-		return WalkVersioningResults{}, err
-	}
-
-	var commonPrefixStrings []string
-	for k := range cpmap {
-		commonPrefixStrings = append(commonPrefixStrings, k)
-	}
-	sort.Strings(commonPrefixStrings)
-	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
-	for _, cp := range commonPrefixStrings {
-		pfx := cp
-		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
-			Prefix: &pfx,
-		})
-	}
-
-	return WalkVersioningResults{
-		CommonPrefixes:      commonPrefixes,
-		ObjectVersions:      objects,
-		DelMarkers:          delMarkers,
-		Truncated:           truncated,
-		NextMarker:          nextMarker,
-		NextVersionIdMarker: nextVersionIdMarker,
-	}, nil
-}

backend/walk_test.go

@@ -15,50 +15,36 @@
 package backend_test
 
 import (
-	"context"
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
 	"io/fs"
-	"sync"
 	"testing"
 	"testing/fstest"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3response"
 )
 
 type walkTest struct {
-	fsys   fs.FS
-	getobj backend.GetObjFunc
-	cases  []testcase
+	fsys     fs.FS
+	expected backend.WalkResults
+	getobj   backend.GetObjFunc
 }
 
-type testcase struct {
-	name      string
-	prefix    string
-	delimiter string
-	marker    string
-	maxObjs   int32
-	expected  backend.WalkResults
-}
-
-func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
+func getObj(path string, d fs.DirEntry) (types.Object, error) {
 	if d.IsDir() {
 		etag := getMD5(path)
 
 		fi, err := d.Info()
 		if err != nil {
-			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
-		mtime := fi.ModTime()
 
-		return s3response.Object{
+		return types.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: &mtime,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
 		}, nil
 	}
 
@@ -66,17 +52,14 @@ func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
 
 	fi, err := d.Info()
 	if err != nil {
-		return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 	}
 
-	size := fi.Size()
-	mtime := fi.ModTime()
-
-	return s3response.Object{
+	return types.Object{
 		ETag:         &etag,
 		Key:          &path,
-		LastModified: &mtime,
-		Size:         &size,
+		LastModified: backend.GetTimePtr(fi.ModTime()),
+		Size:         fi.Size(),
 	}, nil
 }
 
@@ -97,154 +80,50 @@ func TestWalk(t *testing.T) {
 				"photos/2006/February/sample3.jpg": {},
 				"photos/2006/February/sample4.jpg": {},
 			},
-			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "aws example",
-					delimiter: "/",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photos/"),
-						}},
-						Objects: []s3response.Object{{
-							Key: backend.GetPtrFromString("sample.jpg"),
-						}},
-					},
-				},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("photos/"),
+				}},
+				Objects: []types.Object{{
+					Key: backend.GetStringPtr("sample.jpg"),
+				}},
 			},
+			getobj: getObj,
 		},
 		{
 			// test case single dir/single file
 			fsys: fstest.MapFS{
 				"test/file": {},
 			},
-			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "single dir single file",
-					delimiter: "/",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("test/"),
-						}},
-						Objects: []s3response.Object{},
-					},
-				},
-			},
-		},
-		{
-			// non-standard delimiter
-			fsys: fstest.MapFS{
-				"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
-				"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
-				"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("test/"),
+				}},
+				Objects: []types.Object{},
 			},
 			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "different delimiter 1",
-					delimiter: "|",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 2",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 3",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 4",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 5",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|6/Januar|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 6",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|6/Januar|y/sampl|",
-					expected: backend.WalkResults{
-						Objects: []s3response.Object{
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
-							},
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
-							},
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
-							},
-						},
-					},
-				},
-			},
 		},
 	}
 
 	for _, tt := range tests {
-		for _, tc := range tt.cases {
-			res, err := backend.Walk(context.Background(),
-				tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
-				tt.getobj, []string{})
-			if err != nil {
-				t.Errorf("tc.name: walk: %v", err)
-			}
-
-			compareResults(tc.name, res, tc.expected, t)
+		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
+		if err != nil {
+			t.Fatalf("walk: %v", err)
 		}
+
+		compareResults(res, tt.expected, t)
 	}
 }
 
-func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
+func compareResults(got, wanted backend.WalkResults, t *testing.T) {
 	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
-		t.Errorf("%v: unexpected common prefix, got %v wanted %v",
-			name,
+		t.Errorf("unexpected common prefix, got %v wanted %v",
 			printCommonPrefixes(got.CommonPrefixes),
 			printCommonPrefixes(wanted.CommonPrefixes))
 	}
 
 	if !compareObjects(got.Objects, wanted.Objects) {
-		t.Errorf("%v: unexpected object, got %v wanted %v",
-			name,
+		t.Errorf("unexpected object, got %v wanted %v",
 			printObjects(got.Objects),
 			printObjects(wanted.Objects))
 	}
@@ -287,7 +166,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
 	return res + "]"
 }
 
-func compareObjects(a, b []s3response.Object) bool {
+func compareObjects(a, b []types.Object) bool {
 	if len(a) == 0 && len(b) == 0 {
 		return true
 	}
@@ -303,7 +182,7 @@ func compareObjects(a, b []s3response.Object) bool {
 	return false
 }
 
-func containsObject(c s3response.Object, list []s3response.Object) bool {
+func containsObject(c types.Object, list []types.Object) bool {
 	for _, cp := range list {
 		if *c.Key == *cp.Key {
 			return true
@@ -312,67 +191,14 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
 	return false
 }
 
-func printObjects(list []s3response.Object) string {
+func printObjects(list []types.Object) string {
 	res := "["
 	for _, cp := range list {
-		var key string
-		if cp.Key == nil {
-			key = "<nil>"
-		} else {
-			key = *cp.Key
-		}
 		if res == "[" {
-			res = res + key
+			res = res + *cp.Key
 		} else {
-			res = res + ", " + key
+			res = res + ", " + *cp.Key
 		}
 	}
 	return res + "]"
 }
-
-type slowFS struct {
-	fstest.MapFS
-}
-
-const (
-	readDirPause = 100 * time.Millisecond
-
-	// walkTimeOut should be less than the tree traversal time
-	// which is the readdirPause time * the number of directories
-	walkTimeOut = 500 * time.Millisecond
-)
-
-func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
-	time.Sleep(readDirPause)
-	return s.MapFS.ReadDir(name)
-}
-
-func TestWalkStop(t *testing.T) {
-	s := &slowFS{MapFS: fstest.MapFS{
-		"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
-	}}
-
-	ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
-	defer cancel()
-
-	var err error
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		_, err = backend.Walk(ctx, s, "", "/", "", 1000,
-			func(path string, d fs.DirEntry) (s3response.Object, error) {
-				return s3response.Object{}, nil
-			}, []string{})
-	}()
-
-	select {
-	case <-time.After(1 * time.Second):
-		t.Fatalf("walk is not terminated in time")
-	case <-ctx.Done():
-	}
-	wg.Wait()
-	if err != ctx.Err() {
-		t.Fatalf("unexpected error: %v", err)
-	}
-}

cmd/versitygw/admin.go

@@ -17,9 +17,8 @@ package main
 import (
 	"bytes"
 	"crypto/sha256"
-	"crypto/tls"
 	"encoding/hex"
-	"encoding/xml"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,7 +28,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/aws/smithy-go"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3response"
@@ -38,9 +36,7 @@ import (
 var (
 	adminAccess   string
 	adminSecret   string
-	adminRegion   string
 	adminEndpoint string
-	allowInsecure bool
 )
 
 func adminCommand() *cli.Command {
@@ -82,33 +78,10 @@ func adminCommand() *cli.Command {
 						Usage:   "groupID for the new user",
 						Aliases: []string{"gi"},
 					},
-				},
-			},
-			{
-				Name:   "update-user",
-				Usage:  "Updates a user account",
-				Action: updateUser,
-				Flags: []cli.Flag{
-					&cli.StringFlag{
-						Name:     "access",
-						Usage:    "user access key id to be updated",
-						Required: true,
-						Aliases:  []string{"a"},
-					},
-					&cli.StringFlag{
-						Name:    "secret",
-						Usage:   "secret access key for the new user",
-						Aliases: []string{"s"},
-					},
 					&cli.IntFlag{
-						Name:    "user-id",
-						Usage:   "userID for the new user",
-						Aliases: []string{"ui"},
-					},
-					&cli.IntFlag{
-						Name:    "group-id",
-						Usage:   "groupID for the new user",
-						Aliases: []string{"gi"},
+						Name:    "project-id",
+						Usage:   "projectID for the new user",
+						Aliases: []string{"pi"},
 					},
 				},
 			},
@@ -173,14 +146,6 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminSecret,
 			},
-			&cli.StringFlag{
-				Name:        "region",
-				Usage:       "admin s3 region string",
-				EnvVars:     []string{"ADMIN_REGION"},
-				Value:       "us-east-1",
-				Destination: &adminRegion,
-				Aliases:     []string{"r"},
-			},
 			&cli.StringFlag{
 				Name:        "endpoint-url",
 				Usage:       "admin apis endpoint url",
@@ -189,80 +154,65 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminEndpoint,
 			},
-			&cli.BoolFlag{
-				Name:        "allow-insecure",
-				Usage:       "disable tls certificate verification for the admin endpoint",
-				EnvVars:     []string{"ADMIN_ALLOW_INSECURE"},
-				Aliases:     []string{"ai"},
-				Destination: &allowInsecure,
-			},
 		},
 	}
 }
 
-func initHTTPClient() *http.Client {
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
-	}
-	return &http.Client{Transport: tr}
-}
-
 func createUser(ctx *cli.Context) error {
 	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
-	userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
+	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
 	if access == "" || secret == "" {
-		return fmt.Errorf("invalid input parameters for the new user access/secret keys")
+		return fmt.Errorf("invalid input parameters for the new user")
 	}
-	if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
-		return fmt.Errorf("invalid input parameter for role: %v", role)
+	if role != "admin" && role != "user" {
+		return fmt.Errorf("invalid input parameter for role")
 	}
 
 	acc := auth.Account{
-		Access:  access,
-		Secret:  secret,
-		Role:    auth.Role(role),
-		UserID:  userID,
-		GroupID: groupID,
+		Access:    access,
+		Secret:    secret,
+		Role:      role,
+		UserID:    userID,
+		GroupID:   groupID,
+		ProjectID: projectID,
 	}
 
-	accxml, err := xml.Marshal(acc)
+	accJson, err := json.Marshal(acc)
 	if err != nil {
 		return fmt.Errorf("failed to parse user data: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(accxml)
+	hashedPayload := sha256.Sum256(accJson)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
+	fmt.Printf("%s\n", body)
 
 	return nil
 }
@@ -270,7 +220,7 @@ func createUser(ctx *cli.Context) error {
 func deleteUser(ctx *cli.Context) error {
 	access := ctx.String("access")
 	if access == "" {
-		return fmt.Errorf("invalid input parameter for the user access key")
+		return fmt.Errorf("invalid input parameter for the new user")
 	}
 
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
@@ -285,82 +235,25 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
-
-	return nil
-}
-
-func updateUser(ctx *cli.Context) error {
-	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
-	props := auth.MutableProps{}
-	if ctx.IsSet("secret") {
-		props.Secret = &secret
-	}
-	if ctx.IsSet("user-id") {
-		props.UserID = &userId
-	}
-	if ctx.IsSet("group-id") {
-		props.GroupID = &groupId
-	}
-
-	propsxml, err := xml.Marshal(props)
-	if err != nil {
-		return fmt.Errorf("failed to parse user attributes: %w", err)
-	}
-
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
-	if err != nil {
-		return fmt.Errorf("failed to send the request: %w", err)
-	}
-
-	signer := v4.NewSigner()
-
-	hashedPayload := sha256.Sum256(propsxml)
-	hexPayload := hex.EncodeToString(hashedPayload[:])
-
-	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
-
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
-	if signErr != nil {
-		return fmt.Errorf("failed to sign the request: %w", err)
-	}
-
-	client := initHTTPClient()
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to send the request: %w", err)
-	}
 	defer resp.Body.Close()
 
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
+	fmt.Printf("%s\n", body)
 
 	return nil
 }
@@ -378,34 +271,31 @@ func listUsers(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
-
-	var accs auth.ListUserAccountsResult
-	if err := xml.Unmarshal(body, &accs); err != nil {
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
 		return err
 	}
+	fmt.Println(accs)
 
-	printAcctTable(accs.Accounts)
+	printAcctTable(accs)
 
 	return nil
 }
@@ -422,10 +312,10 @@ const (
 func printAcctTable(accs []auth.Account) {
 	w := new(tabwriter.Writer)
 	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
-	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
-	fmt.Fprintln(w, "-------\t----\t------\t-------")
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
 	for _, acc := range accs {
-		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
 	}
 	fmt.Fprintln(w)
 	w.Flush()
@@ -445,27 +335,25 @@ func changeBucketOwner(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
+	fmt.Println(string(body))
 
 	return nil
 }
@@ -495,45 +383,34 @@ func listBuckets(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf(string(body))
 	}
 
-	var result s3response.ListBucketsResult
-	if err := xml.Unmarshal(body, &result); err != nil {
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
 		return err
 	}
 
-	printBuckets(result.Buckets)
+	printBuckets(buckets)
 
 	return nil
 }
-
-func parseApiError(body []byte) error {
-	var apiErr smithy.GenericAPIError
-	err := xml.Unmarshal(body, &apiErr)
-	if err != nil {
-		apiErr.Code = "InternalServerError"
-		apiErr.Message = err.Error()
-	}
-
-	return &apiErr
-}

cmd/versitygw/azure.go

@@ -1,74 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package main
-
-import (
-	"fmt"
-
-	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/backend/azure"
-)
-
-var (
-	azAccount, azKey, azServiceURL, azSASToken string
-)
-
-func azureCommand() *cli.Command {
-	return &cli.Command{
-		Name:        "azure",
-		Usage:       "azure blob storage backend",
-		Description: `direct translation from s3 objects to azure blobs`,
-		Action:      runAzure,
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:        "account",
-				Usage:       "azure account name",
-				EnvVars:     []string{"AZ_ACCOUNT_NAME"},
-				Aliases:     []string{"a"},
-				Destination: &azAccount,
-			},
-			&cli.StringFlag{
-				Name:        "access-key",
-				Usage:       "azure account key",
-				EnvVars:     []string{"AZ_ACCESS_KEY"},
-				Aliases:     []string{"k"},
-				Destination: &azKey,
-			},
-			&cli.StringFlag{
-				Name:        "sas-token",
-				Usage:       "azure blob storage SAS token",
-				EnvVars:     []string{"AZ_SAS_TOKEN"},
-				Aliases:     []string{"st"},
-				Destination: &azSASToken,
-			},
-			&cli.StringFlag{
-				Name:        "url",
-				Usage:       "azure service URL",
-				EnvVars:     []string{"AZ_ENDPOINT"},
-				Aliases:     []string{"u"},
-				Destination: &azServiceURL,
-			},
-		},
-	}
-}
-
-func runAzure(ctx *cli.Context) error {
-	be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
-	if err != nil {
-		return fmt.Errorf("init azure: %w", err)
-	}
-
-	return runGateway(ctx.Context, be)
-}

cmd/versitygw/gateway_test.go

@@ -1,111 +0,0 @@
-package main
-
-import (
-	"context"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/versity/versitygw/backend/meta"
-	"github.com/versity/versitygw/backend/posix"
-	"github.com/versity/versitygw/tests/integration"
-)
-
-const (
-	tdir = "tempdir"
-)
-
-var (
-	wg sync.WaitGroup
-)
-
-func initEnv(dir string) {
-	// both
-	debug = true
-	region = "us-east-1"
-
-	// server
-	rootUserAccess = "user"
-	rootUserSecret = "pass"
-	iamDir = dir
-	port = "127.0.0.1:7070"
-
-	// client
-	awsID = "user"
-	awsSecret = "pass"
-	endpoint = "http://127.0.0.1:7070"
-}
-
-func initPosix(ctx context.Context) {
-	path, err := os.Getwd()
-	if err != nil {
-		log.Fatalf("get current directory: %v", err)
-	}
-
-	tempdir := filepath.Join(path, tdir)
-	initEnv(tempdir)
-
-	err = os.RemoveAll(tempdir)
-	if err != nil {
-		log.Fatalf("remove temp directory: %v", err)
-	}
-
-	err = os.Mkdir(tempdir, 0755)
-	if err != nil {
-		log.Fatalf("make temp directory: %v", err)
-	}
-
-	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
-		NewDirPerm: 0755,
-	})
-	if err != nil {
-		log.Fatalf("init posix: %v", err)
-	}
-
-	wg.Add(1)
-	go func() {
-		err = runGateway(ctx, be)
-		if err != nil && err != context.Canceled {
-			log.Fatalf("run gateway: %v", err)
-		}
-
-		err := os.RemoveAll(tempdir)
-		if err != nil {
-			log.Fatalf("remove temp directory: %v", err)
-		}
-		wg.Done()
-	}()
-
-	// wait for server to start
-	time.Sleep(1 * time.Second)
-}
-
-func TestIntegration(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-
-	initPosix(ctx)
-
-	opts := []integration.Option{
-		integration.WithAccess(awsID),
-		integration.WithSecret(awsSecret),
-		integration.WithRegion(region),
-		integration.WithEndpoint(endpoint),
-	}
-	if debug {
-		opts = append(opts, integration.WithDebug())
-	}
-
-	s := integration.NewS3Conf(opts...)
-
-	// replace below with desired test
-	err := integration.HeadBucket_non_existing_bucket(s)
-	if err != nil {
-		t.Error(err)
-	}
-
-	cancel()
-	wg.Wait()
-}

cmd/versitygw/main.go

@@ -19,17 +19,13 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
-	"net"
-	"net/http"
-	_ "net/http/pprof"
 	"os"
-	"strings"
+	"strconv"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api"
 	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
@@ -37,43 +33,24 @@ import (
 )
 
 var (
-	port, admPort                            string
-	rootUserAccess                           string
-	rootUserSecret                           string
-	region                                   string
-	admCertFile, admKeyFile                  string
-	certFile, keyFile                        string
-	kafkaURL, kafkaTopic, kafkaKey           string
-	natsURL, natsTopic                       string
-	eventWebhookURL                          string
-	eventConfigFilePath                      string
-	logWebhookURL, accessLog                 string
-	adminLogFile                             string
-	healthPath                               string
-	debug                                    bool
-	pprof                                    string
-	quiet                                    bool
-	readonly                                 bool
-	iamDir                                   string
-	ldapURL, ldapBindDN, ldapPassword        string
-	ldapQueryBase, ldapObjClasses            string
-	ldapAccessAtr, ldapSecAtr, ldapRoleAtr   string
-	ldapUserIdAtr, ldapGroupIdAtr            string
-	vaultEndpointURL, vaultSecretStoragePath string
-	vaultMountPath, vaultRootToken           string
-	vaultRoleId, vaultRoleSecret             string
-	vaultServerCert, vaultClientCert         string
-	vaultClientCertKey                       string
-	s3IamAccess, s3IamSecret                 string
-	s3IamRegion, s3IamBucket                 string
-	s3IamEndpoint                            string
-	s3IamSslNoVerify, s3IamDebug             bool
-	iamCacheDisable                          bool
-	iamCacheTTL                              int
-	iamCachePrune                            int
-	metricsService                           string
-	statsdServers                            string
-	dogstatsServers                          string
+	port, admPort                          string
+	rootUserAccess                         string
+	rootUserSecret                         string
+	region                                 string
+	admCertFile, admKeyFile                string
+	certFile, keyFile                      string
+	kafkaURL, kafkaTopic, kafkaKey         string
+	natsURL, natsTopic                     string
+	logWebhookURL                          string
+	accessLog                              string
+	debug                                  bool
+	iamDir                                 string
+	ldapURL, ldapBindDN, ldapPassword      string
+	ldapQueryBase, ldapObjClasses          string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	iamCacheDisable                        bool
+	iamCacheTTL                            int
+	iamCachePrune                          int
 )
 
 var (
@@ -94,10 +71,8 @@ func main() {
 		posixCommand(),
 		scoutfsCommand(),
 		s3Command(),
-		azureCommand(),
 		adminCommand(),
 		testCommand(),
-		utilsCommand(),
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -114,13 +89,10 @@ func main() {
 
 func initApp() *cli.App {
 	return &cli.App{
-		Usage: "Versity S3 Gateway",
-		Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
-to access the supported backend storage as if it was a native S3 service.
-VersityGW is an open-source project licensed under the Apache 2.0 License. The
-source code is hosted on GitHub at https://github.com/versity/versitygw, and
-documentation can be found in the GitHub wiki.`,
-		Copyright: "Copyright (c) 2023-2024 Versity Software",
+		Name:  "versitygw",
+		Usage: "Start S3 gateway service with specified backend storage.",
+		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.`,
 		Action: func(ctx *cli.Context) error {
 			return ctx.App.Command("help").Run(ctx)
 		},
@@ -145,7 +117,6 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "port",
 			Usage:       "gateway listen address <ip>:<port> or :<port>",
-			EnvVars:     []string{"VGW_PORT"},
 			Value:       ":7070",
 			Destination: &port,
 			Aliases:     []string{"p"},
@@ -167,7 +138,6 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "region",
 			Usage:       "s3 region string",
-			EnvVars:     []string{"VGW_REGION"},
 			Value:       "us-east-1",
 			Destination: &region,
 			Aliases:     []string{"r"},
@@ -175,360 +145,153 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "cert",
 			Usage:       "TLS cert file",
-			EnvVars:     []string{"VGW_CERT"},
 			Destination: &certFile,
 		},
 		&cli.StringFlag{
 			Name:        "key",
 			Usage:       "TLS key file",
-			EnvVars:     []string{"VGW_KEY"},
 			Destination: &keyFile,
 		},
 		&cli.StringFlag{
 			Name:        "admin-port",
 			Usage:       "gateway admin server listen address <ip>:<port> or :<port>",
-			EnvVars:     []string{"VGW_ADMIN_PORT"},
 			Destination: &admPort,
 			Aliases:     []string{"ap"},
 		},
 		&cli.StringFlag{
 			Name:        "admin-cert",
 			Usage:       "TLS cert file for admin server",
-			EnvVars:     []string{"VGW_ADMIN_CERT"},
 			Destination: &admCertFile,
 		},
 		&cli.StringFlag{
 			Name:        "admin-cert-key",
 			Usage:       "TLS key file for admin server",
-			EnvVars:     []string{"VGW_ADMIN_CERT_KEY"},
 			Destination: &admKeyFile,
 		},
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
-			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
-		&cli.StringFlag{
-			Name:        "pprof",
-			Usage:       "enable pprof debug on specified port",
-			EnvVars:     []string{"VGW_PPROF"},
-			Destination: &pprof,
-		},
-		&cli.BoolFlag{
-			Name:        "quiet",
-			Usage:       "silence stdout request logging output",
-			EnvVars:     []string{"VGW_QUIET"},
-			Destination: &quiet,
-			Aliases:     []string{"q"},
-		},
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
-			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
+			EnvVars:     []string{"LOGFILE"},
 			Destination: &accessLog,
 		},
-		&cli.StringFlag{
-			Name:        "admin-access-log",
-			Usage:       "enable admin server access logging to specified file",
-			EnvVars:     []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
-			Destination: &adminLogFile,
-		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
-			EnvVars:     []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"},
+			EnvVars:     []string{"WEBHOOK"},
 			Destination: &logWebhookURL,
 		},
 		&cli.StringFlag{
 			Name:        "event-kafka-url",
 			Usage:       "kafka server url to send the bucket notifications.",
-			EnvVars:     []string{"VGW_EVENT_KAFKA_URL"},
 			Destination: &kafkaURL,
 			Aliases:     []string{"eku"},
 		},
 		&cli.StringFlag{
 			Name:        "event-kafka-topic",
 			Usage:       "kafka server pub-sub topic to send the bucket notifications to",
-			EnvVars:     []string{"VGW_EVENT_KAFKA_TOPIC"},
 			Destination: &kafkaTopic,
 			Aliases:     []string{"ekt"},
 		},
 		&cli.StringFlag{
 			Name:        "event-kafka-key",
 			Usage:       "kafka server put-sub topic key to send the bucket notifications to",
-			EnvVars:     []string{"VGW_EVENT_KAFKA_KEY"},
 			Destination: &kafkaKey,
 			Aliases:     []string{"ekk"},
 		},
 		&cli.StringFlag{
 			Name:        "event-nats-url",
 			Usage:       "nats server url to send the bucket notifications",
-			EnvVars:     []string{"VGW_EVENT_NATS_URL"},
 			Destination: &natsURL,
 			Aliases:     []string{"enu"},
 		},
 		&cli.StringFlag{
 			Name:        "event-nats-topic",
 			Usage:       "nats server pub-sub topic to send the bucket notifications to",
-			EnvVars:     []string{"VGW_EVENT_NATS_TOPIC"},
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
-		&cli.StringFlag{
-			Name:        "event-webhook-url",
-			Usage:       "webhook url to send bucket notifications",
-			EnvVars:     []string{"VGW_EVENT_WEBHOOK_URL"},
-			Destination: &eventWebhookURL,
-			Aliases:     []string{"ewu"},
-		},
-		&cli.StringFlag{
-			Name:        "event-filter",
-			Usage:       "bucket event notifications filters configuration file path",
-			EnvVars:     []string{"VGW_EVENT_FILTER"},
-			Destination: &eventConfigFilePath,
-			Aliases:     []string{"ef"},
-		},
 		&cli.StringFlag{
 			Name:        "iam-dir",
 			Usage:       "if defined, run internal iam service within this directory",
-			EnvVars:     []string{"VGW_IAM_DIR"},
 			Destination: &iamDir,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-url",
 			Usage:       "ldap server url to store iam data",
-			EnvVars:     []string{"VGW_IAM_LDAP_URL"},
 			Destination: &ldapURL,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-bind-dn",
 			Usage:       "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
-			EnvVars:     []string{"VGW_IAM_LDAP_BIND_DN"},
 			Destination: &ldapBindDN,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-bind-pass",
 			Usage:       "ldap server user password",
-			EnvVars:     []string{"VGW_IAM_LDAP_BIND_PASS"},
 			Destination: &ldapPassword,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-query-base",
 			Usage:       "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
-			EnvVars:     []string{"VGW_IAM_LDAP_QUERY_BASE"},
 			Destination: &ldapQueryBase,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-object-classes",
 			Usage:       "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
-			EnvVars:     []string{"VGW_IAM_LDAP_OBJECT_CLASSES"},
 			Destination: &ldapObjClasses,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-access-atr",
 			Usage:       "ldap server user access key id attribute name",
-			EnvVars:     []string{"VGW_IAM_LDAP_ACCESS_ATR"},
 			Destination: &ldapAccessAtr,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-secret-atr",
 			Usage:       "ldap server user secret access key attribute name",
-			EnvVars:     []string{"VGW_IAM_LDAP_SECRET_ATR"},
 			Destination: &ldapSecAtr,
 		},
 		&cli.StringFlag{
 			Name:        "iam-ldap-role-atr",
 			Usage:       "ldap server user role attribute name",
-			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
-		&cli.StringFlag{
-			Name:        "iam-ldap-user-id-atr",
-			Usage:       "ldap server user id attribute name",
-			EnvVars:     []string{"VGW_IAM_LDAP_USER_ID_ATR"},
-			Destination: &ldapUserIdAtr,
-		},
-		&cli.StringFlag{
-			Name:        "iam-ldap-group-id-atr",
-			Usage:       "ldap server user group id attribute name",
-			EnvVars:     []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
-			Destination: &ldapGroupIdAtr,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-endpoint-url",
-			Usage:       "vault server url",
-			EnvVars:     []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
-			Destination: &vaultEndpointURL,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-secret-storage-path",
-			Usage:       "vault server secret storage path",
-			EnvVars:     []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
-			Destination: &vaultSecretStoragePath,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-mount-path",
-			Usage:       "vault server mount path",
-			EnvVars:     []string{"VGW_IAM_VAULT_MOUNT_PATH"},
-			Destination: &vaultMountPath,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-root-token",
-			Usage:       "vault server root token",
-			EnvVars:     []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
-			Destination: &vaultRootToken,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-role-id",
-			Usage:       "vault server user role id",
-			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_ID"},
-			Destination: &vaultRoleId,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-role-secret",
-			Usage:       "vault server user role secret",
-			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_SECRET"},
-			Destination: &vaultRoleSecret,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-server_cert",
-			Usage:       "vault server TLS certificate",
-			EnvVars:     []string{"VGW_IAM_VAULT_SERVER_CERT"},
-			Destination: &vaultServerCert,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-client_cert",
-			Usage:       "vault client TLS certificate",
-			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT"},
-			Destination: &vaultClientCert,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-client_cert_key",
-			Usage:       "vault client TLS certificate key",
-			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
-			Destination: &vaultClientCertKey,
-		},
-		&cli.StringFlag{
-			Name:        "s3-iam-access",
-			Usage:       "s3 IAM access key",
-			EnvVars:     []string{"VGW_S3_IAM_ACCESS_KEY"},
-			Destination: &s3IamAccess,
-		},
-		&cli.StringFlag{
-			Name:        "s3-iam-secret",
-			Usage:       "s3 IAM secret key",
-			EnvVars:     []string{"VGW_S3_IAM_SECRET_KEY"},
-			Destination: &s3IamSecret,
-		},
-		&cli.StringFlag{
-			Name:        "s3-iam-region",
-			Usage:       "s3 IAM region",
-			EnvVars:     []string{"VGW_S3_IAM_REGION"},
-			Destination: &s3IamRegion,
-			Value:       "us-east-1",
-		},
-		&cli.StringFlag{
-			Name:        "s3-iam-bucket",
-			Usage:       "s3 IAM bucket",
-			EnvVars:     []string{"VGW_S3_IAM_BUCKET"},
-			Destination: &s3IamBucket,
-		},
-		&cli.StringFlag{
-			Name:        "s3-iam-endpoint",
-			Usage:       "s3 IAM endpoint",
-			EnvVars:     []string{"VGW_S3_IAM_ENDPOINT"},
-			Destination: &s3IamEndpoint,
-		},
-		&cli.BoolFlag{
-			Name:        "s3-iam-noverify",
-			Usage:       "s3 IAM disable ssl verification",
-			EnvVars:     []string{"VGW_S3_IAM_NO_VERIFY"},
-			Destination: &s3IamSslNoVerify,
-		},
-		&cli.BoolFlag{
-			Name:        "s3-iam-debug",
-			Usage:       "s3 IAM debug output",
-			EnvVars:     []string{"VGW_S3_IAM_DEBUG"},
-			Destination: &s3IamDebug,
-		},
 		&cli.BoolFlag{
 			Name:        "iam-cache-disable",
 			Usage:       "disable local iam cache",
-			EnvVars:     []string{"VGW_IAM_CACHE_DISABLE"},
 			Destination: &iamCacheDisable,
 		},
 		&cli.IntFlag{
 			Name:        "iam-cache-ttl",
 			Usage:       "local iam cache entry ttl (seconds)",
-			EnvVars:     []string{"VGW_IAM_CACHE_TTL"},
 			Value:       120,
 			Destination: &iamCacheTTL,
 		},
 		&cli.IntFlag{
 			Name:        "iam-cache-prune",
 			Usage:       "local iam cache cleanup interval (seconds)",
-			EnvVars:     []string{"VGW_IAM_CACHE_PRUNE"},
 			Value:       3600,
 			Destination: &iamCachePrune,
 		},
-		&cli.StringFlag{
-			Name: "health",
-			Usage: `health check endpoint path. Health endpoint will be configured on GET http method: GET <health>
-					NOTICE: the path has to be specified with '/'. e.g /health`,
-			EnvVars:     []string{"VGW_HEALTH"},
-			Destination: &healthPath,
-		},
-		&cli.BoolFlag{
-			Name:        "readonly",
-			Usage:       "allow only read operations across all the gateway",
-			EnvVars:     []string{"VGW_READ_ONLY"},
-			Destination: &readonly,
-		},
-		&cli.StringFlag{
-			Name:        "metrics-service-name",
-			Usage:       "service name tag for metrics, hostname if blank",
-			EnvVars:     []string{"VGW_METRICS_SERVICE_NAME"},
-			Aliases:     []string{"msn"},
-			Destination: &metricsService,
-		},
-		&cli.StringFlag{
-			Name:        "metrics-statsd-servers",
-			Usage:       "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
-			EnvVars:     []string{"VGW_METRICS_STATSD_SERVERS"},
-			Aliases:     []string{"mss"},
-			Destination: &statsdServers,
-		},
-		&cli.StringFlag{
-			Name:        "metrics-dogstatsd-servers",
-			Usage:       "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
-			EnvVars:     []string{"VGW_METRICS_DOGSTATS_SERVERS"},
-			Aliases:     []string{"mds"},
-			Destination: &dogstatsServers,
-		},
 	}
 }
 
-func runGateway(ctx context.Context, be backend.Backend) error {
-	if rootUserAccess == "" || rootUserSecret == "" {
-		return fmt.Errorf("root user access and secret key must be provided")
-	}
-
-	if pprof != "" {
-		// listen on specified port for pprof debug
-		// point browser to http://<ip:port>/debug/pprof/
-		go func() {
-			log.Fatal(http.ListenAndServe(pprof, nil))
-		}()
+func runGateway(ctx *cli.Context, be backend.Backend) error {
+	// int32 max for 32 bit arch
+	blimit := int64(2*1024*1024*1024 - 1)
+	if strconv.IntSize > 32 {
+		// 5GB max for 64 bit arch
+		blimit = int64(5 * 1024 * 1024 * 1024)
 	}
 
 	app := fiber.New(fiber.Config{
-		AppName:               "versitygw",
-		ServerHeader:          "VERSITYGW",
-		StreamRequestBody:     true,
-		DisableKeepalive:      true,
-		Network:               fiber.NetworkTCP,
-		DisableStartupMessage: true,
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
+		BodyLimit:    int(blimit),
 	})
 
 	var opts []s3api.Option
@@ -553,21 +316,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if admPort == "" {
 		opts = append(opts, s3api.WithAdminServer())
 	}
-	if quiet {
-		opts = append(opts, s3api.WithQuiet())
-	}
-	if healthPath != "" {
-		opts = append(opts, s3api.WithHealth(healthPath))
-	}
-	if readonly {
-		opts = append(opts, s3api.WithReadOnly())
-	}
 
 	admApp := fiber.New(fiber.Config{
-		AppName:               "versitygw",
-		ServerHeader:          "VERSITYGW",
-		Network:               fiber.NetworkTCP,
-		DisableStartupMessage: true,
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
 	})
 
 	var admOpts []s3api.AdminOpt
@@ -588,90 +340,51 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
-		RootAccount: auth.Account{
-			Access: rootUserAccess,
-			Secret: rootUserSecret,
-			Role:   auth.RoleAdmin,
-		},
-		Dir:                    iamDir,
-		LDAPServerURL:          ldapURL,
-		LDAPBindDN:             ldapBindDN,
-		LDAPPassword:           ldapPassword,
-		LDAPQueryBase:          ldapQueryBase,
-		LDAPObjClasses:         ldapObjClasses,
-		LDAPAccessAtr:          ldapAccessAtr,
-		LDAPSecretAtr:          ldapSecAtr,
-		LDAPRoleAtr:            ldapRoleAtr,
-		LDAPUserIdAtr:          ldapUserIdAtr,
-		LDAPGroupIdAtr:         ldapGroupIdAtr,
-		VaultEndpointURL:       vaultEndpointURL,
-		VaultSecretStoragePath: vaultSecretStoragePath,
-		VaultMountPath:         vaultMountPath,
-		VaultRootToken:         vaultRootToken,
-		VaultRoleId:            vaultRoleId,
-		VaultRoleSecret:        vaultRoleSecret,
-		VaultServerCert:        vaultServerCert,
-		VaultClientCert:        vaultClientCert,
-		VaultClientCertKey:     vaultClientCertKey,
-		S3Access:               s3IamAccess,
-		S3Secret:               s3IamSecret,
-		S3Region:               s3IamRegion,
-		S3Bucket:               s3IamBucket,
-		S3Endpoint:             s3IamEndpoint,
-		S3DisableSSlVerfiy:     s3IamSslNoVerify,
-		S3Debug:                s3IamDebug,
-		CacheDisable:           iamCacheDisable,
-		CacheTTL:               iamCacheTTL,
-		CachePrune:             iamCachePrune,
+		Dir:            iamDir,
+		LDAPServerURL:  ldapURL,
+		LDAPBindDN:     ldapBindDN,
+		LDAPPassword:   ldapPassword,
+		LDAPQueryBase:  ldapQueryBase,
+		LDAPObjClasses: ldapObjClasses,
+		LDAPAccessAtr:  ldapAccessAtr,
+		LDAPSecretAtr:  ldapSecAtr,
+		LDAPRoleAtr:    ldapRoleAtr,
+		CacheDisable:   iamCacheDisable,
+		CacheTTL:       iamCacheTTL,
+		CachePrune:     iamCachePrune,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)
 	}
 
-	loggers, err := s3log.InitLogger(&s3log.LogConfig{
-		LogFile:      accessLog,
-		WebhookURL:   logWebhookURL,
-		AdminLogFile: adminLogFile,
+	logger, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:    accessLog,
+		WebhookURL: logWebhookURL,
 	})
 	if err != nil {
 		return fmt.Errorf("setup logger: %w", err)
 	}
 
-	metricsManager, err := metrics.NewManager(ctx, metrics.Config{
-		ServiceName:      metricsService,
-		StatsdServers:    statsdServers,
-		DogStatsdServers: dogstatsServers,
-	})
-	if err != nil {
-		return fmt.Errorf("init metrics manager: %w", err)
-	}
-
 	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
-		KafkaURL:             kafkaURL,
-		KafkaTopic:           kafkaTopic,
-		KafkaTopicKey:        kafkaKey,
-		NatsURL:              natsURL,
-		NatsTopic:            natsTopic,
-		WebhookURL:           eventWebhookURL,
-		FilterConfigFilePath: eventConfigFilePath,
+		KafkaURL:      kafkaURL,
+		KafkaTopic:    kafkaTopic,
+		KafkaTopicKey: kafkaKey,
+		NatsURL:       natsURL,
+		NatsTopic:     natsTopic,
 	})
 	if err != nil {
-		return fmt.Errorf("init bucket event notifications: %w", err)
+		return fmt.Errorf("unable to connect to the message broker: %w", err)
 	}
 
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
 		Access: rootUserAccess,
 		Secret: rootUserSecret,
-	}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
+	}, port, region, iam, logger, evSender, opts...)
 	if err != nil {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
-
-	if !quiet {
-		printBanner(port, admPort, certFile != "", admCertFile != "")
-	}
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
 
 	c := make(chan error, 2)
 	go func() { c <- srv.Serve() }()
@@ -684,21 +397,15 @@ Loop:
 	for {
 		select {
 		case <-ctx.Done():
+			err = ctx.Err()
 			break Loop
 		case err = <-c:
 			break Loop
 		case <-sigHup:
-			if loggers.S3Logger != nil {
-				err = loggers.S3Logger.HangUp()
+			if logger != nil {
+				err = logger.HangUp()
 				if err != nil {
-					err = fmt.Errorf("HUP s3 logger: %w", err)
-					break Loop
-				}
-			}
-			if loggers.AdminLogger != nil {
-				err = loggers.AdminLogger.HangUp()
-				if err != nil {
-					err = fmt.Errorf("HUP admin logger: %w", err)
+					err = fmt.Errorf("HUP logger: %w", err)
 					break Loop
 				}
 			}
@@ -710,218 +417,15 @@ Loop:
 
 	err = iam.Shutdown()
 	if err != nil {
-		if saveErr == nil {
-			saveErr = err
-		}
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
-	if loggers.S3Logger != nil {
-		err := loggers.S3Logger.Shutdown()
+	if logger != nil {
+		err := logger.Shutdown()
 		if err != nil {
-			if saveErr == nil {
-				saveErr = err
-			}
-			fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
+			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
 		}
 	}
-	if loggers.AdminLogger != nil {
-		err := loggers.AdminLogger.Shutdown()
-		if err != nil {
-			if saveErr == nil {
-				saveErr = err
-			}
-			fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
-		}
-	}
-
-	if evSender != nil {
-		err := evSender.Close()
-		if err != nil {
-			if saveErr == nil {
-				saveErr = err
-			}
-			fmt.Fprintf(os.Stderr, "close event sender: %v\n", err)
-		}
-	}
-
-	if metricsManager != nil {
-		metricsManager.Close()
-	}
 
 	return saveErr
 }
-
-func printBanner(port, admPort string, ssl, admSsl bool) {
-	interfaces, err := getMatchingIPs(port)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
-		return
-	}
-
-	var admInterfaces []string
-	if admPort != "" {
-		admInterfaces, err = getMatchingIPs(admPort)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
-			return
-		}
-	}
-
-	title := "VersityGW"
-	version := fmt.Sprintf("Version %v, Build %v", Version, Build)
-	urls := []string{}
-
-	hst, prt, err := net.SplitHostPort(port)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
-		return
-	}
-
-	for _, ip := range interfaces {
-		url := fmt.Sprintf("http://%s:%s", ip, prt)
-		if ssl {
-			url = fmt.Sprintf("https://%s:%s", ip, prt)
-		}
-		urls = append(urls, url)
-	}
-
-	if hst == "" {
-		hst = "0.0.0.0"
-	}
-
-	boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
-
-	lines := []string{
-		centerText(title),
-		centerText(version),
-		centerText(boundHost),
-		centerText(""),
-	}
-
-	if len(admInterfaces) > 0 {
-		lines = append(lines,
-			leftText("S3 service listening on:"),
-		)
-	} else {
-		lines = append(lines,
-			leftText("Admin/S3 service listening on:"),
-		)
-	}
-
-	for _, url := range urls {
-		lines = append(lines, leftText("  "+url))
-	}
-
-	if len(admInterfaces) > 0 {
-		lines = append(lines,
-			centerText(""),
-			leftText("Admin service listening on:"),
-		)
-
-		_, prt, err := net.SplitHostPort(admPort)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
-			return
-		}
-
-		for _, ip := range admInterfaces {
-			url := fmt.Sprintf("http://%s:%s", ip, prt)
-			if admSsl {
-				url = fmt.Sprintf("https://%s:%s", ip, prt)
-			}
-			lines = append(lines, leftText("  "+url))
-		}
-	}
-
-	// Print the top border
-	fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
-
-	// Print each line
-	for _, line := range lines {
-		fmt.Printf("│%-*s│\n", columnWidth-2, line)
-	}
-
-	// Print the bottom border
-	fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
-}
-
-// getMatchingIPs returns all IP addresses for local system interfaces that
-// match the input address specification.
-func getMatchingIPs(spec string) ([]string, error) {
-	// Split the input spec into IP and port
-	host, _, err := net.SplitHostPort(spec)
-	if err != nil {
-		return nil, fmt.Errorf("parse address/port: %v", err)
-	}
-
-	// Handle cases where IP is omitted (e.g., ":1234")
-	if host == "" {
-		host = "0.0.0.0"
-	}
-
-	ipaddr, err := net.ResolveIPAddr("ip", host)
-	if err != nil {
-		return nil, err
-	}
-
-	parsedInputIP := ipaddr.IP
-
-	var result []string
-
-	// Get all network interfaces
-	interfaces, err := net.Interfaces()
-	if err != nil {
-		return nil, err
-	}
-
-	for _, iface := range interfaces {
-		// Get all addresses associated with the interface
-		addrs, err := iface.Addrs()
-		if err != nil {
-			return nil, err
-		}
-
-		for _, addr := range addrs {
-			// Parse the address to get the IP part
-			ipAddr, _, err := net.ParseCIDR(addr.String())
-			if err != nil {
-				return nil, err
-			}
-
-			if ipAddr.IsLinkLocalUnicast() {
-				continue
-			}
-			if ipAddr.IsInterfaceLocalMulticast() {
-				continue
-			}
-			if ipAddr.IsLinkLocalMulticast() {
-				continue
-			}
-
-			// Check if the IP matches the input specification
-			if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
-				result = append(result, ipAddr.String())
-			}
-		}
-	}
-
-	return result, nil
-}
-
-const columnWidth = 70
-
-func centerText(text string) string {
-	padding := (columnWidth - 2 - len(text)) / 2
-	if padding < 0 {
-		padding = 0
-	}
-	return strings.Repeat(" ", padding) + text
-}
-
-func leftText(text string) string {
-	if len(text) > columnWidth-2 {
-		return text
-	}
-	return text + strings.Repeat(" ", columnWidth-2-len(text))
-}

cmd/versitygw/posix.go

@@ -16,21 +16,11 @@ package main
 
 import (
 	"fmt"
-	"io/fs"
-	"math"
 
 	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
-var (
-	chownuid, chowngid bool
-	bucketlinks        bool
-	versioningDir      string
-	dirPerms           uint
-)
-
 func posixCommand() *cli.Command {
 	return &cli.Command{
 		Name:  "posix",
@@ -46,40 +36,6 @@ bucket: mybucket
 object: a/b/c/myobject
 will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 		Action: runPosix,
-		Flags: []cli.Flag{
-			&cli.BoolFlag{
-				Name:        "chuid",
-				Usage:       "chown newly created files and directories to client account UID",
-				EnvVars:     []string{"VGW_CHOWN_UID"},
-				Destination: &chownuid,
-			},
-			&cli.BoolFlag{
-				Name:        "chgid",
-				Usage:       "chown newly created files and directories to client account GID",
-				EnvVars:     []string{"VGW_CHOWN_GID"},
-				Destination: &chowngid,
-			},
-			&cli.BoolFlag{
-				Name:        "bucketlinks",
-				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
-				EnvVars:     []string{"VGW_BUCKET_LINKS"},
-				Destination: &bucketlinks,
-			},
-			&cli.StringFlag{
-				Name:        "versioning-dir",
-				Usage:       "the directory path to enable bucket versioning",
-				EnvVars:     []string{"VGW_VERSIONING_DIR"},
-				Destination: &versioningDir,
-			},
-			&cli.UintFlag{
-				Name:        "dir-perms",
-				Usage:       "default directory permissions for new directories",
-				EnvVars:     []string{"VGW_DIR_PERMS"},
-				Destination: &dirPerms,
-				DefaultText: "0755",
-				Value:       0755,
-			},
-		},
 	}
 }
 
@@ -88,26 +44,10 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	gwroot := (ctx.Args().Get(0))
-	err := meta.XattrMeta{}.Test(gwroot)
-	if err != nil {
-		return fmt.Errorf("posix xattr check: %v", err)
-	}
-
-	if dirPerms > math.MaxUint32 {
-		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
-	}
-
-	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
-		ChownUID:      chownuid,
-		ChownGID:      chowngid,
-		BucketLinks:   bucketlinks,
-		VersioningDir: versioningDir,
-		NewDirPerm:    fs.FileMode(dirPerms),
-	})
+	be, err := posix.New(ctx.Args().Get(0))
 	if err != nil {
 		return fmt.Errorf("init posix: %v", err)
 	}
 
-	return runGateway(ctx.Context, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/s3.go

@@ -15,15 +15,11 @@
 package main
 
 import (
-	"fmt"
-
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/s3proxy"
 )
 
 var (
-	s3proxyAccess          string
-	s3proxySecret          string
 	s3proxyEndpoint        string
 	s3proxyRegion          string
 	s3proxyDisableChecksum bool
@@ -39,49 +35,27 @@ func s3Command() *cli.Command {
 to an s3 storage backend service.`,
 		Action: runS3,
 		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:        "access",
-				Usage:       "s3 proxy server access key id",
-				Value:       "",
-				Required:    true,
-				EnvVars:     []string{"VGW_S3_ACCESS_KEY"},
-				Destination: &s3proxyAccess,
-				Aliases:     []string{"a"},
-			},
-			&cli.StringFlag{
-				Name:        "secret",
-				Usage:       "s3 proxy server secret access key",
-				Value:       "",
-				Required:    true,
-				EnvVars:     []string{"VGW_S3_SECRET_KEY"},
-				Destination: &s3proxySecret,
-				Aliases:     []string{"s"},
-			},
 			&cli.StringFlag{
 				Name:        "endpoint",
 				Usage:       "s3 service endpoint, default AWS if not specified",
 				Value:       "",
-				EnvVars:     []string{"VGW_S3_ENDPOINT"},
 				Destination: &s3proxyEndpoint,
 			},
 			&cli.StringFlag{
 				Name:        "region",
 				Usage:       "s3 service region, default 'us-east-1' if not specified",
 				Value:       "us-east-1",
-				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
 				Value:       false,
-				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
-				EnvVars:     []string{"VGW_S3_SSL_SKIP_VERIFY"},
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
@@ -89,7 +63,6 @@ to an s3 storage backend service.`,
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
 				Value:       false,
-				EnvVars:     []string{"VGW_S3_DEBUG"},
 				Destination: &s3proxyDebug,
 			},
 		},
@@ -97,10 +70,7 @@ to an s3 storage backend service.`,
 }
 
 func runS3(ctx *cli.Context) error {
-	be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+	be := s3proxy.New(s3proxyEndpoint, s3proxyRegion,
 		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
-	if err != nil {
-		return fmt.Errorf("init s3 backend: %w", err)
-	}
-	return runGateway(ctx.Context, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/scoutfs.go

@@ -16,8 +16,6 @@ package main
 
 import (
 	"fmt"
-	"io/fs"
-	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/scoutfs"
@@ -50,35 +48,8 @@ move interfaces as well as support for tiered filesystems.`,
 				Name:        "glacier",
 				Usage:       "enable glacier emulation mode",
 				Aliases:     []string{"g"},
-				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
-			&cli.BoolFlag{
-				Name:        "chuid",
-				Usage:       "chown newly created files and directories to client account UID",
-				EnvVars:     []string{"VGW_CHOWN_UID"},
-				Destination: &chownuid,
-			},
-			&cli.BoolFlag{
-				Name:        "chgid",
-				Usage:       "chown newly created files and directories to client account GID",
-				EnvVars:     []string{"VGW_CHOWN_GID"},
-				Destination: &chowngid,
-			},
-			&cli.BoolFlag{
-				Name:        "bucketlinks",
-				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
-				EnvVars:     []string{"VGW_BUCKET_LINKS"},
-				Destination: &bucketlinks,
-			},
-			&cli.UintFlag{
-				Name:        "dir-perms",
-				Usage:       "default directory permissions for new directories",
-				EnvVars:     []string{"VGW_DIR_PERMS"},
-				Destination: &dirPerms,
-				DefaultText: "0755",
-				Value:       0755,
-			},
 		},
 	}
 }
@@ -88,21 +59,15 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	if dirPerms > math.MaxUint32 {
-		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	var opts []scoutfs.Option
+	if glacier {
+		opts = append(opts, scoutfs.WithGlacierEmulation())
 	}
 
-	var opts scoutfs.ScoutfsOpts
-	opts.GlacierMode = glacier
-	opts.ChownUID = chownuid
-	opts.ChownGID = chowngid
-	opts.BucketLinks = bucketlinks
-	opts.NewDirPerm = fs.FileMode(dirPerms)
-
-	be, err := scoutfs.New(ctx.Args().Get(0), opts)
+	be, err := scoutfs.New(ctx.Args().Get(0), opts...)
 	if err != nil {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}
 
-	return runGateway(ctx.Context, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/test.go

@@ -1,43 +1,31 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package main
 
 import (
 	"fmt"
+	"math"
+	"os"
+	"text/tabwriter"
 
 	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/tests/integration"
+	"github.com/versity/versitygw/integration"
 )
 
 var (
-	awsID             string
-	awsSecret         string
-	endpoint          string
-	prefix            string
-	dstBucket         string
-	partSize          int64
-	objSize           int64
-	concurrency       int
-	files             int
-	totalReqs         int
-	upload            bool
-	download          bool
-	pathStyle         bool
-	checksumDisable   bool
-	versioningEnabled bool
-	azureTests        bool
+	awsID           string
+	awsSecret       string
+	endpoint        string
+	prefix          string
+	dstBucket       string
+	proxyURL        string
+	partSize        int64
+	objSize         int64
+	concurrency     int
+	files           int
+	totalReqs       int
+	upload          bool
+	download        bool
+	pathStyle       bool
+	checksumDisable bool
 )
 
 func testCommand() *cli.Command {
@@ -83,49 +71,17 @@ func initTestFlags() []cli.Flag {
 }
 
 func initTestCommands() []*cli.Command {
-	return append([]*cli.Command{
+	return []*cli.Command{
 		{
 			Name:        "full-flow",
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test the bucket object versioning, if the versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-				&cli.BoolFlag{
-					Name:        "azure-test-mode",
-					Usage:       "Skips tests that are not supported by Azure",
-					Destination: &azureTests,
-					Aliases:     []string{"azure"},
-				},
-			},
 		},
 		{
 			Name:   "posix",
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test posix when versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-			},
-		},
-		{
-			Name:   "iam",
-			Usage:  "Tests iam service",
-			Action: getAction(integration.TestIAM),
-		},
-		{
-			Name:   "access-control",
-			Usage:  "Tests gateway access control with bucket ACLs and Policies",
-			Action: getAction(integration.TestAccessControl),
 		},
 		{
 			Name:  "bench",
@@ -166,6 +122,7 @@ func initTestCommands() []*cli.Command {
 					Name:        "bucket",
 					Usage:       "Destination bucket name to read/write data",
 					Destination: &dstBucket,
+					Required:    true,
 				},
 				&cli.Int64Flag{
 					Name:        "partSize",
@@ -191,6 +148,11 @@ func initTestCommands() []*cli.Command {
 					Value:       false,
 					Destination: &checksumDisable,
 				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
 			},
 			Action: func(ctx *cli.Context) error {
 				if upload && download {
@@ -200,10 +162,6 @@ func initTestCommands() []*cli.Command {
 					return fmt.Errorf("must specify one of upload or download")
 				}
 
-				if dstBucket == "" {
-					return fmt.Errorf("must specify bucket")
-				}
-
 				opts := []integration.Option{
 					integration.WithAccess(awsID),
 					integration.WithSecret(awsSecret),
@@ -225,9 +183,47 @@ func initTestCommands() []*cli.Command {
 				s3conf := integration.NewS3Conf(opts...)
 
 				if upload {
-					return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+					if proxyURL == "" {
+						integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
 				} else {
-					return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+					if proxyURL == "" {
+						integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
 				}
 			},
 		},
@@ -259,12 +255,13 @@ func initTestCommands() []*cli.Command {
 					Value:       false,
 					Destination: &checksumDisable,
 				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
 			},
 			Action: func(ctx *cli.Context) error {
-				if dstBucket == "" {
-					return fmt.Errorf("must specify the destination bucket")
-				}
-
 				opts := []integration.Option{
 					integration.WithAccess(awsID),
 					integration.WithSecret(awsSecret),
@@ -281,10 +278,30 @@ func initTestCommands() []*cli.Command {
 
 				s3conf := integration.NewS3Conf(opts...)
 
-				return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+				if proxyURL == "" {
+					_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					return err
+				} else {
+					elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					opts = append(opts, integration.WithEndpoint(proxyURL))
+					s3proxy := integration.NewS3Conf(opts...)
+					proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
+					if err != nil || proxyErr != nil {
+						return nil
+					}
+
+					printProxyResultsTable([][4]string{
+						{"    #    ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
+						{"---------", "--------------", "----------", "----------------------------"},
+						{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
+						{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
+					})
+
+					return nil
+				}
 			},
 		},
-	}, extractIntTests()...)
+	}
 }
 
 type testFunc func(*integration.S3Conf)
@@ -300,12 +317,6 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		if debug {
 			opts = append(opts, integration.WithDebug())
 		}
-		if versioningEnabled {
-			opts = append(opts, integration.WithVersioningEnabled())
-		}
-		if azureTests {
-			opts = append(opts, integration.WithAzureMode())
-		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -319,41 +330,12 @@ func getAction(tf testFunc) func(*cli.Context) error {
 	}
 }
 
-func extractIntTests() (commands []*cli.Command) {
-	tests := integration.GetIntTests()
-	for key, val := range tests {
-		k := key
-		testFunc := val
-		commands = append(commands, &cli.Command{
-			Name:  k,
-			Usage: fmt.Sprintf("Runs %v integration test", key),
-			Action: func(ctx *cli.Context) error {
-				opts := []integration.Option{
-					integration.WithAccess(awsID),
-					integration.WithSecret(awsSecret),
-					integration.WithRegion(region),
-					integration.WithEndpoint(endpoint),
-				}
-				if debug {
-					opts = append(opts, integration.WithDebug())
-				}
-				if versioningEnabled {
-					opts = append(opts, integration.WithVersioningEnabled())
-				}
-
-				s := integration.NewS3Conf(opts...)
-				err := testFunc(s)
-				return err
-			},
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test the bucket object versioning, if the versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-			},
-		})
+func printProxyResultsTable(stats [][4]string) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	for _, elem := range stats {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
 	}
-	return
+	fmt.Fprintln(w)
+	w.Flush()
 }

cmd/versitygw/utils.go

@@ -1,91 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/s3event"
-)
-
-func utilsCommand() *cli.Command {
-	return &cli.Command{
-		Name:  "utils",
-		Usage: "utility helper CLI tool",
-		Subcommands: []*cli.Command{
-			{
-				Name:    "gen-event-filter-config",
-				Aliases: []string{"gefc"},
-				Usage:   "Create a new configuration file for bucket event notifications filter.",
-				Action:  generateEventFiltersConfig,
-				Flags: []cli.Flag{
-					&cli.StringFlag{
-						Name:    "path",
-						Usage:   "the path where the config file has to be created",
-						Aliases: []string{"p"},
-					},
-				},
-			},
-		},
-	}
-}
-
-func generateEventFiltersConfig(ctx *cli.Context) error {
-	pathFlag := ctx.String("path")
-	path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
-	if err != nil {
-		return err
-	}
-
-	config := s3event.EventFilter{
-		s3event.EventObjectCreated:              true,
-		s3event.EventObjectCreatedPut:           true,
-		s3event.EventObjectCreatedPost:          true,
-		s3event.EventObjectCreatedCopy:          true,
-		s3event.EventCompleteMultipartUpload:    true,
-		s3event.EventObjectRemoved:              true,
-		s3event.EventObjectRemovedDelete:        true,
-		s3event.EventObjectRemovedDeleteObjects: true,
-		s3event.EventObjectTagging:              true,
-		s3event.EventObjectTaggingPut:           true,
-		s3event.EventObjectTaggingDelete:        true,
-		s3event.EventObjectAclPut:               true,
-		s3event.EventObjectRestore:              true,
-		s3event.EventObjectRestorePost:          true,
-		s3event.EventObjectRestoreCompleted:     true,
-	}
-
-	configBytes, err := json.MarshalIndent(config, "", "  ")
-	if err != nil {
-		return fmt.Errorf("parse event config: %w", err)
-	}
-
-	file, err := os.Create(path)
-	if err != nil {
-		return fmt.Errorf("create config file: %w", err)
-	}
-	defer file.Close()
-
-	_, err = file.Write(configBytes)
-	if err != nil {
-		return fmt.Errorf("write config file: %w", err)
-	}
-
-	return nil
-}

extra/dashboard/README.md

@@ -1,19 +0,0 @@
-# Versity Gateway Dashboard
-
-This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana. 
-
-The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
-
-This implementation uses the influxql query language. 
-
-## Usage
-
-From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
-
-To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`. 
-
-The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
-
-The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with  username: `admin` and password: `admin`.
-
-To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

extra/dashboard/aws_env_setup.sh

@@ -1,4 +0,0 @@
-export AWS_SECRET_ACCESS_KEY=password
-export AWS_ACCESS_KEY_ID=user
-export AWS_ENDPOINT_URL=http://127.0.0.1:7070
-export AWS_REGION=us-east-1

extra/dashboard/docker-compose-metrics.yml

@@ -1,64 +0,0 @@
-services:
-  telegraf:
-    image: telegraf
-    container_name: telegraf
-    restart: always
-    volumes:
-    - ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
-    depends_on:
-      - influxdb
-    links:
-      - influxdb
-    ports:
-    - '8125:8125/udp'
-
-  influxdb:
-    image: influxdb
-    container_name: influxdb
-    restart: always
-    environment:
-      - DOCKER_INFLUXDB_INIT_MODE=setup
-      - DOCKER_INFLUXDB_INIT_USERNAME=admin
-      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
-      - DOCKER_INFLUXDB_INIT_ORG=myorg
-      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
-      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
-    ports:
-      - '8086:8086'
-    volumes:
-      - influxdb_data:/var/lib/influxdb
-
-  grafana:
-    image: grafana/grafana
-    container_name: grafana-server
-    restart: always
-    depends_on:
-      - influxdb
-    environment:
-      - GF_SECURITY_ADMIN_USER=admin
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-      - GF_INSTALL_PLUGINS=
-    links:
-      - influxdb
-    ports:
-      - '3000:3000'
-    volumes:
-      - ./metrics-exploration/grafana_data:/var/lib/grafana
-      - ./metrics-exploration/provisioning:/etc/grafana/provisioning
-
-  versitygw:
-    image: versity/versitygw:latest
-    container_name: versitygw
-    ports:
-      - "7070:7070"
-    environment:
-      - ROOT_ACCESS_KEY=user
-      - ROOT_SECRET_KEY=password
-      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
-    depends_on:
-      - telegraf
-    command: >
-      posix /tmp/vgw
-
-volumes:
-  influxdb_data: {}

extra/dashboard/docker-compose.yml

@@ -1,64 +0,0 @@
-services:
-  telegraf:
-    image: telegraf
-    container_name: telegraf
-    restart: always
-    volumes:
-    - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
-    depends_on:
-      - influxdb
-    links:
-      - influxdb
-    ports:
-    - '8125:8125/udp'
-
-  influxdb:
-    image: influxdb
-    container_name: influxdb
-    restart: always
-    environment:
-      - DOCKER_INFLUXDB_INIT_MODE=setup
-      - DOCKER_INFLUXDB_INIT_USERNAME=admin
-      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
-      - DOCKER_INFLUXDB_INIT_ORG=myorg
-      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
-      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
-    ports:
-      - '8086:8086'
-    volumes:
-      - influxdb_data:/var/lib/influxdb
-
-  grafana:
-    image: grafana/grafana
-    container_name: grafana-server
-    restart: always
-    depends_on:
-      - influxdb
-    environment:
-      - GF_SECURITY_ADMIN_USER=admin
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-      - GF_INSTALL_PLUGINS=
-    links:
-      - influxdb
-    ports:
-      - '3000:3000'
-    volumes:
-      - ./grafana_data:/var/lib/grafana
-      - ./provisioning:/etc/grafana/provisioning
-
-  versitygw:
-    image: versity/versitygw:latest
-    container_name: versitygw
-    ports:
-      - "7070:7070"
-    environment:
-      - ROOT_ACCESS_KEY=user
-      - ROOT_SECRET_KEY=password
-      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
-    depends_on:
-      - telegraf
-    command: >
-      posix /tmp/vgw
-
-volumes:
-  influxdb_data: {}

extra/dashboard/provisioning/dashboards/influxql.yaml

@@ -1,25 +0,0 @@
-apiVersion: 1
-
-providers:
-  # <string> an unique provider name. Required
-  - name: 'influxql'
-    # <int> Org id. Default to 1
-    orgId: 1
-    # <string> name of the dashboard folder.
-    folder: 'influxql'
-    # <string> folder UID. will be automatically generated if not specified
-    folderUid: ''
-    # <string> provider type. Default to 'file'
-    type: file
-    # <bool> disable dashboard deletion
-    disableDeletion: false
-    # <int> how often Grafana will scan for changed dashboards
-    updateIntervalSeconds: 10
-    # <bool> allow updating provisioned dashboards from the UI
-    allowUiUpdates: true
-    options:
-      # <string, required> path to dashboard files on disk. Required when using the 'file' type
-      path: /etc/grafana/provisioning/dashboards/influxql
-      # <bool> use folder names from filesystem to create folders in Grafana
-      foldersFromFilesStructure: true
-

extra/dashboard/provisioning/datasources/influxdb.yml

@@ -1,13 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: influxdb
-    type: influxdb
-    isDefault: true
-    access: proxy
-    url: http://influxdb:8086
-    jsonData:
-      dbName: 'metrics'
-      httpHeaderName1: 'Authorization'
-    secureJsonData:
-      httpHeaderValue1: 'Token my-super-secret-auth-token'

extra/dashboard/telegraf.conf

@@ -1,34 +0,0 @@
-[global_tags]
-
-[agent]
-  debug = true
-  quiet = false
-  interval = "60s"
-  round_interval = true
-  metric_batch_size = 1000
-  metric_buffer_limit = 10000
-  collection_jitter = "0s"
-  flush_interval = "10s"
-  flush_jitter = "0s"
-  precision = ""
-  hostname = "versitygw"
-  omit_hostname = false
-
-[[outputs.file]]
-  files = ["stdout"]
-
-[[outputs.influxdb_v2]]
-  urls = ["http://influxdb:8086"]
-  timeout = "5s"
-  token = "my-super-secret-auth-token"
-  organization = "myorg"
-  bucket = "metrics"
-
-[[inputs.statsd]]
-  protocol = "udp4"
-  service_address = ":8125"
-  percentiles = [90]
-  metric_separator = "_"
-  datadog_extensions = false
-  allowed_pending_messages = 10000
-  percentile_limit = 1000

extra/dashboard/test.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-. ./aws_env_setup.sh
-
-aws s3 mb s3://test
-aws s3 cp docker-compose.yml s3://test/test.yaml

extra/example.conf

@@ -1,382 +0,0 @@
-###################################
-# VersityGW systemd configuration #
-###################################
-
-# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
-# For example, if the service name is "mygateway", then the file should be
-# named /etc/versitygw.d/mygateway.conf.
-# The systemd template file /lib/systemd/system/versitygw@.service will
-# automatically load the configuration file for the service name.
-# To start the gateway, use the following command:
-# systemctl start versitygw@mygateway
-# To enable the gateway to start on boot, use the following command:
-# systemctl enable versitygw@mygateway
-# To stop the gateway, use the following command:
-# systemctl stop versitygw@mygateway
-
-# There can be multiple gateway services running on the same host. Each
-# gateway service must have a unique service name with a unique configuration
-# file in /etc/versitygw.d/. They must also listen on different ports and/or
-# interfaces using the VGW_PORT option.
-
-##############################
-# VersityGW Required Options #
-##############################
-
-# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3
-# This defines the backend that the VGW will use for data access.
-VGW_BACKEND=posix
-
-# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
-# as the the top level directory for the gateway.
-# All sub directories of the top level directory are treated as buckets,
-# and all files/directories below the "bucket directory" are treated as
-# the objects. The object name is split on "/" separator to translate
-# to posix storage.
-# For example:
-# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
-# bucket: mybucket
-# object: a/b/c/myobject
-# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
-VGW_BACKEND_ARG=
-
-############################
-# VersityGW Global Options #
-############################
-
-# commented options are the default values
-
-# The following must be set, and do not have default values
-# The access and secret options will specify the root account credentials.
-# The root account is granted full authorization to all API requests after
-# authentication.
-ROOT_ACCESS_KEY_ID=
-ROOT_SECRET_ACCESS_KEY=
-
-# The following are optional, and have the default values as listed
-
-# The VGW_PORT option will specify the listening port for the S3 server.
-# This option can use either the form <ip>:<port> which will listen only
-# on the network interface that matches the IP on the specified port, or
-# :<port> which will listen on all network interfaces on the specified port.
-# The <ip> spec can either be IP dotted notation or a resolvable hostname.
-# The <port> spec can either be a numeric port or the service name typically
-# in /etc/services.
-#VGW_PORT=:7070
-
-# The VGW_REGION option will specify the region that the S3 server will
-# report to clients. This option is optional, and defaults to "us-east-1".
-#VGW_REGION=us-east-1
-
-# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
-# private key that the S3 server will use for SSL connections. This option
-# is optional, and defaults to not using SSL.
-#VGW_CERT=
-#VGW_KEY=
-
-# The VGW_ADMIN_PORT option will specify the listening port for the admin
-# server. The admin server endpoint can optionally be set to listen on a
-# different interface or port than the S3 service. This allows for better
-# control of firewall restrictions to the admin endpoint. The certs for this
-# can be different certs than specified for the S3 service. The default when
-# these are not specified is to have the admin server listen on the same
-# endpoint as the S3 service.
-# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
-# server will use SSL.
-#VGW_ADMIN_PORT=
-#VGW_ADMIN_CERT=
-#VGW_ADMIN_CERT_KEY=
-
-# The VGW_QUIET option when set will supress the S3 server request summary
-# logging to stdout.
-#VGW_QUIET=false
-
-# The VGW_HEALTH option when set will specify the URL to accept health checks
-# on. The health check endpoint is often used for load balancers to verify
-# gateway is alive. The health endpoint masks any bucket with this setting.
-# For example, if the health endpoint is set to /health, the gateway will not
-# allow creating or listing contents of a bucket called "health". The health
-# endpoint is unauthenticated, and returns a 200 status for GET.
-#VGW_HEALTH=
-
-###############
-# Access Logs #
-###############
-
-# The VGW_ACCESS_LOG option when set will specify the file to log all S3
-# server requests to. This option is optional, and defaults to not logging.
-# It is suggested to use absolute paths for the server log file because the
-# server may chdir into the backend root directory and change locations for
-# relative paths.
-# The log file format follows the AWS S3 access log format documented in
-# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
-#VGW_ACCESS_LOG=
-
-# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
-# S3 server request access logs to. The access logs are JSON encoded when
-# sent to the webhook.
-#VGW_LOG_WEBHOOK_URL=
-
-##############
-# Event Logs #
-##############
-
-# The gateway events are similar to AWS S3 events, and are documented in the
-# wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
-
-# The VGW_EVENT_FILTER option specifies a config file that contains the
-# event filter rules. The event filter rules are used to determine which
-# events are sent to the configured event services.
-# Use the following to generate a default rules file in /etc/versitygw.d/:
-# versitygw utils gen-event-filter-config -p /etc/versitygw.d
-# The resulting file, /etc/versitygw.d/event_config.json, can be modified and
-# specified in the VGW_EVENT_FILTER option.
-# When VGW_EVENT_FILTER is not specified, all events are sent to the configured
-# event service.
-#VGW_EVENT_FILTER=
-
-# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
-# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
-# configured bucket events will be sent to the kafka service.
-#VGW_EVENT_KAFKA_URL=
-#VGW_EVENT_KAFKA_TOPIC=
-#VGW_EVENT_KAFKA_KEY=
-
-# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
-# and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be
-# sent to the the NATS messaging service.
-#VGW_EVENT_NATS_URL=
-#VGW_EVENT_NATS_TOPIC=
-
-# Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is
-# specified, all configured bucket events will be sent to the webhook.
-#VGW_EVENT_WEBHOOK_URL=
-
-# Bucket events can be filtered for any of the above event types. The
-# VGW_EVENT_FILTER option specifies a config file that contains the event
-# filter rules. The event filter rules are used to determine which events are
-# sent to the configured event services. Run:
-# versitygw utils gen-event-filter-config --path .
-# to generate a default rules file "event_config.json" in the current directory.
-#VGW_EVENT_FILTER=
-
-#######################
-# Debug / Diagnostics #
-#######################
-
-# The VGW_DEBUG option enables verbose debug log output to stdout. This output
-# includes details for signature verification steps. This is generally only
-# useful for debugging the S3 server, and should not be used in production.
-#VGW_DEBUG=false
-
-# The VGW_PPROF option enables the pprof HTTP server for profiling the S3
-# server. See the following for more information:
-# https://pkg.go.dev/net/http/pprof
-# To enable, set the VGW_PPROF option to the listening address for the pprof
-# server. For example, to listen on localhost port 6060, set the option to
-# "localhost:6060".
-#VGW_PPROF=
-
-################
-# IAM services #
-################
-
-# The VGW_IAM_DIR option will enable the internal IAM service with accounts
-# stored in a file under the specified directory. This is provided to minimize
-# dependencies on outside services for basic functionality. The local account
-# files are plain text and only protected with file permissions. This IAM
-# service is added for convenience, but is not considered as secure or scalable
-# as a dedicated IAM service.
-#VGW_IAM_DIR=
-
-# The Vault options will enable the Vault IAM service with accounts stored in
-# the HashiCorp Vault service. The Vault URL is the address and port of the
-# Vault server with the format <IP/host>:<port>. A root taken can be used for
-# testing, but it is recommended to use the role based authentication in
-# production. The Vault server certificate, client certificate, and client
-# certificate key are optional, and will default to not verifying the server
-# certificate and not using client certificates. The Vault server certificate
-# is used to verify the Vault server, and the client certificate and key are
-# used to authenticate the gateway to the Vault server. See wiki documentation
-# for an example of using Vault in dev mode with the gateway.
-#VGW_IAM_VAULT_ENDPOINT_URL=
-#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
-#VGW_IAM_VAULT_MOUNT_PATH=
-#VGW_IAM_VAULT_ROOT_TOKEN=
-#VGW_IAM_VAULT_ROLE_ID=
-#VGW_IAM_VAULT_ROLE_SECRET=
-#VGW_IAM_VAULT_SERVER_CERT=
-#VGW_IAM_VAULT_CLIENT_CERT=
-#VGW_IAM_VAULT_CLIENT_CERT_KEY=
-
-# The VGW_S3 IAM service is similar to the internal IAM service, but instead
-# stores the account information JSON encoded in an S3 object. This should use
-# a bucket that is not accessible to general users when using s3 backend to
-# prevent access to account credentials. This IAM service is added for
-# convenience, but is not considered as secure or scalable as a dedicated IAM
-# service.
-#VGW_S3_IAM_ACCESS_KEY=
-#VGW_S3_IAM_SECRET_KEY=
-#VGW_S3_IAM_REGION=
-#VGW_S3_IAM_ENDPOINT=
-#VGW_S3_IAM_BUCKET=
-#VGW_S3_IAM_NO_VERIFY=
-
-# The LDAP options will enable the LDAP IAM service with accounts stored in an
-# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
-# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
-# secret credentials and role respectively. The other options are used to
-# connect to the LDAP service.
-#VGW_IAM_LDAP_URL=
-#VGW_IAM_LDAP_BASE_DN=
-#VGW_IAM_LDAP_BIND_DN=
-#VGW_IAM_LDAP_BIND_PASS=
-#VGW_IAM_LDAP_QUERY_BASE=
-#VGW_IAM_LDAP_OBJECT_CLASSES=
-#VGW_IAM_LDAP_ACCESS_ATR=
-#VGW_IAM_LDAP_SECRET_ATR=
-#VGW_IAM_LDAP_ROLE_ATR=
-#VGW_IAM_LDAP_USER_ID_ATR=
-#VGW_IAM_LDAP_GROUP_ID_ATR=
-
-###############
-# IAM caching #
-###############
-
-# The IAM cache is intended to ease the load on the IAM service and increase
-# the Gateway performance by caching accounts and credentials for the TTL time
-# interval. Disabling this will cause a request to the configured IAM service
-# for each incoming request to retrieve the corresponding account credentials.
-# The cache is enabled by default. The TTL specifies how long to cache
-# credentials, and the prune value determines the interval for expired entries
-# to be removed from the cache. Increasing the TTL may lessen the load on the
-# IAM service backend, but may have out of date account info until the next
-# interval. Increasing the prune value may reduce memory use at the cost of
-# added CPU to check cache expirations.
-#VGW_IAM_CACHE_DISABLE=false
-#VGW_IAM_CACHE_TTL=120
-#VGW_IAM_CACHE_PRUNE=3600
-
-###########
-# Metrics #
-###########
-
-# The metrics service name is a tag that is added to all metrics to help
-# identify the source of the metrics. This is especially useful when multiple
-# gateways are running. The default is the hostname of the system.
-#VGW_METRICS_SERVICE_NAME=$HOSTNAME
-
-# The metrics service will send metrics to the configured statsd servers. The
-# servers are specified as a comma separated list of host:port pairs. The
-# default is to not send metrics to any statsd servers. The gateway uses
-# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
-#VGW_METRICS_STATSD_SERVERS=
-
-# The metrics service will send metrics to the configured dogstatsd servers.
-# The servers are specified as a comma separated list of host:port pairs. The
-# default is to not send metrics to any dogstatsd servers. Generally
-# DataDog recommends installing a local agent to collect metrics and forward
-# them to the DataDog service. In this case the option value would be the
-# local agent address: 127.0.0.1:8125.
-#VGW_METRICS_DOGSTATS_SERVERS=
-
-######################################
-# VersityGW Backend Specific Options #
-######################################
-
-#########
-# posix #
-#########
-
-# The posix backend translates S3 requests to file access in a local filesystem.
-# The posix backend requires a filesystem that supports extended attributes.
-# The top level directory for the gateway must be provided. All sub directories
-# of the top level directory are treated as buckets, and all files/directories
-# below the "bucket directory" are treated as the objects. The object
-# name is split on "/" separator to translate to posix storage.
-# For example:
-# top level (VGW_BACKEND_ARG): /mnt/fs/gwroot
-# bucket: mybucket
-# object: a/b/c/myobject
-# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
-
-# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
-# change the ownership of newly created files and directories to the IAM
-# account UID/GID.
-#VGW_CHOWN_UID=false
-#VGW_CHOWN_GID=false
-
-# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
-# to directories at the top level gateway directory as buckets.
-#VGW_BUCKET_LINKS=false
-
-# The default permissions mode when creating new directories is 0755. Use
-# VGW_DIR_PERMS option to set a different mode for any new directory that the
-# gateway creates. This applies to buckets created through the gateway as well
-# as any parent directories automatically created with object uploads.
-#VGW_DIR_PERMS=0755
-
-###########
-# scoutfs #
-###########
-
-# The scoutfs backend requires a ScoutFS filesystem type for the backend
-# path. The object to posix name mappings follow the same rules as posix for
-# scoutfs. The glacier mode functionality requires ScoutAM to be configured
-# for tiering data from the ScoutFS filesystem to a mass stroage system.
-# The mass storage system is often one or more tape libraries. Due to the
-# high latency of tape, the glacier mode functionality is designed to
-# give feedback to clients about object state and offer ability to request
-# data to be staged back to disk without the client dealing with long
-# request timeout settings.
-
-# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
-# GET object:  if file offline, return invalid object state
-# HEAD object: if file offline, set obj storage class to GLACIER
-#              if file offline and staging, x-amz-restore: ongoing-request="true"
-#              if file offline and not staging, x-amz-restore: ongoing-request="false"
-#              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
-#              note: this expiry-date is not used but provided for client glacier compatibility
-# ListObjects: if file offline, set obj storage class to GLACIER
-# RestoreObject: add batch stage request to file
-#VGW_SCOUTFS_GLACIER=false
-
-# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
-# change the ownership of newly created files and directories to the IAM
-# account UID/GID.
-#VGW_CHOWN_UID=false
-#VGW_CHOWN_GID=false
-
-# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
-# to directories at the top level gateway directory as buckets.
-#VGW_BUCKET_LINKS=false
-
-# The default permissions mode when creating new directories is 0755. Use
-# VGW_DIR_PERMS option to set a different mode for any new directory that the
-# gateway creates. This applies to buckets created through the gateway as well
-# as any parent directories automatically created with object uploads.
-#VGW_DIR_PERMS=0755
-
-######
-# s3 #
-######
-
-# The s3 backend allows the gateway to forward requests to an S3 compatible
-# service. This allows the gateway to act as a proxy for another S3 service.
-# The backend S3 access is all done with a single configured account. The
-# gateway will manage incoming multi-tenant access with the gateway configured
-# IAM service. This gives stroage admins the ability to manage local gateway
-# accounts while maintaining full control and a single account for the backend
-# S3 service.
-
-# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
-# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
-# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
-#VGW_S3_ACCESS_KEY=
-#VGW_S3_SECRET_KEY=
-#VGW_S3_REGION=
-#VGW_S3_ENDPOINT=
-#VGW_S3_DISABLE_CHECKSUM=false
-#VGW_S3_SSL_SKIP_VERIFY=false
-#VGW_S3_DEBUG=false

extra/posttrans.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-systemctl daemon-reload

extra/versitygw@.service

@@ -1,33 +0,0 @@
-[Unit]
-Description=VersityGW
-Documentation=https://github.com/versity/versitygw/wiki
-Wants=network-online.target
-After=network-online.target remote-fs.target
-AssertFileIsExecutable=/usr/bin/versitygw
-AssertPathExists=/etc/versitygw.d/%i.conf
-
-[Service]
-WorkingDirectory=/root
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=versitygw-%i
-
-User=root
-Group=root
-
-EnvironmentFile=/etc/versitygw.d/%i.conf
-
-ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"'
-
-# Let systemd restart this service always
-Restart=always
-
-# Specifies the maximum file descriptor number that can be opened by this process
-LimitNOFILE=65536
-
-# Specifies the maximum number of threads this process can create
-TasksMax=infinity
-
-[Install]
-WantedBy=multi-user.target
-

go.mod

@@ -1,82 +1,60 @@
 module github.com/versity/versitygw
 
-go 1.21.0
+go 1.20
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0
-	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.32.6
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0
-	github.com/aws/smithy-go v1.22.1
-	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/gofiber/fiber/v2 v2.52.5
-	github.com/google/go-cmp v0.6.0
-	github.com/google/uuid v1.6.0
-	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.37.0
-	github.com/oklog/ulid/v2 v2.1.0
-	github.com/pkg/xattr v0.4.10
-	github.com/segmentio/kafka-go v0.4.47
-	github.com/smira/go-statsd v1.3.4
-	github.com/urfave/cli/v2 v2.27.5
-	github.com/valyala/fasthttp v1.57.0
-	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sync v0.9.0
-	golang.org/x/sys v0.27.0
+	github.com/aws/aws-sdk-go-v2 v1.22.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
+	github.com/aws/smithy-go v1.16.0
+	github.com/go-ldap/ldap/v3 v3.4.6
+	github.com/gofiber/fiber/v2 v2.50.0
+	github.com/google/uuid v1.4.0
+	github.com/nats-io/nats.go v1.31.0
+	github.com/pkg/xattr v0.4.9
+	github.com/segmentio/kafka-go v0.4.44
+	github.com/urfave/cli/v2 v2.25.7
+	github.com/valyala/fasthttp v1.50.0
+	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
+	golang.org/x/sys v0.14.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
-	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
-	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
-	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.8 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.21 // indirect
-	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.29.0 // indirect
-	golang.org/x/net v0.31.0 // indirect
-	golang.org/x/text v0.20.0 // indirect
-	golang.org/x/time v0.8.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.28.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.47
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.24.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.15.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 )

go.sum

@@ -1,286 +1,175 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
-github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
-github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
-github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
-github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4=
-github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
-github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo=
-github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42 h1:vEnk9vtjJ62OO2wOhEmgKMZgNcn1w0aF7XCiNXO5rK0=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42/go.mod h1:GUOPbPJWRZsdt1OJ355upCrry4d3ZFgdX6rhT7gtkto=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0 h1:HrHFR8RoS4l4EvodRMFcJMYQ8o3UhmALn2nbInXaxZA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8=
-github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
-github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
-github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
+github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
+github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
+github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
+github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
+github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
-github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
-github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
-github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
-github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
-github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
-github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
-github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
-github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
-github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
-github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
-github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
-github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
-github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
-github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
-github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
-github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
-github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
-github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
-github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
-github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
-github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
-github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
-github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
-github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
-github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
-github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
-github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
+github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
+github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
+github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
+github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
-github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
-github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
-github.com/nats-io/nkeys v0.4.8 h1:+wee30071y3vCZAYRsnrmIPaOe47A/SkK/UBDPdIV70=
-github.com/nats-io/nkeys v0.4.8/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
+github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
+github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
+github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
-github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
-github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
-github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
-github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
+github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
-github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
-github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
-github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
+github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4=
+github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
-github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
+github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
-github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE=
+github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
+github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
-github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
-github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
+github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
+github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
-github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
-golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
-golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
-golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
-golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
-golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
-golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
-golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration/bench.go

@@ -1,17 +1,3 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package integration
 
 import (
@@ -31,16 +17,16 @@ type prefResult struct {
 	err     error
 }
 
-func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
 	var sg sync.WaitGroup
 	results := make([]prefResult, files)
 	start := time.Now()
 	if objSize == 0 {
-		return fmt.Errorf("must specify object size for upload")
+		return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
 	}
 
 	if objSize > (int64(10000) * s.PartSize) {
-		return fmt.Errorf("object size can not exceed 10000 * chunksize")
+		return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
 	}
 
 	runF("performance test: upload objects")
@@ -59,13 +45,13 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
 		}(i)
 	}
 	sg.Wait()
-	elapsed := time.Since(start)
+	elapsed = time.Since(start)
 
 	var tot int64
 	for i, res := range results {
 		if res.err != nil {
 			failF("%v: %v\n", i, res.err)
-			break
+			return 0, time.Since(start), res.err
 		}
 		tot += res.size
 		fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -77,10 +63,10 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
 	passF("run upload: %v in %v (%v MB/s)\n",
 		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
 
-	return nil
+	return tot, time.Since(start), nil
 }
 
-func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
 	var sg sync.WaitGroup
 	results := make([]prefResult, files)
 	start := time.Now()
@@ -100,13 +86,13 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
 		}(i)
 	}
 	sg.Wait()
-	elapsed := time.Since(start)
+	elapsed = time.Since(start)
 
 	var tot int64
 	for i, res := range results {
 		if res.err != nil {
 			failF("%v: %v\n", i, res.err)
-			break
+			return 0, elapsed, err
 		}
 		tot += res.size
 		fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -118,10 +104,10 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
 	passF("run download: %v in %v (%v MB/s)\n",
 		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
 
-	return nil
+	return tot, elapsed, nil
 }
 
-func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
+func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
 	client := s3.NewFromConfig(s.Config())
 	var wg sync.WaitGroup
 	var resErr error
@@ -146,11 +132,11 @@ func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
 	wg.Wait()
 	if resErr != nil {
 		failF("performance test failed with error: %w", resErr)
-		return nil
+		return time.Since(startTime), 0, resErr
 	}
 	elapsedTime := time.Since(startTime)
 	rps := int(float64(totalReqs) / elapsedTime.Seconds())
 
 	passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
-	return nil
+	return elapsedTime, rps, nil
 }

integration/data-io.go

@@ -1,17 +1,3 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package integration
 
 import (

integration/group-tests.go

@@ -0,0 +1,224 @@
+package integration
+
+func TestAuthentication(s *S3Conf) {
+	Authentication_empty_auth_header(s)
+	Authentication_invalid_auth_header(s)
+	Authentication_unsupported_signature_version(s)
+	Authentication_malformed_credentials(s)
+	Authentication_malformed_credentials_invalid_parts(s)
+	Authentication_credentials_terminated_string(s)
+	Authentication_credentials_incorrect_service(s)
+	Authentication_credentials_incorrect_region(s)
+	Authentication_credentials_invalid_date(s)
+	Authentication_credentials_future_date(s)
+	Authentication_credentials_past_date(s)
+	Authentication_credentials_non_existing_access_key(s)
+	Authentication_invalid_signed_headers(s)
+	Authentication_missing_date_header(s)
+	Authentication_invalid_date_header(s)
+	Authentication_date_mismatch(s)
+	Authentication_incorrect_payload_hash(s)
+	Authentication_incorrect_md5(s)
+	Authentication_signature_error_incorrect_secret_key(s)
+}
+
+func TestCreateBucket(s *S3Conf) {
+	CreateBucket_invalid_bucket_name(s)
+	CreateBucket_existing_bucket(s)
+	CreateBucket_as_user(s)
+	CreateDeleteBucket_success(s)
+}
+
+func TestHeadBucket(s *S3Conf) {
+	HeadBucket_non_existing_bucket(s)
+	HeadBucket_success(s)
+}
+
+func TestListBuckets(s *S3Conf) {
+	ListBuckets_as_user(s)
+	ListBuckets_as_admin(s)
+	ListBuckets_success(s)
+}
+
+func TestDeleteBucket(s *S3Conf) {
+	DeleteBucket_non_existing_bucket(s)
+	DeleteBucket_non_empty_bucket(s)
+	DeleteBucket_success_status_code(s)
+}
+
+func TestPutObject(s *S3Conf) {
+	PutObject_non_existing_bucket(s)
+	PutObject_special_chars(s)
+	PutObject_invalid_long_tags(s)
+	PutObject_success(s)
+}
+
+func TestHeadObject(s *S3Conf) {
+	HeadObject_non_existing_object(s)
+	HeadObject_success(s)
+}
+
+func TestGetObject(s *S3Conf) {
+	GetObject_non_existing_key(s)
+	GetObject_invalid_ranges(s)
+	GetObject_with_meta(s)
+	GetObject_success(s)
+	GetObject_by_range_success(s)
+}
+
+func TestListObjects(s *S3Conf) {
+	ListObjects_non_existing_bucket(s)
+	ListObjects_with_prefix(s)
+	ListObject_truncated(s)
+	ListObjects_invalid_max_keys(s)
+	ListObjects_max_keys_0(s)
+	ListObjects_delimiter(s)
+	ListObjects_max_keys_none(s)
+	ListObjects_marker_not_from_obj_list(s)
+}
+
+func TestDeleteObject(s *S3Conf) {
+	DeleteObject_non_existing_object(s)
+	DeleteObject_success(s)
+	DeleteObject_success_status_code(s)
+}
+
+func TestDeleteObjects(s *S3Conf) {
+	DeleteObjects_empty_input(s)
+	DeleteObjects_non_existing_objects(s)
+	DeleteObjects_success(s)
+}
+
+func TestCopyObject(s *S3Conf) {
+	CopyObject_non_existing_dst_bucket(s)
+	CopyObject_not_owned_source_bucket(s)
+	CopyObject_copy_to_itself(s)
+	CopyObject_to_itself_with_new_metadata(s)
+	CopyObject_success(s)
+}
+
+func TestPutObjectTagging(s *S3Conf) {
+	PutObjectTagging_non_existing_object(s)
+	PutObjectTagging_long_tags(s)
+	PutObjectTagging_success(s)
+}
+
+func TestGetObjectTagging(s *S3Conf) {
+	GetObjectTagging_non_existing_object(s)
+	GetObjectTagging_success(s)
+}
+
+func TestDeleteObjectTagging(s *S3Conf) {
+	DeleteObjectTagging_non_existing_object(s)
+	DeleteObjectTagging_success_status(s)
+	DeleteObjectTagging_success(s)
+}
+
+func TestCreateMultipartUpload(s *S3Conf) {
+	CreateMultipartUpload_non_existing_bucket(s)
+	CreateMultipartUpload_success(s)
+}
+
+func TestUploadPart(s *S3Conf) {
+	UploadPart_non_existing_bucket(s)
+	UploadPart_invalid_part_number(s)
+	UploadPart_non_existing_key(s)
+	UploadPart_non_existing_mp_upload(s)
+	UploadPart_success(s)
+}
+
+func TestUploadPartCopy(s *S3Conf) {
+	UploadPartCopy_non_existing_bucket(s)
+	UploadPartCopy_incorrect_uploadId(s)
+	UploadPartCopy_incorrect_object_key(s)
+	UploadPartCopy_invalid_part_number(s)
+	UploadPartCopy_invalid_copy_source(s)
+	UploadPartCopy_non_existing_source_bucket(s)
+	UploadPartCopy_non_existing_source_object_key(s)
+	UploadPartCopy_success(s)
+	UploadPartCopy_by_range_invalid_range(s)
+	UploadPartCopy_greater_range_than_obj_size(s)
+	UploadPartCopy_by_range_success(s)
+}
+
+func TestListParts(s *S3Conf) {
+	ListParts_incorrect_uploadId(s)
+	ListParts_incorrect_object_key(s)
+	ListParts_success(s)
+}
+
+func TestListMultipartUploads(s *S3Conf) {
+	ListMultipartUploads_non_existing_bucket(s)
+	ListMultipartUploads_empty_result(s)
+	ListMultipartUploads_invalid_max_uploads(s)
+	ListMultipartUploads_max_uploads(s)
+	ListMultipartUploads_incorrect_next_key_marker(s)
+	ListMultipartUploads_ignore_upload_id_marker(s)
+	ListMultipartUploads_success(s)
+}
+
+func TestAbortMultipartUpload(s *S3Conf) {
+	AbortMultipartUpload_non_existing_bucket(s)
+	AbortMultipartUpload_incorrect_uploadId(s)
+	AbortMultipartUpload_incorrect_object_key(s)
+	AbortMultipartUpload_success(s)
+	AbortMultipartUpload_success_status_code(s)
+}
+
+func TestCompleteMultipartUpload(s *S3Conf) {
+	CompletedMultipartUpload_non_existing_bucket(s)
+	CompleteMultipartUpload_invalid_part_number(s)
+	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_success(s)
+}
+
+func TestPutBucketAcl(s *S3Conf) {
+	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_invalid_acl_canned_and_acp(s)
+	PutBucketAcl_invalid_acl_canned_and_grants(s)
+	PutBucketAcl_invalid_acl_acp_and_grants(s)
+	PutBucketAcl_invalid_owner(s)
+	PutBucketAcl_success_access_denied(s)
+	PutBucketAcl_success_grants(s)
+	PutBucketAcl_success_canned_acl(s)
+	PutBucketAcl_success_acp(s)
+}
+
+func TestGetBucketAcl(s *S3Conf) {
+	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_access_denied(s)
+	GetBucketAcl_success(s)
+}
+
+func TestFullFlow(s *S3Conf) {
+	TestAuthentication(s)
+	TestCreateBucket(s)
+	TestHeadBucket(s)
+	TestListBuckets(s)
+	TestDeleteBucket(s)
+	TestPutObject(s)
+	TestHeadObject(s)
+	TestGetObject(s)
+	TestListObjects(s)
+	TestDeleteObject(s)
+	TestDeleteObjects(s)
+	TestCopyObject(s)
+	TestPutObjectTagging(s)
+	TestDeleteObjectTagging(s)
+	TestCreateMultipartUpload(s)
+	TestUploadPart(s)
+	TestUploadPartCopy(s)
+	TestListParts(s)
+	TestListMultipartUploads(s)
+	TestAbortMultipartUpload(s)
+	TestCompleteMultipartUpload(s)
+	TestPutBucketAcl(s)
+	TestGetBucketAcl(s)
+}
+
+func TestPosix(s *S3Conf) {
+	PutObject_overwrite_dir_obj(s)
+	PutObject_overwrite_file_obj(s)
+	PutObject_dir_obj_with_data(s)
+	CreateMultipartUpload_dir_obj(s)
+}

integration/output.go

@@ -0,0 +1,31 @@
+package integration
+
+import "fmt"
+
+var (
+	colorReset = "\033[0m"
+	colorRed   = "\033[31m"
+	colorGreen = "\033[32m"
+	colorCyan  = "\033[36m"
+)
+
+var (
+	RunCount  = 0
+	PassCount = 0
+	FailCount = 0
+)
+
+func runF(format string, a ...interface{}) {
+	RunCount++
+	fmt.Printf(colorCyan+"RUN  "+colorReset+format+"\n", a...)
+}
+
+func failF(format string, a ...interface{}) {
+	FailCount++
+	fmt.Printf(colorRed+"FAIL "+colorReset+format+"\n", a...)
+}
+
+func passF(format string, a ...interface{}) {
+	PassCount++
+	fmt.Printf(colorGreen+"PASS "+colorReset+format+"\n", a...)
+}

integration/s3conf.go

@@ -1,17 +1,3 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package integration
 
 import (
@@ -31,17 +17,15 @@ import (
 )
 
 type S3Conf struct {
-	awsID             string
-	awsSecret         string
-	awsRegion         string
-	endpoint          string
-	checksumDisable   bool
-	pathStyle         bool
-	PartSize          int64
-	Concurrency       int
-	debug             bool
-	versioningEnabled bool
-	azureTests        bool
+	awsID           string
+	awsSecret       string
+	awsRegion       string
+	endpoint        string
+	checksumDisable bool
+	pathStyle       bool
+	PartSize        int64
+	Concurrency     int
+	debug           bool
 }
 
 func NewS3Conf(opts ...Option) *S3Conf {
@@ -82,12 +66,6 @@ func WithConcurrency(c int) Option {
 func WithDebug() Option {
 	return func(s *S3Conf) { s.debug = true }
 }
-func WithVersioningEnabled() Option {
-	return func(s *S3Conf) { s.versioningEnabled = true }
-}
-func WithAzureMode() Option {
-	return func(s *S3Conf) { s.azureTests = true }
-}
 
 func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
 	// TODO support token/IAM
@@ -101,8 +79,13 @@ func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
 	return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "")
 }
 
-func (c *S3Conf) GetClient() *s3.Client {
-	return s3.NewFromConfig(c.Config())
+func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               c.endpoint,
+		SigningRegion:     c.awsRegion,
+		HostnameImmutable: true,
+	}, nil
 }
 
 func (c *S3Conf) Config() aws.Config {
@@ -115,7 +98,11 @@ func (c *S3Conf) Config() aws.Config {
 		config.WithRegion(c.awsRegion),
 		config.WithCredentialsProvider(creds),
 		config.WithHTTPClient(client),
-		config.WithRetryMaxAttempts(1),
+	}
+
+	if c.endpoint != "" && c.endpoint != "aws" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(c))
 	}
 
 	if c.checksumDisable {
@@ -134,15 +121,11 @@ func (c *S3Conf) Config() aws.Config {
 		log.Fatalln("error:", err)
 	}
 
-	if c.endpoint != "" && c.endpoint != "aws" {
-		cfg.BaseEndpoint = &c.endpoint
-	}
-
 	return cfg
 }
 
 func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
-	uploader := manager.NewUploader(c.GetClient())
+	uploader := manager.NewUploader(s3.NewFromConfig(c.Config()))
 	uploader.PartSize = c.PartSize
 	uploader.Concurrency = c.Concurrency
 
@@ -157,7 +140,7 @@ func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
 }
 
 func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
-	downloader := manager.NewDownloader(c.GetClient())
+	downloader := manager.NewDownloader(s3.NewFromConfig(c.Config()))
 	downloader.PartSize = c.PartSize
 	downloader.Concurrency = c.Concurrency
 

integration/utils.go

@@ -0,0 +1,549 @@
+package integration
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	rnd "math/rand"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	bcktCount  = 0
+	succUsrCrt = "The user has been created successfully"
+	failUsrCrt = "failed to create a user: update iam data: account already exists"
+)
+
+func getBucketName() string {
+	bcktCount++
+	return fmt.Sprintf("test-bucket-%v", bcktCount)
+}
+
+func setup(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func teardown(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	deleteObject := func(bucket, key, versionId *string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
+			Bucket:    bucket,
+			Key:       key,
+			VersionId: versionId,
+		})
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to delete object %v: %v", *key, err)
+		}
+		return nil
+	}
+
+	in := &s3.ListObjectsV2Input{Bucket: &bucket}
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListObjectsV2(ctx, in)
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to list objects: %v", err)
+		}
+
+		for _, item := range out.Contents {
+			err = deleteObject(&bucket, item.Key, nil)
+			if err != nil {
+				return err
+			}
+		}
+
+		if out.IsTruncated {
+			in.ContinuationToken = out.ContinuationToken
+		} else {
+			break
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
+	runF(testName)
+	bucketName := getBucketName()
+	err := setup(s, bucketName)
+	if err != nil {
+		failF("%v: failed to create a bucket: %v", testName, err.Error())
+		return
+	}
+	client := s3.NewFromConfig(s.Config())
+	handlerErr := handler(client, bucketName)
+	if handlerErr != nil {
+		failF("%v: %v", testName, handlerErr.Error())
+	}
+
+	err = teardown(s, bucketName)
+	if err != nil {
+		if handlerErr == nil {
+			failF("%v: failed to delete the bucket: %v", testName, err.Error())
+		} else {
+			fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
+		}
+	}
+	if handlerErr == nil {
+		passF(testName)
+	}
+}
+
+type authConfig struct {
+	testName string
+	path     string
+	method   string
+	body     []byte
+	service  string
+	date     time.Time
+}
+
+func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
+	runF(cfg.testName)
+	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err.Error())
+		return
+	}
+
+	err = handler(req)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err.Error())
+		return
+	}
+	passF(cfg.testName)
+}
+
+func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
+	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(body)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date)
+	if signErr != nil {
+		return nil, fmt.Errorf("failed to sign the request: %w", signErr)
+	}
+
+	return req, nil
+}
+
+func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var errResp s3err.APIErrorResponse
+	err = xml.Unmarshal(body, &errResp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != apiErr.HTTPStatusCode {
+		return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
+	}
+	if errResp.Code != apiErr.Code {
+		return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
+	}
+	if errResp.Message != apiErr.Description {
+		return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
+	}
+
+	return nil
+}
+
+func checkApiErr(err error, apiErr s3err.APIError) error {
+	if err == nil {
+		return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
+			return nil
+		}
+
+		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+	} else {
+		return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
+	}
+}
+
+func checkSdkApiErr(err error, code string) error {
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() != code {
+			return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
+		}
+		return nil
+	}
+	return err
+}
+
+func putObjects(client *s3.Client, objs []string, bucket string) error {
+	for _, key := range objs {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Key:    &key,
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
+	data = make([]byte, lgth)
+	rand.Read(data)
+	csum = sha256.Sum256(data)
+	r := bytes.NewReader(data)
+	input.Body = r
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err = client.PutObject(ctx, input)
+	cancel()
+
+	return
+}
+
+func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: &bucket,
+		Key:    &key,
+	})
+	cancel()
+	return out, err
+}
+
+func isEqual(a, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i, d := range a {
+		if d != b[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+	for i, item := range list1 {
+		if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareParts(parts1, parts2 []types.Part) bool {
+	if len(parts1) != len(parts2) {
+		return false
+	}
+
+	for i, prt := range parts1 {
+		if prt.PartNumber != parts2[i].PartNumber {
+			return false
+		}
+		if *prt.ETag != *parts2[i].ETag {
+			return false
+		}
+	}
+	return true
+}
+
+func areTagsSame(tags1, tags2 []types.Tag) bool {
+	if len(tags1) != len(tags2) {
+		return false
+	}
+
+	for _, tag := range tags1 {
+		if !containsTag(tag, tags2) {
+			return false
+		}
+	}
+	return true
+}
+
+func containsTag(tag types.Tag, list []types.Tag) bool {
+	for _, item := range list {
+		if *item.Key == *tag.Key && *item.Value == *tag.Value {
+			return true
+		}
+	}
+	return false
+}
+
+func compareGrants(grts1, grts2 []types.Grant) bool {
+	if len(grts1) != len(grts2) {
+		return false
+	}
+
+	for i, grt := range grts1 {
+		if grt.Permission != grts2[i].Permission {
+			return false
+		}
+		if *grt.Grantee.ID != *grts2[i].Grantee.ID {
+			return false
+		}
+	}
+	return true
+}
+
+func execCommand(args ...string) ([]byte, error) {
+	cmd := exec.Command("./versitygw", args...)
+
+	return cmd.CombinedOutput()
+}
+
+func getString(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
+}
+
+func getPtr(str string) *string {
+	return &str
+}
+
+func areMapsSame(mp1, mp2 map[string]string) bool {
+	if len(mp1) != len(mp2) {
+		return false
+	}
+	for key, val := range mp1 {
+		if mp2[key] != val {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[*elem.Name] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[elem.Name]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareObjects(list1 []string, list2 []types.Object) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Prefix]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
+	dr := NewDataReader(size, size)
+	datafile := "rand.data"
+	w, err := os.Create(datafile)
+	if err != nil {
+		return parts, err
+	}
+	defer w.Close()
+
+	_, err = io.Copy(w, dr)
+	if err != nil {
+		return parts, err
+	}
+
+	fileInfo, err := w.Stat()
+	if err != nil {
+		return parts, err
+	}
+
+	partSize := fileInfo.Size() / int64(partCount)
+	var offset int64
+
+	for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
+		partStart := (partNumber - 1) * partSize
+		partEnd := partStart + partSize - 1
+		if partEnd > fileInfo.Size()-1 {
+			partEnd = fileInfo.Size() - 1
+		}
+		partBuffer := make([]byte, partEnd-partStart+1)
+		_, err := w.ReadAt(partBuffer, partStart)
+		if err != nil {
+			return parts, err
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     &bucket,
+			Key:        &key,
+			UploadId:   &uploadId,
+			Body:       bytes.NewReader(partBuffer),
+			PartNumber: int32(partNumber),
+		})
+		cancel()
+		if err != nil {
+			return parts, err
+		} else {
+			parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
+			offset += partSize
+		}
+	}
+
+	return parts, err
+}
+
+type user struct {
+	access string
+	secret string
+	role   string
+}
+
+func createUsers(s *S3Conf, users []user) error {
+	for _, usr := range users {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
+			return fmt.Errorf("failed to create a user account")
+		}
+	}
+	return nil
+}
+
+func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
+	for _, bucket := range buckets {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
+			return fmt.Errorf(string(out))
+		}
+	}
+
+	return nil
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func genRandString(length int) string {
+	source := rnd.NewSource(time.Now().UnixNano())
+	random := rnd.New(source)
+	result := make([]byte, length)
+	for i := range result {
+		result[i] = charset[random.Intn(len(charset))]
+	}
+	return string(result)
+}