feat: posix read permission check wip

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,23 +1,27 @@
 ---
-name: Bug Report
+name: Bug report
 about: Create a report to help us improve
-title: '[Bug] - <Short Description>'
+title: ''
 labels: bug
 assignees: ''
 
 ---
 
 **Describe the bug**
-<!-- A clear and concise description of what the bug is. -->
+A clear and concise description of what the bug is.
 
 **To Reproduce**
-<!-- Steps to reproduce the behavior. -->
+Steps to reproduce the behavior.
 
 **Expected behavior**
-<!-- A clear and concise description of what you expected to happen. -->
+A clear and concise description of what you expected to happen.
 
 **Server Version**
-<!-- output of: './versitygw -version && uname -a'  -->
+ output of
+```
+./versitygw -version
+uname -a
+```
 
 **Additional context**
-<!-- Describe s3 client and version if applicable.
+Describe s3 client and version if applicable.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,14 +1,14 @@
 ---
-name: Feature Request
+name: Feature request
 about: Suggest an idea for this project
-title: '[Feature] - <Short Description>'
+title: ''
 labels: enhancement
 assignees: ''
 
 ---
 
 **Describe the solution you'd like**
-<!-- A clear and concise description of what you want to happen. -->
+A clear and concise description of what you want to happen.
 
 **Additional context**
-<!-- Add any other context or screenshots about the feature request here. -->
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/test_case.md

@@ -1,33 +0,0 @@
----
-name: Test Case Request
-about: Request new test cases or additional test coverage
-title: '[Test Case] - <Short Description>'
-labels: 'testcase'
-assignees: ''
-
----
-
-## Description
-<!-- Please provide a detailed description of the test case or test coverage request. -->
-
-## Purpose
-<!-- Explain why this test case is important and what it aims to achieve. -->
-
-## Scope
-<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
-
-## Acceptance Criteria
-<!-- List the criteria that must be met for the test case to be considered complete. -->
-
-1. 
-2. 
-3. 
-
-## Additional Context
-<!-- Add any other context or screenshots about the feature request here. -->
-
-## Resources
-<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
-
-
-**Thank you for contributing to our project!**

.github/dependabot.yml

@@ -8,7 +8,3 @@ updates:
       dev-dependencies:
         patterns:
           - "*"
-    allow:
-      # Allow both direct and indirect updates for all packages
-      - dependency-type: "all"
-

.github/workflows/azurite.yml

@@ -1,37 +0,0 @@
-name: azurite functional tests
-
-on: pull_request
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-              
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 'stable'
-        id: go
-
-      - name: Set up Docker Compose
-        run: |
-          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
-
-      - name: Wait for Azurite to be ready
-        run: sleep 40
-
-      - name: Get Dependencies
-        run: |
-          go mod download
-  
-      - name: Build and Run
-        run: |
-          make
-          ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
-  
-      - name: Shut down services
-        run: |
-          docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

.github/workflows/docker-bats.yml

@@ -1,29 +0,0 @@
-name: docker bats tests
-
-on: pull_request
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Build Docker Image
-        run: |
-          cp tests/.env.docker.default tests/.env.docker
-          cp tests/.secrets.default tests/.secrets
-          # see https://github.com/versity/versitygw/issues/1034
-          docker build \
-            --build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
-            --build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
-            --build-arg="MC_FOLDER=linux-amd64" \
-            --progress=plain \
-            -f tests/Dockerfile_test_bats \
-            -t bats_test .
-
-      - name: Run Docker Container
-        run: |
-          docker compose -f tests/docker-compose-bats.yml --project-directory . \
-          up --exit-code-from s3api_np_only s3api_np_only

.github/workflows/docker.yaml

@@ -12,16 +12,9 @@ jobs:
       packages: write
       contents: read
     steps:
-      - name: Checkout
+      - name: Check out the repo
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Log in to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -50,8 +43,3 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            VERSION=${{ github.event.release.tag_name }}
-            TIME=${{ github.event.release.published_at }}
-            BUILD=${{ github.sha }}

.github/workflows/functional.yml

@@ -1,25 +1,24 @@
 name: functional tests
-
 on: pull_request
-
 jobs:
+
   build:
     name: RunTests
     runs-on: ubuntu-latest
     steps:
 
-    - name: Checkout
-      uses: actions/checkout@v4
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
 
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
 
     - name: Get Dependencies
       run: |
-        go mod download
+        go get -v -t -d ./...
 
     - name: Build and Run
       run: |

.github/workflows/go.yml

@@ -8,10 +8,10 @@ jobs:
     steps:
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
 
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go

.github/workflows/goreleaser.yml

@@ -15,21 +15,14 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
-      - name: Fetch tags
-        run: git fetch --force --tags
-
-      - name: Setup Go
-        uses: actions/setup-go@v5
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v4
         with:
           go-version: stable
-
-      - name: Run Releaser
-        uses: goreleaser/goreleaser-action@v5
+      - uses: goreleaser/goreleaser-action@v4
         with:
           distribution: goreleaser
           version: latest

.github/workflows/shellcheck.yml

@@ -1,16 +0,0 @@
-name: shellcheck
-on: pull_request
-jobs:
-
-  build:
-    name: Run shellcheck
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Check out code
-      uses: actions/checkout@v4
-
-    - name: Run checks
-      run: |
-        shellcheck --version
-        shellcheck -e SC1091 tests/*.sh tests/*/*.sh

.github/workflows/static.yml

@@ -7,18 +7,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 1
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
 
     - name: Set up Go
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v4
       with:
         go-version: 'stable'
       id: go
  
     - name: "staticcheck"
-      uses: dominikh/staticcheck-action@v1
-      with:
-        version: "latest"
+      uses: dominikh/staticcheck-action@v1.3.0
+      with: 
+        install-go: false

.github/workflows/system.yml

@@ -4,107 +4,18 @@ jobs:
   build:
     name: RunTests
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - set: "mc, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "mc-non-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "mc, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "mc-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "REST, posix, non-static, all, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-non-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, user, non-static, s3 IAM"
-            IAM_TYPE: s3
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, multipart, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-multipart"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, object, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-object"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, user, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          # TODO fix/debug s3 gateway
-          #- set: "s3api, s3, multipart|object, non-static, folder IAM"
-          #  IAM_TYPE: folder
-          #  RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-          #  RECREATE_BUCKETS: "true"
-          #  BACKEND: "s3"
-          #- set: "s3api, s3, policy|user, non-static, folder IAM"
-          #  IAM_TYPE: folder
-          #  RUN_SET: "s3api-policy,s3api-user"
-          #  RECREATE_BUCKETS: "true"
-          #  BACKEND: "s3"
-          - set: "s3cmd, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, non-user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-non-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
+
+      - name: Install ShellCheck
+        run: sudo apt-get install shellcheck
+
+      - name: Run ShellCheck
+        run: shellcheck -S warning ./tests/*.sh
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
         id: go
@@ -117,8 +28,6 @@ jobs:
         run: |
           git clone https://github.com/bats-core/bats-core.git
           cd bats-core && ./install.sh $HOME
-          git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
-          git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
 
       - name: Install s3cmd
         run: |
@@ -129,74 +38,18 @@ jobs:
           curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
           chmod 755 /usr/local/bin/mc
 
-      - name: Install xmllint (for rest)
-        run: |
-          sudo apt-get install libxml2-utils
-
-      # see https://github.com/versity/versitygw/issues/1034
-      - name: Install AWS cli
-        run: |
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" 
-          unzip -o awscliv2.zip 
-          ./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
-          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-
       - name: Build and run
-        env:
-          IAM_TYPE: ${{ matrix.IAM_TYPE }}
-          RUN_SET: ${{ matrix.RUN_SET }}
-          AWS_PROFILE: versity
-          VERSITY_EXE: ${{ github.workspace }}/versitygw
-          RUN_VERSITYGW: true
-          BACKEND: ${{ matrix.BACKEND }}
-          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
-          CERT: ${{ github.workspace }}/cert.pem
-          KEY: ${{ github.workspace }}/versitygw.pem
-          LOCAL_FOLDER: /tmp/gw
-          BUCKET_ONE_NAME: versity-gwtest-bucket-one
-          BUCKET_TWO_NAME: versity-gwtest-bucket-two
-          USERS_FOLDER: /tmp/iam
-          USERS_BUCKET: versity-gwtest-iam
-          AWS_ENDPOINT_URL: https://127.0.0.1:7070
-          PORT: 7070
-          S3CMD_CONFIG: tests/s3cfg.local.default
-          MC_ALIAS: versity
-          LOG_LEVEL: 4
-          GOCOVERDIR: ${{ github.workspace }}/cover
-          USERNAME_ONE: ABCDEFG
-          PASSWORD_ONE: 1234567
-          USERNAME_TWO: HIJKLMN
-          PASSWORD_TWO: 8901234
-          TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
-          REMOVE_TEST_FILE_FOLDER: true
-          VERSIONING_DIR: ${{ github.workspace }}/versioning
-          COMMAND_LOG: command.log
-          TIME_LOG: time.log
-          PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
         run: |
           make testbin
+          echo $PATH
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
-          export AWS_ACCESS_KEY_ID_TWO=user
-          export AWS_SECRET_ACCESS_KEY_TWO=pass
-          export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity
-          mkdir $LOCAL_FOLDER
+          mkdir /tmp/gw
           export WORKSPACE=$GITHUB_WORKSPACE
-          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
-          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir $GOCOVERDIR $USERS_FOLDER
-          if [[ $RECREATE_BUCKETS == "false" ]]; then
-            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
-          fi
-          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
-
-      - name: Time report
-        run: cat ${{ github.workspace }}/time.log
-
-      - name: Coverage report
-        run: |
-          go tool covdata percent -i=cover
+          openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
+          openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh

.gitignore

@@ -41,29 +41,11 @@ VERSION
 dist/
 
 # secrets file for local github-actions testing
-tests/.secrets*
+tests/.secrets
 
 # IAM users files often created in testing
 users.json
-users.json.backup
 
 # env files for testing
-**/.env*
-**/!.env.default
-
-# s3cmd config files (testing)
-tests/s3cfg.local*
-tests/!s3cfg.local.default
-
-# keys
-*.pem
-
-# patches
-*.patch
-
-# grafana's local database (kept on filesystem for survival between instantiations)
-metrics-exploration/grafana_data/**
-
-# bats tools
-/tests/bats-assert
-/tests/bats-support
+.env*
+!.env.default

.goreleaser.yaml

@@ -6,7 +6,6 @@ builds:
   - goos:
       - linux
       - darwin
-      - freebsd
       # windows is untested, we can start doing windows releases
       # if someone is interested in taking on testing
       # - windows
@@ -32,28 +31,11 @@ archives:
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
-
-    # Set this to true if you want all files in the archive to be in a single directory.
-    # If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
-    # you'll get a folder 'goreleaser_Linux_arm64'.
-    # If set to false, all files are extracted separately.
-    # You can also set it to a custom folder name (templating is supported).
-    wrap_in_directory: true
-
     # use zip for windows archives
     format_overrides:
     - goos: windows
       format: zip
 
-    # Additional files/globs you want to add to the archive.
-    #
-    # Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
-    # Templates: allowed
-    files:
-      - README.md
-      - LICENSE
-      - NOTICE
-
 checksum:
   name_template: 'checksums.txt'
 
@@ -100,27 +82,6 @@ nfpms:
 
     rpm:
       group: "System Environment/Daemons"
-      # RPM specific scripts.
-      scripts:
-        # The pretrans script runs before all RPM package transactions / stages.
-        #pretrans: ./extra/pretrans.sh
-        # The posttrans script runs after all RPM package transactions / stages.
-        posttrans: ./extra/posttrans.sh
-
-    contents:
-      - src: extra/versitygw@.service
-        dst: /lib/systemd/system/versitygw@.service
-
-      - src: extra/example.conf
-        dst: /etc/versitygw.d/example.conf
-        type: config
-  
-      - dst: /etc/versitygw.d
-        type: dir
-        file_info:
-          mode: 0700
-
-
 
 # yaml-language-server: $schema=https://goreleaser.com/static/schema.json
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile

@@ -1,15 +1,5 @@
 FROM golang:latest
 
-# Set build arguments with default values
-ARG VERSION="none"
-ARG BUILD="none"
-ARG TIME="none"
-
-# Set environment variables
-ENV VERSION=${VERSION}
-ENV BUILD=${BUILD}
-ENV TIME=${TIME}
-
 WORKDIR /app
 
 COPY go.mod ./
@@ -19,7 +9,7 @@ COPY ./ ./
 
 WORKDIR /app/cmd/versitygw
 ENV CGO_ENABLED=0
-RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
+RUN go build -o versitygw
 
 FROM alpine:latest
 
@@ -32,4 +22,4 @@ RUN mkdir -p $SETUP_DIR
 
 COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
 
-ENTRYPOINT [ "/app/versitygw" ]
+ENTRYPOINT [ "/app/versitygw" ]

Dockerfile.dev

@@ -6,7 +6,7 @@ COPY go.mod ./
 RUN go mod download
 
 COPY ./ ./
-COPY ./tests/certs/* /etc/pki/tls/certs/
+COPY certs/* /etc/pki/tls/certs/
 
 ARG IAM_DIR=/tmp/vgw
 ARG SETUP_DIR=/tmp/vgw

Dockerfile_test_bats

@@ -1,13 +1,6 @@
-FROM ubuntu:latest
+FROM --platform=linux/arm64 ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG SECRETS_FILE=tests/.secrets
-ARG CONFIG_FILE=tests/.env.docker
-ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz
-# see https://github.com/versity/versitygw/issues/1034
-ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip
-ARG MC_FOLDER=linux-arm64
-
 ENV TZ=Etc/UTC
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -19,8 +12,6 @@ RUN apt-get update && \
     tzdata \
     s3cmd \
     jq \
-    bc \
-    libxml2-utils \
     ca-certificates && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/*
@@ -29,20 +20,20 @@ RUN apt-get update && \
 WORKDIR /tmp
 
 # Install AWS cli
-RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
 
 # Install mc
-RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
+RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
   --create-dirs \
   -o /usr/local/minio-binaries/mc && \
 chmod -R 755 /usr/local/minio-binaries
-ENV PATH=/usr/local/minio-binaries:${PATH}
+ENV PATH="/usr/local/minio-binaries":${PATH}
 
 # Download Go 1.21 (adjust the version and platform as needed)
-RUN wget https://golang.org/dl/${GO_LIBRARY}
+RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
 
 # Extract the downloaded archive
-RUN tar -xvf $GO_LIBRARY -C /usr/local
+RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
 
 # Set Go environment variables
 ENV PATH="/usr/local/go/bin:${PATH}"
@@ -65,14 +56,12 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
 USER tester
 COPY --chown=tester:tester . /home/tester
 
-# add bats support libraries
-RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
-RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
-
 WORKDIR /home/tester
+RUN cp tests/.env.docker.default tests/.env.docker
+RUN cp tests/s3cfg.local.default tests/s3cfg.local
 RUN make
 
-RUN . $SECRETS_FILE && \
+RUN . tests/.secrets && \
     export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
@@ -85,8 +74,6 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
         -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
 
 ENV WORKSPACE=.
-ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
-#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
+ENV VERSITYGW_TEST_ENV=tests/.env.docker
 
-ENTRYPOINT ["tests/run.sh"]
-CMD ["s3api,s3,s3cmd,mc,rest"]
+CMD ["tests/run_all.sh"]

Makefile

@@ -18,10 +18,6 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 
-# docker-compose
-DCCMD=docker-compose
-DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
-
 BIN=versitygw
 
 VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -63,31 +59,38 @@ cleanall: clean
 	rm -f $(BIN)
 	rm -f versitygw-*.tar
 	rm -f versitygw-*.tar.gz
+	rm -f versitygw.spec
+
+%.spec: %.spec.in
+	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
+	mv $@+ $@
 
 TARFILE = $(BIN)-$(VERSION).tar
 
-dist:
+dist: $(BIN).spec
 	echo $(VERSION) >VERSION
 	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
+	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
 	rm -f VERSION
+	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
 
 # Creates and runs S3 gateway instance in a docker container
 .PHONY: up-posix
 up-posix:
-	$(DOCKERCOMPOSE) up posix
+	docker compose --env-file .env.dev up posix
 
 # Creates and runs S3 gateway proxy instance in a docker container
 .PHONY: up-proxy
 up-proxy:
-	$(DOCKERCOMPOSE) up proxy
+	docker compose --env-file .env.dev up proxy
 
 # Creates and runs S3 gateway to azurite instance in a docker container
 .PHONY: up-azurite
 up-azurite:
-	$(DOCKERCOMPOSE) up azurite azuritegw
+	docker compose --env-file .env.dev up azurite azuritegw
 
 # Creates and runs both S3 gateway and proxy server instances in docker containers
 .PHONY: up-app
 up-app:
-	$(DOCKERCOMPOSE) up
+	docker compose --env-file .env.dev up

README.md

@@ -6,41 +6,24 @@
   <a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
 </picture>
 
- [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
+ [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)  
 
-### Binary release builds
-Download [latest release](https://github.com/versity/versitygw/releases)
- | Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
- |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
- |    ✔️    |  ✔️  |   ✔️   |  ✔️   |  ✔️   |  ✔️   |
- 
-### Use Cases
-* Turn your local filesystem into an S3 server with a single command!
-* Proxy S3 requests to S3 storage
-* Simple to deploy S3 server with a single command
-* Protocol compatibility in `posix` allows common access to files via posix or S3
-* Simplified interface for adding new storage system support
+**News:**<br>
+* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
 
-### News
-Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
 
-### Mailing List
-Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
-
-### Documentation
 See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
 
-### Need help?
-Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
-<br>
-Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
+* Share filesystem directory via S3 protocol
+* Proxy S3 requests to S3 storage
+* Simple to deploy S3 server with a single command
+* Protocol compatibility in `posix` allows common access to files via posix or S3 
 
-### Overview
 Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
 
 The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
 
-The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versity’s open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.  
+The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.  
 
 The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance. 
 
@@ -68,7 +51,7 @@ The command format is
 ```
 versitygw [global options] command [command options] [arguments...]
 ```
-The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
+The global options are specified before the backend type and the backend options are specified after.
 
 ***
 

auth/acl.go

@@ -15,170 +15,37 @@
 package auth
 
 import (
-	"context"
 	"encoding/json"
-	"encoding/xml"
-	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 )
 
 type ACL struct {
+	ACL      types.BucketCannedACL
 	Owner    string
 	Grantees []Grantee
 }
 
 type Grantee struct {
-	Permission Permission
+	Permission types.Permission
 	Access     string
-	Type       types.Type
 }
 
 type GetBucketAclOutput struct {
-	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
 	Owner             *types.Owner
 	AccessControlList AccessControlList
 }
 
-type PutBucketAclInput struct {
-	Bucket              *string
-	ACL                 types.BucketCannedACL
-	AccessControlPolicy *AccessControlPolicy
-	GrantFullControl    *string
-	GrantRead           *string
-	GrantReadACP        *string
-	GrantWrite          *string
-	GrantWriteACP       *string
+type AccessControlList struct {
+	Grants []types.Grant `xml:"Grant"`
 }
-
 type AccessControlPolicy struct {
 	AccessControlList AccessControlList `xml:"AccessControlList"`
-	Owner             *types.Owner
-}
-
-func (acp *AccessControlPolicy) Validate() error {
-	if !acp.AccessControlList.isValid() {
-		return s3err.GetAPIError(s3err.ErrMalformedACL)
-	}
-
-	// The Owner can't be nil
-	if acp.Owner == nil {
-		return s3err.GetAPIError(s3err.ErrMalformedACL)
-	}
-
-	// The Owner ID can't be empty
-	if acp.Owner.ID == nil || *acp.Owner.ID == "" {
-		return s3err.GetAPIError(s3err.ErrMalformedACL)
-	}
-
-	return nil
-}
-
-type AccessControlList struct {
-	Grants []Grant `xml:"Grant"`
-}
-
-// Validates the AccessControlList
-func (acl *AccessControlList) isValid() bool {
-	for _, el := range acl.Grants {
-		if !el.isValid() {
-			return false
-		}
-	}
-
-	return true
-}
-
-type Permission string
-
-const (
-	PermissionFullControl Permission = "FULL_CONTROL"
-	PermissionWrite       Permission = "WRITE"
-	PermissionWriteAcp    Permission = "WRITE_ACP"
-	PermissionRead        Permission = "READ"
-	PermissionReadAcp     Permission = "READ_ACP"
-)
-
-// Check if the permission is valid
-func (p Permission) isValid() bool {
-	return p == PermissionFullControl ||
-		p == PermissionRead ||
-		p == PermissionReadAcp ||
-		p == PermissionWrite ||
-		p == PermissionWriteAcp
-}
-
-type Grant struct {
-	Grantee    *Grt       `xml:"Grantee"`
-	Permission Permission `xml:"Permission"`
-}
-
-// Checks if Grant is valid
-func (g *Grant) isValid() bool {
-	return g.Permission.isValid() && g.Grantee.isValid()
-}
-
-type Grt struct {
-	XMLNS string     `xml:"xmlns:xsi,attr"`
-	Type  types.Type `xml:"xsi:type,attr"`
-	ID    string     `xml:"ID"`
-}
-
-// Custom Unmarshalling for Grt to parse xsi:type properly
-func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
-	// Iterate through the XML tokens to process the attributes
-	for _, attr := range start.Attr {
-		// Check if the attribute is xsi:type and belongs to the xsi namespace
-		if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
-			g.Type = types.Type(attr.Value)
-		}
-		// Handle xmlns:xsi
-		if attr.Name.Local == "xmlns:xsi" {
-			g.XMLNS = attr.Value
-		}
-	}
-
-	// Decode the inner XML elements like ID
-	for {
-		t, err := d.Token()
-		if err != nil {
-			return err
-		}
-
-		switch se := t.(type) {
-		case xml.StartElement:
-			if se.Name.Local == "ID" {
-				if err := d.DecodeElement(&g.ID, &se); err != nil {
-					return err
-				}
-			}
-		case xml.EndElement:
-			if se.Name.Local == start.Name.Local {
-				return nil
-			}
-		}
-	}
-}
-
-// Validates Grt
-func (g *Grt) isValid() bool {
-	// Validate the Type
-	// Only these 2 types are supported in the gateway
-	if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
-		return false
-	}
-
-	// The ID prop shouldn't be empty
-	if g.ID == "" {
-		return false
-	}
-
-	return true
+	Owner             types.Owner
 }
 
 func ParseACL(data []byte) (ACL, error) {
@@ -193,35 +60,17 @@ func ParseACL(data []byte) (ACL, error) {
 	return acl, nil
 }
 
-func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
-	grants := []Grant{}
-
-	if len(data) == 0 {
-		return GetBucketAclOutput{
-			Owner: &types.Owner{
-				ID: &owner,
-			},
-			AccessControlList: AccessControlList{
-				Grants: grants,
-			},
-		}, nil
-	}
-
+func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
 	var acl ACL
 	if err := json.Unmarshal(data, &acl); err != nil {
 		return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
 	}
 
+	grants := []types.Grant{}
+
 	for _, elem := range acl.Grantees {
 		acs := elem.Access
-		grants = append(grants, Grant{
-			Grantee: &Grt{
-				XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
-				ID:    acs,
-				Type:  elem.Type,
-			},
-			Permission: elem.Permission,
-		})
+		grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
 	}
 
 	return GetBucketAclOutput{
@@ -234,43 +83,20 @@ func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
 	}, nil
 }
 
-func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
+func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
 	if input == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
-
-	defaultGrantees := []Grantee{
-		{
-			Permission: PermissionFullControl,
-			Access:     acl.Owner,
-			Type:       types.TypeCanonicalUser,
-		},
+	if acl.Owner != *input.AccessControlPolicy.Owner.ID {
+		return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
 	}
 
 	// if the ACL is specified, set the ACL, else replace the grantees
 	if input.ACL != "" {
-		switch input.ACL {
-		case types.BucketCannedACLPublicRead:
-			defaultGrantees = append(defaultGrantees, Grantee{
-				Permission: PermissionRead,
-				Access:     "all-users",
-				Type:       types.TypeGroup,
-			})
-		case types.BucketCannedACLPublicReadWrite:
-			defaultGrantees = append(defaultGrantees, []Grantee{
-				{
-					Permission: PermissionRead,
-					Access:     "all-users",
-					Type:       types.TypeGroup,
-				},
-				{
-					Permission: PermissionWrite,
-					Access:     "all-users",
-					Type:       types.TypeGroup,
-				},
-			}...)
-		}
+		acl.ACL = input.ACL
+		acl.Grantees = []Grantee{}
 	} else {
+		grantees := []Grantee{}
 		accs := []string{}
 
 		if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -279,71 +105,45 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 			if input.GrantFullControl != nil && *input.GrantFullControl != "" {
 				fullControlList = splitUnique(*input.GrantFullControl, ",")
 				for _, str := range fullControlList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: PermissionFullControl,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
 				}
 			}
 			if input.GrantRead != nil && *input.GrantRead != "" {
 				readList = splitUnique(*input.GrantRead, ",")
 				for _, str := range readList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: PermissionRead,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
 				}
 			}
 			if input.GrantReadACP != nil && *input.GrantReadACP != "" {
 				readACPList = splitUnique(*input.GrantReadACP, ",")
 				for _, str := range readACPList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: PermissionReadAcp,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
 				}
 			}
 			if input.GrantWrite != nil && *input.GrantWrite != "" {
 				writeList = splitUnique(*input.GrantWrite, ",")
 				for _, str := range writeList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: PermissionWrite,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
 				}
 			}
 			if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
 				writeACPList = splitUnique(*input.GrantWriteACP, ",")
 				for _, str := range writeACPList {
-					defaultGrantees = append(defaultGrantees, Grantee{
-						Access:     str,
-						Permission: PermissionWriteAcp,
-						Type:       types.TypeCanonicalUser,
-					})
+					grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
 				}
 			}
 
 			accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
 		} else {
 			cache := make(map[string]bool)
-			for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
-				if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
+			for _, grt := range input.AccessControlPolicy.Grants {
+				if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
 					return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 				}
-
-				access := grt.Grantee.ID
-				defaultGrantees = append(defaultGrantees, Grantee{
-					Access:     access,
-					Permission: grt.Permission,
-					Type:       types.TypeCanonicalUser,
-				})
-				if _, ok := cache[access]; !ok {
-					cache[access] = true
-					accs = append(accs, access)
+				grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
+				if _, ok := cache[*grt.Grantee.ID]; !ok {
+					cache[*grt.Grantee.ID] = true
+					accs = append(accs, *grt.Grantee.ID)
 				}
 			}
 		}
@@ -356,9 +156,10 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
 		if len(accList) > 0 {
 			return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
 		}
-	}
 
-	acl.Grantees = defaultGrantees
+		acl.Grantees = grantees
+		acl.ACL = ""
+	}
 
 	result, err := json.Marshal(acl)
 	if err != nil {
@@ -378,8 +179,8 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
 				result = append(result, acc)
 				continue
 			}
-			if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
-				return nil, err
+			if err == ErrNotSupported {
+				return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 			}
 			return nil, fmt.Errorf("check user account: %w", err)
 		}
@@ -402,36 +203,42 @@ func splitUnique(s, divider string) []string {
 	return result
 }
 
-func verifyACL(acl ACL, access string, permission Permission) error {
-	grantee := Grantee{
-		Access:     access,
-		Permission: permission,
-		Type:       types.TypeCanonicalUser,
-	}
-	granteeFullCtrl := Grantee{
-		Access:     access,
-		Permission: PermissionFullControl,
-		Type:       types.TypeCanonicalUser,
-	}
-	granteeAllUsers := Grantee{
-		Access:     "all-users",
-		Permission: permission,
-		Type:       types.TypeGroup,
-	}
-
-	isFound := false
-
-	for _, grt := range acl.Grantees {
-		if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
-			isFound = true
-			break
-		}
-	}
-
-	if isFound {
+func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool) error {
+	if isRoot {
 		return nil
 	}
 
+	if acl.Owner == access {
+		return nil
+	}
+
+	if acl.ACL != "" {
+		if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+		if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
+			return s3err.GetAPIError(s3err.ErrAccessDenied)
+		}
+
+		return nil
+	} else {
+		grantee := Grantee{Access: access, Permission: permission}
+		granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
+
+		isFound := false
+
+		for _, grt := range acl.Grantees {
+			if grt == grantee || grt == granteeFullCtrl {
+				isFound = true
+				break
+			}
+		}
+
+		if isFound {
+			return nil
+		}
+	}
+
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
 
@@ -466,87 +273,3 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 	// Return access denied in all other cases
 	return s3err.GetAPIError(s3err.ErrAccessDenied)
 }
-
-type AccessOptions struct {
-	Acl           ACL
-	AclPermission Permission
-	IsRoot        bool
-	Acc           Account
-	Bucket        string
-	Object        string
-	Action        Action
-	Readonly      bool
-}
-
-func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
-	if opts.Readonly {
-		if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
-			return s3err.GetAPIError(s3err.ErrAccessDenied)
-		}
-	}
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
-		return nil
-	}
-
-	policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
-	if policyErr != nil {
-		if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-			return policyErr
-		}
-	} else {
-		return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
-	}
-
-	if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
-	if opts.IsRoot {
-		return nil
-	}
-	if opts.Acc.Role == RoleAdmin {
-		return nil
-	}
-
-	// Verify destination bucket access
-	if err := VerifyAccess(ctx, be, opts); err != nil {
-		return err
-	}
-	// Verify source bucket access
-	srcBucket, srcObject, found := strings.Cut(copySource, "/")
-	if !found {
-		return s3err.GetAPIError(s3err.ErrInvalidCopySource)
-	}
-
-	// Get source bucket ACL
-	srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
-	if err != nil {
-		return err
-	}
-
-	var srcBucketAcl ACL
-	if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
-		return err
-	}
-
-	if err := VerifyAccess(ctx, be, AccessOptions{
-		Acl:           srcBucketAcl,
-		AclPermission: PermissionRead,
-		IsRoot:        opts.IsRoot,
-		Acc:           opts.Acc,
-		Bucket:        srcBucket,
-		Object:        srcObject,
-		Action:        GetObjectAction,
-	}); err != nil {
-		return err
-	}
-
-	return nil
-}

auth/bucket_policy.go

@@ -1,185 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"errors"
-	"net/http"
-
-	"github.com/versity/versitygw/s3err"
-)
-
-type policyErr string
-
-func (p policyErr) Error() string {
-	return string(p)
-}
-
-const (
-	policyErrResourceMismatch     = policyErr("Action does not apply to any resource(s) in statement")
-	policyErrInvalidResource      = policyErr("Policy has invalid resource")
-	policyErrInvalidPrincipal     = policyErr("Invalid principal in policy")
-	policyErrInvalidAction        = policyErr("Policy has invalid action")
-	policyErrInvalidPolicy        = policyErr("This policy contains invalid Json")
-	policyErrInvalidFirstChar     = policyErr("Policies must be valid JSON and the first byte must be '{'")
-	policyErrEmptyStatement       = policyErr("Could not parse the policy: Statement is empty!")
-	policyErrMissingStatmentField = policyErr("Missing required field Statement")
-)
-
-type BucketPolicy struct {
-	Statement []BucketPolicyItem `json:"Statement"`
-}
-
-func (bp *BucketPolicy) UnmarshalJSON(data []byte) error {
-	var tmp struct {
-		Statement *[]BucketPolicyItem `json:"Statement"`
-	}
-
-	if err := json.Unmarshal(data, &tmp); err != nil {
-		return err
-	}
-
-	// If Statement is nil (not present in JSON), return an error
-	if tmp.Statement == nil {
-		return policyErrMissingStatmentField
-	}
-
-	// Assign the parsed value to the actual struct
-	bp.Statement = *tmp.Statement
-	return nil
-}
-
-func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
-	for _, statement := range bp.Statement {
-		err := statement.Validate(bucket, iam)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
-	var isAllowed bool
-	for _, statement := range bp.Statement {
-		if statement.findMatch(principal, action, resource) {
-			switch statement.Effect {
-			case BucketPolicyAccessTypeAllow:
-				isAllowed = true
-			case BucketPolicyAccessTypeDeny:
-				return false
-			}
-		}
-	}
-
-	return isAllowed
-}
-
-type BucketPolicyItem struct {
-	Effect     BucketPolicyAccessType `json:"Effect"`
-	Principals Principals             `json:"Principal"`
-	Actions    Actions                `json:"Action"`
-	Resources  Resources              `json:"Resource"`
-}
-
-func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
-	if err := bpi.Effect.Validate(); err != nil {
-		return err
-	}
-	if err := bpi.Principals.Validate(iam); err != nil {
-		return err
-	}
-	if err := bpi.Resources.Validate(bucket); err != nil {
-		return err
-	}
-
-	containsObjectAction := bpi.Resources.ContainsObjectPattern()
-	containsBucketAction := bpi.Resources.ContainsBucketPattern()
-
-	for action := range bpi.Actions {
-		isObjectAction := action.IsObjectAction()
-		if isObjectAction == nil {
-			break
-		}
-		if *isObjectAction && !containsObjectAction {
-			return policyErrResourceMismatch
-		}
-		if !*isObjectAction && !containsBucketAction {
-			return policyErrResourceMismatch
-		}
-	}
-
-	return nil
-}
-
-func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
-	if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
-		return true
-	}
-
-	return false
-}
-
-func getMalformedPolicyError(err error) error {
-	return s3err.APIError{
-		Code:           "MalformedPolicy",
-		Description:    err.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	}
-}
-
-func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
-	if len(policyBin) == 0 || policyBin[0] != '{' {
-		return getMalformedPolicyError(policyErrInvalidFirstChar)
-	}
-	var policy BucketPolicy
-	if err := json.Unmarshal(policyBin, &policy); err != nil {
-		var pe policyErr
-		if errors.As(err, &pe) {
-			return getMalformedPolicyError(err)
-		}
-		return getMalformedPolicyError(policyErrInvalidPolicy)
-	}
-
-	if len(policy.Statement) == 0 {
-		return getMalformedPolicyError(policyErrEmptyStatement)
-	}
-
-	if err := policy.Validate(bucket, iam); err != nil {
-		return getMalformedPolicyError(err)
-	}
-
-	return nil
-}
-
-func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
-	var bucketPolicy BucketPolicy
-	if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
-		return err
-	}
-
-	resource := bucket
-	if object != "" {
-		resource += "/" + object
-	}
-
-	if !bucketPolicy.isAllowed(access, action, resource) {
-		return s3err.GetAPIError(s3err.ErrAccessDenied)
-	}
-
-	return nil
-}

auth/bucket_policy_actions.go

@@ -1,254 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"strings"
-)
-
-type Action string
-
-const (
-	GetBucketAclAction                     Action = "s3:GetBucketAcl"
-	CreateBucketAction                     Action = "s3:CreateBucket"
-	PutBucketAclAction                     Action = "s3:PutBucketAcl"
-	DeleteBucketAction                     Action = "s3:DeleteBucket"
-	PutBucketVersioningAction              Action = "s3:PutBucketVersioning"
-	GetBucketVersioningAction              Action = "s3:GetBucketVersioning"
-	PutBucketPolicyAction                  Action = "s3:PutBucketPolicy"
-	GetBucketPolicyAction                  Action = "s3:GetBucketPolicy"
-	DeleteBucketPolicyAction               Action = "s3:DeleteBucketPolicy"
-	AbortMultipartUploadAction             Action = "s3:AbortMultipartUpload"
-	ListMultipartUploadPartsAction         Action = "s3:ListMultipartUploadParts"
-	ListBucketMultipartUploadsAction       Action = "s3:ListBucketMultipartUploads"
-	PutObjectAction                        Action = "s3:PutObject"
-	GetObjectAction                        Action = "s3:GetObject"
-	GetObjectVersionAction                 Action = "s3:GetObjectVersion"
-	DeleteObjectAction                     Action = "s3:DeleteObject"
-	GetObjectAclAction                     Action = "s3:GetObjectAcl"
-	GetObjectAttributesAction              Action = "s3:GetObjectAttributes"
-	PutObjectAclAction                     Action = "s3:PutObjectAcl"
-	RestoreObjectAction                    Action = "s3:RestoreObject"
-	GetBucketTaggingAction                 Action = "s3:GetBucketTagging"
-	PutBucketTaggingAction                 Action = "s3:PutBucketTagging"
-	GetObjectTaggingAction                 Action = "s3:GetObjectTagging"
-	PutObjectTaggingAction                 Action = "s3:PutObjectTagging"
-	DeleteObjectTaggingAction              Action = "s3:DeleteObjectTagging"
-	ListBucketVersionsAction               Action = "s3:ListBucketVersions"
-	ListBucketAction                       Action = "s3:ListBucket"
-	GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
-	PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
-	GetObjectLegalHoldAction               Action = "s3:GetObjectLegalHold"
-	PutObjectLegalHoldAction               Action = "s3:PutObjectLegalHold"
-	GetObjectRetentionAction               Action = "s3:GetObjectRetention"
-	PutObjectRetentionAction               Action = "s3:PutObjectRetention"
-	BypassGovernanceRetentionAction        Action = "s3:BypassGovernanceRetention"
-	PutBucketOwnershipControlsAction       Action = "s3:PutBucketOwnershipControls"
-	GetBucketOwnershipControlsAction       Action = "s3:GetBucketOwnershipControls"
-	PutBucketCorsAction                    Action = "s3:PutBucketCORS"
-	GetBucketCorsAction                    Action = "s3:GetBucketCORS"
-	AllActions                             Action = "s3:*"
-)
-
-var supportedActionList = map[Action]struct{}{
-	GetBucketAclAction:                     {},
-	CreateBucketAction:                     {},
-	PutBucketAclAction:                     {},
-	DeleteBucketAction:                     {},
-	PutBucketVersioningAction:              {},
-	GetBucketVersioningAction:              {},
-	PutBucketPolicyAction:                  {},
-	GetBucketPolicyAction:                  {},
-	DeleteBucketPolicyAction:               {},
-	AbortMultipartUploadAction:             {},
-	ListMultipartUploadPartsAction:         {},
-	ListBucketMultipartUploadsAction:       {},
-	PutObjectAction:                        {},
-	GetObjectAction:                        {},
-	GetObjectVersionAction:                 {},
-	DeleteObjectAction:                     {},
-	GetObjectAclAction:                     {},
-	GetObjectAttributesAction:              {},
-	PutObjectAclAction:                     {},
-	RestoreObjectAction:                    {},
-	GetBucketTaggingAction:                 {},
-	PutBucketTaggingAction:                 {},
-	GetObjectTaggingAction:                 {},
-	PutObjectTaggingAction:                 {},
-	DeleteObjectTaggingAction:              {},
-	ListBucketVersionsAction:               {},
-	ListBucketAction:                       {},
-	PutBucketObjectLockConfigurationAction: {},
-	GetObjectLegalHoldAction:               {},
-	PutObjectLegalHoldAction:               {},
-	GetObjectRetentionAction:               {},
-	PutObjectRetentionAction:               {},
-	BypassGovernanceRetentionAction:        {},
-	PutBucketOwnershipControlsAction:       {},
-	GetBucketOwnershipControlsAction:       {},
-	PutBucketCorsAction:                    {},
-	GetBucketCorsAction:                    {},
-	AllActions:                             {},
-}
-
-var supportedObjectActionList = map[Action]struct{}{
-	AbortMultipartUploadAction:      {},
-	ListMultipartUploadPartsAction:  {},
-	PutObjectAction:                 {},
-	GetObjectAction:                 {},
-	GetObjectVersionAction:          {},
-	DeleteObjectAction:              {},
-	GetObjectAclAction:              {},
-	GetObjectAttributesAction:       {},
-	PutObjectAclAction:              {},
-	RestoreObjectAction:             {},
-	GetObjectTaggingAction:          {},
-	PutObjectTaggingAction:          {},
-	DeleteObjectTaggingAction:       {},
-	GetObjectLegalHoldAction:        {},
-	PutObjectLegalHoldAction:        {},
-	GetObjectRetentionAction:        {},
-	PutObjectRetentionAction:        {},
-	BypassGovernanceRetentionAction: {},
-	AllActions:                      {},
-}
-
-// Validates Action: it should either wildcard match with supported actions list or be in it
-func (a Action) IsValid() error {
-	if !strings.HasPrefix(string(a), "s3:") {
-		return policyErrInvalidAction
-	}
-
-	if a == AllActions {
-		return nil
-	}
-
-	if a[len(a)-1] == '*' {
-		pattern := strings.TrimSuffix(string(a), "*")
-		for act := range supportedActionList {
-			if strings.HasPrefix(string(act), pattern) {
-				return nil
-			}
-		}
-
-		return policyErrInvalidAction
-	}
-
-	_, found := supportedActionList[a]
-	if !found {
-		return policyErrInvalidAction
-	}
-	return nil
-}
-
-func getBoolPtr(bl bool) *bool {
-	return &bl
-}
-
-// Checks if the action is object action
-// nil points to 's3:*'
-func (a Action) IsObjectAction() *bool {
-	if a == AllActions {
-		return nil
-	}
-	if a[len(a)-1] == '*' {
-		pattern := strings.TrimSuffix(string(a), "*")
-		for act := range supportedObjectActionList {
-			if strings.HasPrefix(string(act), pattern) {
-				return getBoolPtr(true)
-			}
-		}
-
-		return getBoolPtr(false)
-	}
-
-	_, found := supportedObjectActionList[a]
-	return &found
-}
-
-func (a Action) WildCardMatch(act Action) bool {
-	if strings.HasSuffix(string(a), "*") {
-		pattern := strings.TrimSuffix(string(a), "*")
-		return strings.HasPrefix(string(act), pattern)
-	}
-	return false
-}
-
-type Actions map[Action]struct{}
-
-// Override UnmarshalJSON method to decode both []string and string properties
-func (a *Actions) UnmarshalJSON(data []byte) error {
-	ss := []string{}
-	var err error
-	if err = json.Unmarshal(data, &ss); err == nil {
-		if len(ss) == 0 {
-			return policyErrInvalidAction
-		}
-		*a = make(Actions)
-		for _, s := range ss {
-			err = a.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	} else {
-		var s string
-		if err = json.Unmarshal(data, &s); err == nil {
-			if s == "" {
-				return policyErrInvalidAction
-			}
-			*a = make(Actions)
-			err = a.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return err
-}
-
-// Validates and adds a new Action to Actions map
-func (a Actions) Add(str string) error {
-	action := Action(str)
-	err := action.IsValid()
-	if err != nil {
-		return err
-	}
-
-	a[action] = struct{}{}
-	return nil
-}
-
-func (a Actions) FindMatch(action Action) bool {
-	_, ok := a[AllActions]
-	if ok {
-		return true
-	}
-	// First O(1) check for non wildcard actions
-	_, found := a[action]
-	if found {
-		return true
-	}
-
-	for act := range a {
-		if strings.HasSuffix(string(act), "*") && act.WildCardMatch(action) {
-			return true
-		}
-	}
-
-	return false
-}

auth/bucket_policy_effect.go

@@ -1,35 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import "fmt"
-
-type BucketPolicyAccessType string
-
-const (
-	BucketPolicyAccessTypeDeny  BucketPolicyAccessType = "Deny"
-	BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
-)
-
-// Checks policy statement Effect to be valid ("Deny", "Allow")
-func (bpat BucketPolicyAccessType) Validate() error {
-	switch bpat {
-	case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
-		return nil
-	}
-
-	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
-	return fmt.Errorf("Invalid effect: %v", bpat)
-}

auth/bucket_policy_principals.go

@@ -1,123 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-)
-
-type Principals map[string]struct{}
-
-func (p Principals) Add(key string) {
-	p[key] = struct{}{}
-}
-
-// Override UnmarshalJSON method to decode both []string and string properties
-func (p *Principals) UnmarshalJSON(data []byte) error {
-	ss := []string{}
-	var s string
-	var k struct {
-		AWS string
-	}
-
-	var err error
-
-	if err = json.Unmarshal(data, &ss); err == nil {
-		if len(ss) == 0 {
-			return policyErrInvalidPrincipal
-		}
-		*p = make(Principals)
-		for _, s := range ss {
-			p.Add(s)
-		}
-		return nil
-	} else if err = json.Unmarshal(data, &s); err == nil {
-		if s == "" {
-			return policyErrInvalidPrincipal
-		}
-		*p = make(Principals)
-		p.Add(s)
-
-		return nil
-	} else if err = json.Unmarshal(data, &k); err == nil {
-		if k.AWS == "" {
-			return policyErrInvalidPrincipal
-		}
-		*p = make(Principals)
-		p.Add(k.AWS)
-
-		return nil
-	} else {
-		var sk struct {
-			AWS []string
-		}
-		if err = json.Unmarshal(data, &sk); err == nil {
-			if len(sk.AWS) == 0 {
-				return policyErrInvalidPrincipal
-			}
-			*p = make(Principals)
-			for _, s := range sk.AWS {
-				p.Add(s)
-			}
-		}
-	}
-
-	return err
-}
-
-// Converts Principals map to a slice, by omitting "*"
-func (p Principals) ToSlice() []string {
-	principals := []string{}
-	for p := range p {
-		if p == "*" {
-			continue
-		}
-		principals = append(principals, p)
-	}
-
-	return principals
-}
-
-// Validates Principals by checking user account access keys existence
-func (p Principals) Validate(iam IAMService) error {
-	_, containsWildCard := p["*"]
-	if containsWildCard {
-		if len(p) == 1 {
-			return nil
-		}
-		return policyErrInvalidPrincipal
-	}
-
-	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
-	if err != nil {
-		return err
-	}
-	if len(accs) > 0 {
-		return policyErrInvalidPrincipal
-	}
-
-	return nil
-}
-
-func (p Principals) Contains(userAccess string) bool {
-	// "*" means it matches for any user account
-	_, ok := p["*"]
-	if ok {
-		return true
-	}
-
-	_, found := p[userAccess]
-	return found
-}

auth/bucket_policy_resources.go

@@ -1,160 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"strings"
-)
-
-type Resources map[string]struct{}
-
-const ResourceArnPrefix = "arn:aws:s3:::"
-
-// Override UnmarshalJSON method to decode both []string and string properties
-func (r *Resources) UnmarshalJSON(data []byte) error {
-	ss := []string{}
-	var err error
-	if err = json.Unmarshal(data, &ss); err == nil {
-		if len(ss) == 0 {
-			return policyErrInvalidResource
-		}
-		*r = make(Resources)
-		for _, s := range ss {
-			err = r.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	} else {
-		var s string
-		if err = json.Unmarshal(data, &s); err == nil {
-			if s == "" {
-				return policyErrInvalidResource
-			}
-			*r = make(Resources)
-			err = r.Add(s)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return err
-}
-
-// Adds and validates a new resource to Resources map
-func (r Resources) Add(rc string) error {
-	ok, pattern := isValidResource(rc)
-	if !ok {
-		return policyErrInvalidResource
-	}
-
-	r[pattern] = struct{}{}
-
-	return nil
-}
-
-// Checks if the resources contain object pattern
-func (r Resources) ContainsObjectPattern() bool {
-	for resource := range r {
-		if resource == "*" || strings.Contains(resource, "/") {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Checks if the resources contain bucket pattern
-func (r Resources) ContainsBucketPattern() bool {
-	for resource := range r {
-		if resource == "*" || !strings.Contains(resource, "/") {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
-func (r Resources) Validate(bucket string) error {
-	for resource := range r {
-		if !strings.HasPrefix(resource, bucket) {
-			return policyErrInvalidResource
-		}
-	}
-
-	return nil
-}
-
-func (r Resources) FindMatch(resource string) bool {
-	for res := range r {
-		if r.Match(res, resource) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Match checks if the input string matches the given pattern with wildcards (`*`, `?`).
-// - `?` matches exactly one occurrence of any character.
-// - `*` matches arbitrary many (including zero) occurrences of any character.
-func (r Resources) Match(pattern, input string) bool {
-	pIdx, sIdx := 0, 0
-	starIdx, matchIdx := -1, 0
-
-	for sIdx < len(input) {
-		if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
-			sIdx++
-			pIdx++
-		} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
-			starIdx = pIdx
-			matchIdx = sIdx
-			pIdx++
-		} else if starIdx != -1 {
-			pIdx = starIdx + 1
-			matchIdx++
-			sIdx = matchIdx
-		} else {
-			return false
-		}
-	}
-
-	for pIdx < len(pattern) && pattern[pIdx] == '*' {
-		pIdx++
-	}
-
-	return pIdx == len(pattern)
-}
-
-// Checks the resource to have arn prefix and not starting with /
-func isValidResource(rc string) (isValid bool, pattern string) {
-	if !strings.HasPrefix(rc, ResourceArnPrefix) {
-		return false, ""
-	}
-
-	res := strings.TrimPrefix(rc, ResourceArnPrefix)
-	if res == "" {
-		return false, ""
-	}
-	// The resource can't start with / (bucket name comes first)
-	if strings.HasPrefix(res, "/") {
-		return false, ""
-	}
-
-	return true, res
-}

auth/bucket_policy_resources_test.go

@@ -1,182 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"encoding/json"
-	"testing"
-)
-
-func TestUnmarshalJSON(t *testing.T) {
-	var r Resources
-
-	cases := []struct {
-		input    string
-		expected int
-		wantErr  bool
-	}{
-		{`"arn:aws:s3:::my-bucket/*"`, 1, false},
-		{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
-		{`""`, 0, true},
-		{`[]`, 0, true},
-		{`["invalid-bucket"]`, 0, true},
-	}
-
-	for _, tc := range cases {
-		r = Resources{}
-		err := json.Unmarshal([]byte(tc.input), &r)
-		if (err != nil) != tc.wantErr {
-			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
-		}
-		if len(r) != tc.expected {
-			t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
-		}
-	}
-}
-
-func TestAdd(t *testing.T) {
-	r := Resources{}
-
-	cases := []struct {
-		input   string
-		wantErr bool
-	}{
-		{"arn:aws:s3:::valid-bucket/*", false},
-		{"arn:aws:s3:::valid-bucket/object", false},
-		{"invalid-bucket/*", true},
-		{"/invalid-start", true},
-	}
-
-	for _, tc := range cases {
-		err := r.Add(tc.input)
-		if (err != nil) != tc.wantErr {
-			t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
-		}
-	}
-}
-
-func TestContainsObjectPattern(t *testing.T) {
-	cases := []struct {
-		resources []string
-		expected  bool
-	}{
-		{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
-		{[]string{"arn:aws:s3:::my-bucket/*"}, true},
-		{[]string{"arn:aws:s3:::my-bucket"}, false},
-	}
-
-	for _, tc := range cases {
-		r := Resources{}
-		for _, res := range tc.resources {
-			r.Add(res)
-		}
-		if r.ContainsObjectPattern() != tc.expected {
-			t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
-		}
-	}
-}
-
-func TestContainsBucketPattern(t *testing.T) {
-	cases := []struct {
-		resources []string
-		expected  bool
-	}{
-		{[]string{"arn:aws:s3:::my-bucket"}, true},
-		{[]string{"arn:aws:s3:::my-bucket/*"}, false},
-		{[]string{"arn:aws:s3:::my-bucket/object"}, false},
-	}
-
-	for _, tc := range cases {
-		r := Resources{}
-		for _, res := range tc.resources {
-			r.Add(res)
-		}
-		if r.ContainsBucketPattern() != tc.expected {
-			t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
-		}
-	}
-}
-
-func TestValidate(t *testing.T) {
-	cases := []struct {
-		resources []string
-		bucket    string
-		expected  bool
-	}{
-		{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
-		{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
-		{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
-	}
-
-	for _, tc := range cases {
-		r := Resources{}
-		for _, res := range tc.resources {
-			r.Add(res)
-		}
-		if (r.Validate(tc.bucket) == nil) != tc.expected {
-			t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
-		}
-	}
-}
-
-func TestFindMatch(t *testing.T) {
-	cases := []struct {
-		resources []string
-		input     string
-		expected  bool
-	}{
-		{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
-		{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
-		{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
-		{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
-	}
-
-	for _, tc := range cases {
-		r := Resources{}
-		for _, res := range tc.resources {
-			r.Add(res)
-		}
-		if r.FindMatch(tc.input) != tc.expected {
-			t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
-		}
-	}
-}
-
-func TestMatch(t *testing.T) {
-	r := Resources{}
-	cases := []struct {
-		pattern  string
-		input    string
-		expected bool
-	}{
-		{"my-bucket/*", "my-bucket/object", true},
-		{"my-bucket/?bject", "my-bucket/object", true},
-		{"my-bucket/*", "other-bucket/object", false},
-		{"*", "any-bucket/object", true},
-		{"my-bucket/*", "my-bucket/subdir/object", true},
-		{"my-bucket/*", "other-bucket", false},
-		{"my-bucket/*/*", "my-bucket/hello", false},
-		{"my-bucket/*/*", "my-bucket/hello/world", true},
-		{"foo/???/bar", "foo/qux/bar", true},
-		{"foo/???/bar", "foo/quxx/bar", false},
-		{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
-		{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
-	}
-	for _, tc := range cases {
-		if r.Match(tc.pattern, tc.input) != tc.expected {
-			t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
-		}
-	}
-}

auth/iam.go

@@ -28,49 +28,14 @@ const (
 	RoleUserPlus Role = "userplus"
 )
 
-func (r Role) IsValid() bool {
-	switch r {
-	case RoleAdmin:
-		return true
-	case RoleUser:
-		return true
-	case RoleUserPlus:
-		return true
-	default:
-		return false
-	}
-}
-
 // Account is a gateway IAM account
 type Account struct {
-	Access  string `json:"access"`
-	Secret  string `json:"secret"`
-	Role    Role   `json:"role"`
-	UserID  int    `json:"userID"`
-	GroupID int    `json:"groupID"`
-}
-
-type ListUserAccountsResult struct {
-	Accounts []Account
-}
-
-// Mutable props, which could be changed when updating an IAM account
-type MutableProps struct {
-	Secret  *string `json:"secret"`
-	UserID  *int    `json:"userID"`
-	GroupID *int    `json:"groupID"`
-}
-
-func updateAcc(acc *Account, props MutableProps) {
-	if props.Secret != nil {
-		acc.Secret = *props.Secret
-	}
-	if props.GroupID != nil {
-		acc.GroupID = *props.GroupID
-	}
-	if props.UserID != nil {
-		acc.UserID = *props.UserID
-	}
+	Access    string `json:"access"`
+	Secret    string `json:"secret"`
+	Role      Role   `json:"role"`
+	UserID    int    `json:"userID"`
+	GroupID   int    `json:"groupID"`
+	ProjectID int    `json:"projectID"`
 }
 
 // IAMService is the interface for all IAM service implementations
@@ -79,57 +44,33 @@ func updateAcc(acc *Account, props MutableProps) {
 type IAMService interface {
 	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
-	UpdateUserAccount(access string, props MutableProps) error
 	DeleteUserAccount(access string) error
 	ListUserAccounts() ([]Account, error)
 	Shutdown() error
 }
 
-var (
-	// ErrUserExists is returned when the user already exists
-	ErrUserExists = errors.New("user already exists")
-	// ErrNoSuchUser is returned when the user does not exist
-	ErrNoSuchUser = errors.New("user not found")
-)
+var ErrNoSuchUser = errors.New("user not found")
 
 type Opts struct {
-	RootAccount            Account
-	Dir                    string
-	LDAPServerURL          string
-	LDAPBindDN             string
-	LDAPPassword           string
-	LDAPQueryBase          string
-	LDAPObjClasses         string
-	LDAPAccessAtr          string
-	LDAPSecretAtr          string
-	LDAPRoleAtr            string
-	LDAPUserIdAtr          string
-	LDAPGroupIdAtr         string
-	VaultEndpointURL       string
-	VaultSecretStoragePath string
-	VaultMountPath         string
-	VaultRootToken         string
-	VaultRoleId            string
-	VaultRoleSecret        string
-	VaultServerCert        string
-	VaultClientCert        string
-	VaultClientCertKey     string
-	S3Access               string
-	S3Secret               string
-	S3Region               string
-	S3Bucket               string
-	S3Endpoint             string
-	S3DisableSSlVerfiy     bool
-	S3Debug                bool
-	CacheDisable           bool
-	CacheTTL               int
-	CachePrune             int
-	IpaHost                string
-	IpaVaultName           string
-	IpaUser                string
-	IpaPassword            string
-	IpaInsecure            bool
-	IpaDebug               bool
+	Dir                string
+	LDAPServerURL      string
+	LDAPBindDN         string
+	LDAPPassword       string
+	LDAPQueryBase      string
+	LDAPObjClasses     string
+	LDAPAccessAtr      string
+	LDAPSecretAtr      string
+	LDAPRoleAtr        string
+	S3Access           string
+	S3Secret           string
+	S3Region           string
+	S3Bucket           string
+	S3Endpoint         string
+	S3DisableSSlVerfiy bool
+	S3Debug            bool
+	CacheDisable       bool
+	CacheTTL           int
+	CachePrune         int
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -138,30 +79,22 @@ func New(o *Opts) (IAMService, error) {
 
 	switch {
 	case o.Dir != "":
-		svc, err = NewInternal(o.RootAccount, o.Dir)
+		svc, err = NewInternal(o.Dir)
 		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
-		svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
-			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
-			o.LDAPGroupIdAtr, o.LDAPObjClasses)
+		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
+			o.LDAPObjClasses)
 		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
 	case o.S3Endpoint != "":
-		svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
 			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
 		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
 			o.S3Endpoint, o.S3Bucket)
-	case o.VaultEndpointURL != "":
-		svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
-			o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
-			o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
-		fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
-	case o.IpaHost != "":
-		svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure, o.IpaDebug)
-		fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
 	default:
 		// if no iam options selected, default to the single user mode
 		fmt.Println("No IAM service configured, enabling single account mode")
-		return NewIAMServiceSingle(o.RootAccount), nil
+		return IAMServiceSingle{}, nil
 	}
 
 	if err != nil {

auth/iam_cache.go

@@ -66,21 +66,6 @@ func (i *icache) get(k string) (Account, bool) {
 	return v.value, true
 }
 
-func (i *icache) update(k string, props MutableProps) {
-	i.Lock()
-	defer i.Unlock()
-
-	item, found := i.items[k]
-	if found {
-		updateAcc(&item.value, props)
-
-		// refresh the expiration date
-		item.exp = time.Now().Add(i.expire)
-
-		i.items[k] = item
-	}
-}
-
 func (i *icache) Delete(k string) {
 	i.Lock()
 	delete(i.items, k)
@@ -181,16 +166,6 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
 	return nil
 }
 
-func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
-	err := c.service.UpdateUserAccount(access, props)
-	if err != nil {
-		return err
-	}
-
-	c.iamcache.update(access, props)
-	return nil
-}
-
 // ListUserAccounts is a passthrough to the underlying service and
 // does not make use of the cache
 func (c *IAMCache) ListUserAccounts() ([]Account, error) {

auth/iam_internal.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"sync"
 	"time"
 )
 
@@ -33,15 +32,7 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	// This mutex will help with racing updates to the IAM data
-	// from multiple requests to this gateway instance, but
-	// will not help with racing updates to multiple load balanced
-	// gateway instances. This is a limitation of the internal
-	// IAM service. All account updates should be sent to a single
-	// gateway instance if possible.
-	sync.RWMutex
-	dir     string
-	rootAcc Account
+	dir string
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -55,10 +46,9 @@ type iAMConfig struct {
 var _ IAMService = &IAMServiceInternal{}
 
 // NewInternal creates a new instance for the Internal IAM service
-func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
+func NewInternal(dir string) (*IAMServiceInternal, error) {
 	i := &IAMServiceInternal{
-		dir:     dir,
-		rootAcc: rootAcc,
+		dir: dir,
 	}
 
 	err := i.initIAM()
@@ -72,13 +62,6 @@ func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
-	if account.Access == s.rootAcc.Access {
-		return ErrUserExists
-	}
-
-	s.Lock()
-	defer s.Unlock()
-
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -87,7 +70,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 
 		_, ok := conf.AccessAccounts[account.Access]
 		if ok {
-			return nil, ErrUserExists
+			return nil, fmt.Errorf("account already exists")
 		}
 		conf.AccessAccounts[account.Access] = account
 
@@ -103,13 +86,6 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
-	if access == s.rootAcc.Access {
-		return s.rootAcc, nil
-	}
-
-	s.RLock()
-	defer s.RUnlock()
-
 	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -123,41 +99,9 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
-// UpdateUserAccount updates the specified user account fields. Returns
-// ErrNoSuchUser if the account does not exist.
-func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
-	s.Lock()
-	defer s.Unlock()
-
-	return s.storeIAM(func(data []byte) ([]byte, error) {
-		conf, err := parseIAM(data)
-		if err != nil {
-			return nil, fmt.Errorf("get iam data: %w", err)
-		}
-
-		acc, found := conf.AccessAccounts[access]
-		if !found {
-			return nil, ErrNoSuchUser
-		}
-
-		updateAcc(&acc, props)
-		conf.AccessAccounts[access] = acc
-
-		b, err := json.Marshal(conf)
-		if err != nil {
-			return nil, fmt.Errorf("failed to serialize iam: %w", err)
-		}
-
-		return b, nil
-	})
-}
-
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
-	s.Lock()
-	defer s.Unlock()
-
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -177,9 +121,6 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 
 // ListUserAccounts lists all the user accounts stored.
 func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
-	s.RLock()
-	defer s.RUnlock()
-
 	conf, err := s.getIAM()
 	if err != nil {
 		return []Account{}, fmt.Errorf("get iam data: %w", err)
@@ -194,11 +135,12 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
 	var accs []Account
 	for _, k := range keys {
 		accs = append(accs, Account{
-			Access:  k,
-			Secret:  conf.AccessAccounts[k].Secret,
-			Role:    conf.AccessAccounts[k].Role,
-			UserID:  conf.AccessAccounts[k].UserID,
-			GroupID: conf.AccessAccounts[k].GroupID,
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
 		})
 	}
 
@@ -247,10 +189,6 @@ func parseIAM(b []byte) (iAMConfig, error) {
 		return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
 	}
 
-	if conf.AccessAccounts == nil {
-		conf.AccessAccounts = make(map[string]Account)
-	}
-
 	return conf, nil
 }
 

auth/iam_ipa.go

@@ -1,446 +0,0 @@
-// Copyright 2025 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"crypto/aes"
-	"crypto/cipher"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/base64"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/http/cookiejar"
-	"net/url"
-	"strconv"
-	"strings"
-)
-
-const IpaVersion = "2.254"
-
-type IpaIAMService struct {
-	client          http.Client
-	id              int
-	version         string
-	host            string
-	vaultName       string
-	username        string
-	password        string
-	kraTransportKey *rsa.PublicKey
-	debug           bool
-	rootAcc         Account
-}
-
-var _ IAMService = &IpaIAMService{}
-
-func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
-
-	ipa := IpaIAMService{
-		id:        0,
-		version:   IpaVersion,
-		host:      host,
-		vaultName: vaultName,
-		username:  username,
-		password:  password,
-		debug:     debug,
-		rootAcc:   rootAcc,
-	}
-	jar, err := cookiejar.New(nil)
-	if err != nil {
-		// this should never happen
-		return nil, fmt.Errorf("cookie jar creation: %w", err)
-	}
-
-	mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
-	tr := &http.Transport{
-		TLSClientConfig: mTLSConfig,
-	}
-	ipa.client = http.Client{Jar: jar, Transport: tr}
-
-	err = ipa.login()
-	if err != nil {
-		return nil, fmt.Errorf("ipa login failed: %w", err)
-	}
-
-	req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
-	if err != nil {
-		return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
-	}
-	vaultConfig := struct {
-		Kra_Server_Server             []string
-		Transport_Cert                Base64EncodedWrapped
-		Wrapping_default_algorithm    string
-		Wrapping_supported_algorithms []string
-	}{}
-	err = ipa.rpc(req, &vaultConfig)
-	if err != nil {
-		return nil, fmt.Errorf("ipa vault config: %w", err)
-	}
-
-	cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
-	if err != nil {
-		return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
-	}
-
-	ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
-
-	isSupported := false
-	for _, algo := range vaultConfig.Wrapping_supported_algorithms {
-		if algo == "aes-128-cbc" {
-			isSupported = true
-			break
-		}
-	}
-
-	if !isSupported {
-		return nil,
-			fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
-				vaultConfig.Wrapping_supported_algorithms)
-	}
-	return &ipa, nil
-}
-
-func (ipa *IpaIAMService) CreateAccount(account Account) error {
-	return fmt.Errorf("not implemented")
-}
-
-func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
-	if access == ipa.rootAcc.Access {
-		return ipa.rootAcc, nil
-	}
-
-	req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
-	if err != nil {
-		return Account{}, fmt.Errorf("ipa user_show: %w", err)
-	}
-
-	userResult := struct {
-		Gidnumber []string
-		Uidnumber []string
-	}{}
-
-	err = ipa.rpc(req, &userResult)
-	if err != nil {
-		return Account{}, err
-	}
-
-	uid, err := strconv.Atoi(userResult.Uidnumber[0])
-	if err != nil {
-		return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
-	}
-	gid, err := strconv.Atoi(userResult.Gidnumber[0])
-	if err != nil {
-		return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
-	}
-
-	account := Account{
-		Access:  access,
-		Role:    RoleUser,
-		UserID:  uid,
-		GroupID: gid,
-	}
-
-	session_key := make([]byte, 16)
-
-	_, err = rand.Read(session_key)
-	if err != nil {
-		return account, fmt.Errorf("ipa cannot generate session key: %w", err)
-	}
-
-	encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
-	if err != nil {
-		return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
-	}
-
-	req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
-		map[string]any{"username": access,
-			"session_key":   Base64EncodedWrapped(encryptedKey),
-			"wrapping_algo": "aes-128-cbc"})
-	if err != nil {
-		return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
-	}
-
-	data := struct {
-		Vault_data Base64EncodedWrapped
-		Nonce      Base64EncodedWrapped
-	}{}
-
-	err = ipa.rpc(req, &data)
-	if err != nil {
-		return account, err
-	}
-
-	aes, err := aes.NewCipher(session_key)
-	if err != nil {
-		return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
-	}
-	cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
-	cbc.CryptBlocks(data.Vault_data, data.Vault_data)
-	secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
-	if err != nil {
-		return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
-	}
-
-	secret := struct {
-		Data Base64Encoded
-	}{}
-	json.Unmarshal(secretUnpaddedJson, &secret)
-	account.Secret = string(secret.Data)
-
-	return account, nil
-}
-
-func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
-	return fmt.Errorf("not implemented")
-}
-
-func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
-	return fmt.Errorf("not implemented")
-}
-
-func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
-	return []Account{}, fmt.Errorf("not implemented")
-}
-
-func (ipa *IpaIAMService) Shutdown() error {
-	return nil
-}
-
-// Implementation
-
-func (ipa *IpaIAMService) login() error {
-	form := url.Values{}
-	form.Set("user", ipa.username)
-	form.Set("password", ipa.password)
-
-	req, err := http.NewRequest(
-		"POST",
-		fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
-		strings.NewReader(form.Encode()))
-	if err != nil {
-		return err
-	}
-
-	req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-
-	resp, err := ipa.client.Do(req)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode == 401 {
-		return errors.New("cannot login to FreeIPA: invalid credentials")
-	}
-
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
-	}
-
-	return nil
-}
-
-type rpcRequest = string
-
-type rpcResponse struct {
-	Result    json.RawMessage
-	Principal string
-	Id        int
-	Version   string
-}
-
-func (p rpcResponse) String() string {
-	return string(p.Result)
-}
-
-var errRpc = errors.New("IPA RPC error")
-
-func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
-	err := ipa.login()
-	if err != nil {
-		return err
-	}
-
-	res, err := ipa.rpcInternal(req)
-	if err != nil {
-		return err
-	}
-
-	return json.Unmarshal(res.Result, value)
-}
-
-func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
-	httpReq, err := http.NewRequest("POST",
-		fmt.Sprintf("%s/ipa/session/json", ipa.host),
-		strings.NewReader(req))
-	if err != nil {
-		return rpcResponse{}, err
-	}
-
-	ipa.log(fmt.Sprintf("%v", req))
-	httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
-	httpReq.Header.Set("Content-Type", "application/json")
-
-	httpResp, err := ipa.client.Do(httpReq)
-	if err != nil {
-		return rpcResponse{}, err
-	}
-
-	bytes, err := io.ReadAll(httpResp.Body)
-	ipa.log(string(bytes))
-	if err != nil {
-		return rpcResponse{}, err
-	}
-
-	result := struct {
-		Result struct {
-			Json    json.RawMessage `json:"result"`
-			Value   string          `json:"value"`
-			Summary any             `json:"summary"`
-		} `json:"result"`
-		Error     json.RawMessage `json:"error"`
-		Id        int             `json:"id"`
-		Principal string          `json:"principal"`
-		Version   string          `json:"version"`
-	}{}
-
-	err = json.Unmarshal(bytes, &result)
-	if err != nil {
-		return rpcResponse{}, err
-	}
-	if string(result.Error) != "null" {
-		return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
-	}
-
-	return rpcResponse{
-		Result:    result.Result.Json,
-		Principal: result.Principal,
-		Id:        result.Id,
-		Version:   result.Version,
-	}, nil
-}
-
-func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
-
-	id := ipa.id
-	ipa.id++
-
-	dict["version"] = ipa.version
-
-	jmethod, errMethod := json.Marshal(method)
-	jargs, errArgs := json.Marshal(args)
-	jdict, errDict := json.Marshal(dict)
-
-	err := errors.Join(errMethod, errArgs, errDict)
-	if err != nil {
-		return "", fmt.Errorf("ipa request invalid: %w", err)
-	}
-
-	request := map[string]interface{}{
-		"id":     id,
-		"method": json.RawMessage(jmethod),
-		"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
-	}
-
-	requestJSON, err := json.Marshal(request)
-	if err != nil {
-		return "", fmt.Errorf("failed to marshal request: %w", err)
-	}
-
-	return string(requestJSON), nil
-}
-
-// pkcs7Unpad validates and unpads data from the given bytes slice.
-// The returned value will be 1 to n bytes smaller depending on the
-// amount of padding, where n is the block size.
-func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
-	if blocksize <= 0 {
-		return nil, errors.New("invalid blocksize")
-	}
-	if len(b) == 0 {
-		return nil, errors.New("invalid PKCS7 data (empty or not padded)")
-	}
-	if len(b)%blocksize != 0 {
-		return nil, errors.New("invalid padding on input")
-	}
-	c := b[len(b)-1]
-	n := int(c)
-	if n == 0 || n > len(b) {
-		return nil, errors.New("invalid padding on input")
-	}
-	for i := 0; i < n; i++ {
-		if b[len(b)-n+i] != c {
-			return nil, errors.New("invalid padding on input")
-		}
-	}
-	return b[:len(b)-n], nil
-}
-
-/*
-e.g.
-
-	"value" {
-		"__base64__": "aGVsbG93b3JsZAo="
-	 }
-*/
-type Base64EncodedWrapped []byte
-
-func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
-	intermediate := struct {
-		Base64 string `json:"__base64__"`
-	}{}
-	err := json.Unmarshal(data, &intermediate)
-	if err != nil {
-		return err
-	}
-	*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
-	return err
-}
-
-func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
-	intermediate := struct {
-		Base64 string `json:"__base64__"`
-	}{Base64: base64.StdEncoding.EncodeToString(*b)}
-	return json.Marshal(intermediate)
-}
-
-/*
-e.g.
-
-	"value": "aGVsbG93b3JsZAo="
-*/
-type Base64Encoded []byte
-
-func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
-	var intermediate string
-	err := json.Unmarshal(data, &intermediate)
-	if err != nil {
-		return err
-	}
-	*b, err = base64.StdEncoding.DecodeString(intermediate)
-	return err
-}
-
-func (ipa *IpaIAMService) log(msg string) {
-	if ipa.debug {
-		log.Println(msg)
-	}
-}

auth/iam_ldap.go

@@ -1,22 +1,7 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package auth
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/go-ldap/ldap/v3"
@@ -29,19 +14,15 @@ type LdapIAMService struct {
 	accessAtr  string
 	secretAtr  string
 	roleAtr    string
-	groupIdAtr string
-	userIdAtr  string
-	rootAcc    Account
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
-	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
-		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
-	conn, err := ldap.DialURL(url)
+	conn, err := ldap.Dial("tcp", url)
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
 	}
@@ -57,23 +38,15 @@ func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAt
 		accessAtr:  accAtr,
 		secretAtr:  secAtr,
 		roleAtr:    roleAtr,
-		userIdAtr:  userIdAtr,
-		groupIdAtr: groupIdAtr,
-		rootAcc:    rootAcc,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
-	if ld.rootAcc.Access == account.Access {
-		return ErrUserExists
-	}
-	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
 	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
-	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
-	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -84,9 +57,6 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
 }
 
 func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
-	if access == ld.rootAcc.Access {
-		return ld.rootAcc, nil
-	}
 	searchRequest := ldap.NewSearchRequest(
 		ld.queryBase,
 		ldap.ScopeWholeSubtree,
@@ -95,7 +65,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
 		nil,
 	)
 
@@ -104,50 +74,14 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		return Account{}, err
 	}
 
-	if len(result.Entries) == 0 {
-		return Account{}, ErrNoSuchUser
-	}
-
 	entry := result.Entries[0]
-	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
-	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for group-id %q: %w",
-			entry.GetAttributeValue(ld.groupIdAtr), err)
-	}
-	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
-	if err != nil {
-		return Account{}, fmt.Errorf("invalid entry value for user-id %q: %w",
-			entry.GetAttributeValue(ld.userIdAtr), err)
-	}
 	return Account{
-		Access:  entry.GetAttributeValue(ld.accessAtr),
-		Secret:  entry.GetAttributeValue(ld.secretAtr),
-		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
-		GroupID: groupId,
-		UserID:  userId,
+		Access: entry.GetAttributeValue(ld.accessAtr),
+		Secret: entry.GetAttributeValue(ld.secretAtr),
+		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
 	}, nil
 }
 
-func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
-	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
-	if props.Secret != nil {
-		req.Replace(ld.secretAtr, []string{*props.Secret})
-	}
-	if props.GroupID != nil {
-		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
-	}
-	if props.UserID != nil {
-		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
-	}
-
-	err := ld.conn.Modify(req)
-	//TODO: Handle non existing user case
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 func (ld *LdapIAMService) DeleteUserAccount(access string) error {
 	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
 
@@ -172,7 +106,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(&%v)", searchFilter),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
 		nil,
 	)
 
@@ -183,22 +117,10 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 
 	result := []Account{}
 	for _, el := range resp.Entries {
-		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
-		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for group-id %q: %w",
-				el.GetAttributeValue(ld.groupIdAtr), err)
-		}
-		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
-		if err != nil {
-			return nil, fmt.Errorf("invalid entry value for user-id %q: %w",
-				el.GetAttributeValue(ld.userIdAtr), err)
-		}
 		result = append(result, Account{
-			Access:  el.GetAttributeValue(ld.accessAtr),
-			Secret:  el.GetAttributeValue(ld.secretAtr),
-			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
-			GroupID: groupId,
-			UserID:  userId,
+			Access: el.GetAttributeValue(ld.accessAtr),
+			Secret: el.GetAttributeValue(ld.secretAtr),
+			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
 		})
 	}
 

auth/iam_s3_object.go

@@ -24,7 +24,6 @@ import (
 	"io"
 	"net/http"
 	"sort"
-	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -42,14 +41,6 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
-	// This mutex will help with racing updates to the IAM data
-	// from multiple requests to this gateway instance, but
-	// will not help with racing updates to multiple load balanced
-	// gateway instances. This is a limitation of the internal
-	// IAM service. All account updates should be sent to a single
-	// gateway instance if possible.
-	sync.RWMutex
-
 	access        string
 	secret        string
 	region        string
@@ -57,13 +48,12 @@ type IAMServiceS3 struct {
 	endpoint      string
 	sslSkipVerify bool
 	debug         bool
-	rootAcc       Account
 	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}
 
-func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
 	if access == "" {
 		return nil, fmt.Errorf("must provide s3 IAM service access key")
 	}
@@ -88,7 +78,6 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
 		endpoint:      endpoint,
 		sslSkipVerify: sslSkipVerify,
 		debug:         debug,
-		rootAcc:       rootAcc,
 	}
 
 	cfg, err := i.getConfig()
@@ -96,25 +85,11 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
 		return nil, fmt.Errorf("init s3 IAM: %v", err)
 	}
 
-	if endpoint != "" {
-		i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
-			o.BaseEndpoint = &endpoint
-		})
-		return i, nil
-	}
-
 	i.client = s3.NewFromConfig(cfg)
 	return i, nil
 }
 
 func (s *IAMServiceS3) CreateAccount(account Account) error {
-	if s.rootAcc.Access == account.Access {
-		return ErrUserExists
-	}
-
-	s.Lock()
-	defer s.Unlock()
-
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -122,7 +97,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 
 	_, ok := conf.AccessAccounts[account.Access]
 	if ok {
-		return ErrUserExists
+		return fmt.Errorf("account already exists")
 	}
 	conf.AccessAccounts[account.Access] = account
 
@@ -130,13 +105,6 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 }
 
 func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
-	if access == s.rootAcc.Access {
-		return s.rootAcc, nil
-	}
-
-	s.RLock()
-	defer s.RUnlock()
-
 	conf, err := s.getAccounts()
 	if err != nil {
 		return Account{}, err
@@ -150,30 +118,7 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
-func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
-	s.Lock()
-	defer s.Unlock()
-
-	conf, err := s.getAccounts()
-	if err != nil {
-		return err
-	}
-
-	acc, ok := conf.AccessAccounts[access]
-	if !ok {
-		return ErrNoSuchUser
-	}
-
-	updateAcc(&acc, props)
-	conf.AccessAccounts[access] = acc
-
-	return s.storeAccts(conf)
-}
-
 func (s *IAMServiceS3) DeleteUserAccount(access string) error {
-	s.Lock()
-	defer s.Unlock()
-
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -189,9 +134,6 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
 }
 
 func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
-	s.RLock()
-	defer s.RUnlock()
-
 	conf, err := s.getAccounts()
 	if err != nil {
 		return nil, err
@@ -206,17 +148,28 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
 	var accs []Account
 	for _, k := range keys {
 		accs = append(accs, Account{
-			Access:  k,
-			Secret:  conf.AccessAccounts[k].Secret,
-			Role:    conf.AccessAccounts[k].Role,
-			UserID:  conf.AccessAccounts[k].UserID,
-			GroupID: conf.AccessAccounts[k].GroupID,
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
 		})
 	}
 
 	return accs, nil
 }
 
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.region,
+		HostnameImmutable: true,
+	}, nil
+}
+
 func (s *IAMServiceS3) Shutdown() error {
 	return nil
 }
@@ -235,6 +188,11 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
 		config.WithHTTPClient(client),
 	}
 
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
 	if s.debug {
 		opts = append(opts,
 			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
@@ -252,15 +210,15 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
 	})
 	if err != nil {
 		// if the error is object not exists,
-		// init empty accounts struct and return that
+		// init empty accounts stuct and return that
 		var nsk *types.NoSuchKey
 		if errors.As(err, &nsk) {
-			return iAMConfig{AccessAccounts: map[string]Account{}}, nil
+			return iAMConfig{}, nil
 		}
 		var apiErr smithy.APIError
 		if errors.As(err, &apiErr) {
 			if apiErr.ErrorCode() == "NotFound" {
-				return iAMConfig{AccessAccounts: map[string]Account{}}, nil
+				return iAMConfig{}, nil
 			}
 		}
 

auth/iam_single.go

@@ -15,49 +15,34 @@
 package auth
 
 import (
-	"github.com/versity/versitygw/s3err"
+	"errors"
 )
 
 // IAMServiceSingle manages the single tenant (root-only) IAM service
-type IAMServiceSingle struct {
-	root Account
-}
+type IAMServiceSingle struct{}
 
 var _ IAMService = &IAMServiceSingle{}
 
-func NewIAMServiceSingle(r Account) IAMService {
-	return &IAMServiceSingle{
-		root: r,
-	}
-}
+var ErrNotSupported = errors.New("method is not supported")
 
 // CreateAccount not valid in single tenant mode
 func (IAMServiceSingle) CreateAccount(account Account) error {
-	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+	return ErrNotSupported
 }
 
-// GetUserAccount returns root account, if the root access key
-// is provided and "ErrAdminUserNotFound" otherwise
-func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) {
-	if access == s.root.Access {
-		return s.root, nil
-	}
-	return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound)
-}
-
-// UpdateUserAccount no accounts in single tenant mode
-func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
-	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+// GetUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
+	return Account{}, ErrNotSupported
 }
 
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
-	return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+	return ErrNotSupported
 }
 
 // ListUserAccounts no accounts in single tenant mode
 func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
-	return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
+	return []Account{}, nil
 }
 
 // Shutdown graceful termination of service

auth/iam_vault.go

@@ -1,256 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strings"
-	"time"
-
-	vault "github.com/hashicorp/vault-client-go"
-	"github.com/hashicorp/vault-client-go/schema"
-)
-
-type VaultIAMService struct {
-	client            *vault.Client
-	reqOpts           []vault.RequestOption
-	secretStoragePath string
-	rootAcc           Account
-}
-
-var _ IAMService = &VaultIAMService{}
-
-func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
-	opts := []vault.ClientOption{
-		vault.WithAddress(endpoint),
-		// set request timeout to 10 secs
-		vault.WithRequestTimeout(10 * time.Second),
-	}
-	if serverCert != "" {
-		tls := vault.TLSConfiguration{}
-
-		tls.ServerCertificate.FromBytes = []byte(serverCert)
-		if clientCert != "" {
-			if clientCertKey == "" {
-				return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
-			}
-
-			tls.ClientCertificate.FromBytes = []byte(clientCert)
-			tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
-		}
-
-		opts = append(opts, vault.WithTLS(tls))
-	}
-
-	client, err := vault.New(opts...)
-	if err != nil {
-		return nil, fmt.Errorf("init vault client: %w", err)
-	}
-
-	reqOpts := []vault.RequestOption{}
-	// if mount path is not specified, it defaults to "approle"
-	if mountPath != "" {
-		reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
-	}
-
-	// Authentication
-	switch {
-	case rootToken != "":
-		err := client.SetToken(rootToken)
-		if err != nil {
-			return nil, fmt.Errorf("root token authentication failure: %w", err)
-		}
-	case roleID != "":
-		if roleSecret == "" {
-			return nil, fmt.Errorf("role id and role secret must both be specified")
-		}
-
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
-			RoleId:   roleID,
-			SecretId: roleSecret,
-		}, reqOpts...)
-		cancel()
-		if err != nil {
-			return nil, fmt.Errorf("approle authentication failure: %w", err)
-		}
-
-		if err := client.SetToken(resp.Auth.ClientToken); err != nil {
-			return nil, fmt.Errorf("approle authentication set token failure: %w", err)
-		}
-	default:
-		return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
-	}
-
-	return &VaultIAMService{
-		client:            client,
-		reqOpts:           reqOpts,
-		secretStoragePath: secretStoragePath,
-		rootAcc:           rootAcc,
-	}, nil
-}
-
-func (vt *VaultIAMService) CreateAccount(account Account) error {
-	if vt.rootAcc.Access == account.Access {
-		return ErrUserExists
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
-		Data: map[string]any{
-			account.Access: account,
-		},
-		Options: map[string]interface{}{
-			"cas": 0,
-		},
-	}, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		if strings.Contains(err.Error(), "check-and-set") {
-			return ErrUserExists
-		}
-		return err
-	}
-
-	return nil
-}
-
-func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
-	if vt.rootAcc.Access == access {
-		return vt.rootAcc, nil
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		return Account{}, err
-	}
-
-	acc, err := parseVaultUserAccount(resp.Data.Data, access)
-	if err != nil {
-		return Account{}, err
-	}
-
-	return acc, nil
-}
-
-func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
-	//TODO: We need something like a transaction here ?
-	acc, err := vt.GetUserAccount(access)
-	if err != nil {
-		return err
-	}
-
-	updateAcc(&acc, props)
-
-	err = vt.DeleteUserAccount(access)
-	if err != nil {
-		return err
-	}
-
-	err = vt.CreateAccount(acc)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (vt *VaultIAMService) DeleteUserAccount(access string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
-	cancel()
-	if err != nil {
-		if vault.IsErrorStatus(err, 404) {
-			return []Account{}, nil
-		}
-		return nil, err
-	}
-
-	accs := []Account{}
-
-	for _, acss := range resp.Data.Keys {
-		acc, err := vt.GetUserAccount(acss)
-		if err != nil {
-			return nil, err
-		}
-		accs = append(accs, acc)
-	}
-
-	return accs, nil
-}
-
-// the client doesn't have explicit shutdown, as it uses http.Client
-func (vt *VaultIAMService) Shutdown() error {
-	return nil
-}
-
-var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
-
-func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
-	usrAcc, ok := data[access].(map[string]interface{})
-	if !ok {
-		return acc, errInvalidUser
-	}
-
-	acss, ok := usrAcc["access"].(string)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	secret, ok := usrAcc["secret"].(string)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	role, ok := usrAcc["role"].(string)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	userIdJson, ok := usrAcc["userID"].(json.Number)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	userId, err := userIdJson.Int64()
-	if err != nil {
-		return acc, errInvalidUser
-	}
-	groupIdJson, ok := usrAcc["groupID"].(json.Number)
-	if !ok {
-		return acc, errInvalidUser
-	}
-	groupId, err := groupIdJson.Int64()
-	if err != nil {
-		return acc, errInvalidUser
-	}
-
-	return Account{
-		Access:  acss,
-		Secret:  secret,
-		Role:    Role(role),
-		UserID:  int(userId),
-		GroupID: int(groupId),
-	}, nil
-}

auth/object_lock.go

@@ -1,269 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package auth
-
-import (
-	"context"
-	"encoding/json"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3response"
-)
-
-type BucketLockConfig struct {
-	Enabled          bool
-	DefaultRetention *types.DefaultRetention
-	CreatedAt        *time.Time
-}
-
-func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
-	var lockConfig types.ObjectLockConfiguration
-	if err := xml.Unmarshal(input, &lockConfig); err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-	}
-
-	if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
-		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-	}
-
-	config := BucketLockConfig{
-		Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
-	}
-
-	if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
-		retention := lockConfig.Rule.DefaultRetention
-
-		if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
-			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-		}
-		if retention.Years != nil && retention.Days != nil {
-			return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-		}
-
-		if retention.Days != nil && *retention.Days <= 0 {
-			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
-		}
-		if retention.Years != nil && *retention.Years <= 0 {
-			return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
-		}
-
-		config.DefaultRetention = retention
-		now := time.Now()
-		config.CreatedAt = &now
-	}
-
-	return json.Marshal(config)
-}
-
-func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
-	var config BucketLockConfig
-	if err := json.Unmarshal(input, &config); err != nil {
-		return nil, fmt.Errorf("parse object lock config: %w", err)
-	}
-
-	result := &types.ObjectLockConfiguration{
-		Rule: &types.ObjectLockRule{
-			DefaultRetention: config.DefaultRetention,
-		},
-	}
-
-	if config.Enabled {
-		result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
-	}
-
-	return result, nil
-}
-
-func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
-	var retention s3response.PutObjectRetentionInput
-	if err := xml.Unmarshal(input, &retention); err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-	}
-
-	if retention.RetainUntilDate.Before(time.Now()) {
-		return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
-	}
-	switch retention.Mode {
-	case types.ObjectLockRetentionModeCompliance:
-	case types.ObjectLockRetentionModeGovernance:
-	default:
-		return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
-	}
-
-	return json.Marshal(retention)
-}
-
-func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
-	var retention types.ObjectLockRetention
-	if err := json.Unmarshal(input, &retention); err != nil {
-		return nil, fmt.Errorf("parse object lock retention: %w", err)
-	}
-
-	return &retention, nil
-}
-
-func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResult {
-	if status == nil {
-		return nil
-	}
-
-	if *status {
-		return &s3response.GetObjectLegalHoldResult{
-			Status: types.ObjectLockLegalHoldStatusOn,
-		}
-	}
-
-	return &s3response.GetObjectLegalHoldResult{
-		Status: types.ObjectLockLegalHoldStatusOff,
-	}
-}
-
-func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
-	data, err := be.GetObjectLockConfiguration(ctx, bucket)
-	if err != nil {
-		if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
-			return nil
-		}
-
-		return err
-	}
-
-	var bucketLockConfig BucketLockConfig
-	if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
-		return fmt.Errorf("parse object lock config: %w", err)
-	}
-
-	if !bucketLockConfig.Enabled {
-		return nil
-	}
-
-	checkDefaultRetention := false
-
-	if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
-		expirationDate := *bucketLockConfig.CreatedAt
-		if bucketLockConfig.DefaultRetention.Days != nil {
-			expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
-		}
-		if bucketLockConfig.DefaultRetention.Years != nil {
-			expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
-		}
-
-		if expirationDate.After(time.Now()) {
-			checkDefaultRetention = true
-		}
-	}
-
-	for _, obj := range objects {
-		var key, versionId string
-		if obj.Key != nil {
-			key = *obj.Key
-		}
-		if obj.VersionId != nil {
-			versionId = *obj.VersionId
-		}
-		checkRetention := true
-		retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
-		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
-			continue
-		}
-		if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
-			checkRetention = false
-		}
-		if err != nil && checkRetention {
-			return err
-		}
-
-		if checkRetention {
-			retention, err := ParseObjectLockRetentionOutput(retentionData)
-			if err != nil {
-				return err
-			}
-
-			if retention.Mode != "" && retention.RetainUntilDate != nil {
-				if retention.RetainUntilDate.After(time.Now()) {
-					switch retention.Mode {
-					case types.ObjectLockRetentionModeGovernance:
-						if !bypass {
-							return s3err.GetAPIError(s3err.ErrObjectLocked)
-						} else {
-							policy, err := be.GetBucketPolicy(ctx, bucket)
-							if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-								return s3err.GetAPIError(s3err.ErrObjectLocked)
-							}
-							if err != nil {
-								return err
-							}
-							err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
-							if err != nil {
-								return s3err.GetAPIError(s3err.ErrObjectLocked)
-							}
-						}
-					case types.ObjectLockRetentionModeCompliance:
-						return s3err.GetAPIError(s3err.ErrObjectLocked)
-					}
-				}
-			}
-		}
-
-		checkLegalHold := true
-
-		status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
-		if err != nil {
-			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
-				continue
-			}
-			if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
-				checkLegalHold = false
-			} else {
-				return err
-			}
-		}
-
-		if checkLegalHold && *status {
-			return s3err.GetAPIError(s3err.ErrObjectLocked)
-		}
-
-		if checkDefaultRetention {
-			switch bucketLockConfig.DefaultRetention.Mode {
-			case types.ObjectLockRetentionModeGovernance:
-				if !bypass {
-					return s3err.GetAPIError(s3err.ErrObjectLocked)
-				} else {
-					policy, err := be.GetBucketPolicy(ctx, bucket)
-					if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
-						return s3err.GetAPIError(s3err.ErrObjectLocked)
-					}
-					if err != nil {
-						return err
-					}
-					err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
-					if err != nil {
-						return s3err.GetAPIError(s3err.ErrObjectLocked)
-					}
-				}
-			case types.ObjectLockRetentionModeCompliance:
-				return s3err.GetAPIError(s3err.ErrObjectLocked)
-			}
-		}
-	}
-
-	return nil
-}

aws/signer/v4/functional_test.go

@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
+			t.Errorf("expected %v, but recieved %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
+			t.Errorf("expected %v, but recieved %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but received %v", e, a)
+			t.Errorf("expected %v, but recieved %v", e, a)
 		}
 	}
 }
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
 
 		actual := req.Header.Get("Authorization")
 		if e, a := c.ExpSig, actual; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
+			t.Errorf("expected %v, but recieved %v", e, a)
 		}
 		if e, a := c.OrigURI, req.URL.Path; e != a {
-			t.Errorf("expected %v, but received %v", e, a)
+			t.Errorf("expected %v, but recieved %v", e, a)
 		}
 		if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
-			t.Errorf("expected %v, but received %v", e, a)
+			t.Errorf("expected %v, but recieved %v", e, a)
 		}
 	}
 }

backend/acl/acl.go

@@ -0,0 +1,272 @@
+// Copyright 2020 the authors.
+//
+// Licensed under the Apache License, Version 2.0 (the LICENSE-APACHE file) or
+// the MIT license (the LICENSE-MIT file) at your option. This file may not be
+// copied, modified, or distributed except according to those terms.
+
+// code modified from https://github.com/joshlf/go-acl
+
+package acl
+
+import (
+	"encoding/binary"
+	"fmt"
+	"math"
+	"os"
+	"strings"
+	"syscall"
+)
+
+// ACL represents an access control list as defined
+// in the POSIX.1e draft standard. If an ACL is not
+// valid (see the IsValid method), the behavior of
+// the functions and methods of this package is
+// undefined.
+type ACL []Entry
+
+// ToUnix returns the unix permissions bitmask
+// encoded by a. If a is not valid as defined
+// by a.IsValid, the behavior of ToUnix is
+// undefined.
+func ToUnix(a ACL) os.FileMode {
+	var perms os.FileMode
+	for _, e := range a {
+		switch e.Tag {
+		case TagUserObj:
+			perms |= (e.perms() << 6)
+		case TagGroupObj:
+			perms |= (e.perms() << 3)
+		case TagOther:
+			perms |= e.perms()
+		}
+	}
+	return perms
+}
+
+// String implements the POSIX.1e short text form.
+// For example:
+//
+//	u::rwx,g::r-x,o::---,u:dvader:r--,m::r--
+//
+// This output is produced by an ACL in which the file owner
+// has read, write, and execute; the file group has read and
+// execute; other has no permissions; the user dvader has
+// read; and the mask is read.
+func (a ACL) String() string {
+	strs := make([]string, len(a))
+	for i, e := range a {
+		strs[i] = e.String()
+	}
+	return strings.Join(strs, ",")
+}
+
+// StringLong implements the POSIX.1e long text form.
+// The long text form of the example given above is:
+//
+//	user::rwx
+//	group::r-x
+//	other::---
+//	user:dvader:r--
+//	mask::r--
+func (a ACL) StringLong() string {
+	lines := make([]string, len(a))
+	mask := os.FileMode(7)
+	for _, e := range a {
+		if e.Tag == TagMask {
+			mask = e.perms()
+			break
+		}
+	}
+	for i, e := range a {
+		if (e.Tag == TagUser || e.Tag == TagGroupObj || e.Tag == TagGroup) &&
+			mask|e.perms() != mask {
+			effective := mask & e.perms()
+			lines[i] = fmt.Sprintf("%-20s#effective:%s", e.StringLong(), permString(effective))
+		} else {
+			lines[i] = e.StringLong()
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
+// Tag is the type of an ACL entry tag.
+type Tag tag
+
+const (
+	TagUserObj  Tag = tagUserObj  // Permissions of the file owner
+	TagUser         = tagUser     // Permissions of a specified user
+	TagGroupObj     = tagGroupObj // Permissions of the file group
+	TagGroup        = tagGroup    // Permissions of a specified group
+
+	// Maximum allowed access rights of any entry
+	// with the tag TagUser, TagGroupObj, or TagGroup
+	TagMask  = tagMask
+	TagOther = tagOther // Permissions of a process not matching any other entry
+)
+
+// String implements the POSIX.1e short text form.
+func (t Tag) String() string {
+	switch t {
+	case TagUser, TagUserObj:
+		return "u"
+	case TagGroup, TagGroupObj:
+		return "g"
+	case TagOther:
+		return "o"
+	case TagMask:
+		return "m"
+	default:
+		// TODO(joshlf): what to do in this case?
+		return "?" // non-standard, but not specified in POSIX.1e
+	}
+}
+
+// StringLong implements the POSIX.1e long text form.
+func (t Tag) StringLong() string {
+	switch t {
+	case TagUser, TagUserObj:
+		return "user"
+	case TagGroup, TagGroupObj:
+		return "group"
+	case TagOther:
+		return "other"
+	case TagMask:
+		return "mask"
+	default:
+		// TODO(joshlf): what to do in this case?
+		return "????" // non-standard, but not specified in POSIX.1e
+	}
+}
+
+// Entry represents an entry in an ACL.
+type Entry struct {
+	Tag Tag
+
+	// TODO(joshlf): it would be nice if we could handle
+	// the UID/user name or GID/group name transition
+	// transparently under the hood rather than pushing
+	// the responsibility to the user. However, there are
+	// some subtle considerations:
+	//   - It must be valid to provide a UID/GID for a
+	//     user or group that does not exist (setfactl
+	//     supports this)
+	//   - If the qualifier can be either a UID/GID or
+	//     a user name/group name, there should probably
+	//     be a better way of encoding it (that is,
+	//     better than just setting it to one or the
+	//     other and letting the user implement custom
+	//     logic to tell the difference)
+
+	// The Qualifier specifies what entity (user or group)
+	// this entry applies to. If the Tag is TagUser, it is
+	// a UID; if the Tag is TagGroup, it is a GID; otherwise
+	// the field is ignored. Note that the qualifier must
+	// be a UID or GID - it cannot be, for example, a user name.
+	Qualifier string
+
+	// ACL permissions are taken from a traditional rwx
+	// (read/write/execute) permissions vector. The Perms
+	// field stores these as the lowest three bits -
+	// the bits in any higher positions are ignored.
+	Perms os.FileMode
+}
+
+// Use e.perms() to make sure that only
+// the lowest three bits are set - some
+// algorithms may inadvertently break
+// otherwise (including libacl itself).
+func (e Entry) perms() os.FileMode { return 7 & e.Perms }
+
+var permStrings = []string{
+	0: "---",
+	1: "--x",
+	2: "-w-",
+	3: "-wx",
+	4: "r--",
+	5: "r-x",
+	6: "rw-",
+	7: "rwx",
+}
+
+// assumes perm has only lowest three bits set
+func permString(perm os.FileMode) string {
+	return permStrings[int(perm)]
+}
+
+// String implements the POSIX.1e short text form.
+func (e Entry) String() string {
+	middle := "::"
+	if e.Tag == TagUser || e.Tag == TagGroup {
+		middle = ":" + formatQualifier(e.Qualifier, e.Tag) + ":"
+	}
+	return fmt.Sprintf("%s%s%s", e.Tag, middle, permString(e.perms()))
+}
+
+// StringLong implements the POSIX.1e long text form.
+func (e Entry) StringLong() string {
+	middle := "::"
+	if e.Tag == TagUser || e.Tag == TagGroup {
+		middle = ":" + formatQualifier(e.Qualifier, e.Tag) + ":"
+	}
+	return fmt.Sprintf("%s%s%s", e.Tag.StringLong(), middle, permString(e.perms()))
+}
+
+// overwrite in other files to implement platform-specific behavior
+var formatQualifier = func(q string, _ Tag) string { return q }
+
+/*
+	NOTE: This implementation is largely based on Linux's libacl.
+*/
+
+type tag int
+
+const (
+	// defined in sys/acl.h
+	tagUndefined Tag = 0x00
+	tagUserObj   Tag = 0x01
+	tagUser      Tag = 0x02
+	tagGroupObj  Tag = 0x04
+	tagGroup     Tag = 0x08
+	tagMask      Tag = 0x10
+	tagOther     Tag = 0x20
+
+	// defined in include/acl_ea.h (see libacl source)
+	aclEAAccess    = "system.posix_acl_access"
+	aclEADefault   = "system.posix_acl_default"
+	aclEAVersion   = 2
+	aclEAEntrySize = 8
+	aclUndefinedID = math.MaxUint32 // defined in sys/acl.h
+)
+
+func AclFromXattr(xattr []byte) (acl ACL, err error) {
+	if len(xattr) < 4 {
+		return nil, syscall.EINVAL
+	}
+	version := binary.LittleEndian.Uint32(xattr)
+	xattr = xattr[4:]
+	if version != aclEAVersion {
+		return nil, syscall.EINVAL
+	}
+	if len(xattr)%aclEAEntrySize != 0 {
+		return nil, syscall.EINVAL
+	}
+
+	for len(xattr) > 0 {
+		etag := binary.LittleEndian.Uint16(xattr)
+		sperm := binary.LittleEndian.Uint16(xattr[2:])
+		qid := binary.LittleEndian.Uint32(xattr[4:])
+
+		ent := Entry{
+			Tag:   Tag(etag),
+			Perms: os.FileMode(sperm),
+		}
+		if ent.Tag == TagUser || ent.Tag == TagGroup {
+			ent.Qualifier = fmt.Sprint(qid)
+		}
+
+		acl = append(acl, ent)
+		xattr = xattr[8:]
+	}
+
+	return acl, nil
+}

backend/acl/perm.go

@@ -0,0 +1,63 @@
+package acl
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+const (
+	Read os.FileMode = 1 << iota
+)
+
+func (a ACL) IsReadAllowed(uid, gid uint32) bool {
+	return a.isAllowed(uid, gid, Read)
+}
+
+func (a ACL) isAllowed(uid, gid uint32, perm os.FileMode) bool {
+	for _, e := range a {
+		if e.matches(uid, gid) {
+			return e.isAllowed(perm)
+		}
+	}
+	return false
+}
+
+func (e Entry) matches(uid, gid uint32) bool {
+	switch e.Tag {
+	case TagUserObj:
+		return e.Qualifier == fmt.Sprintf("%v", uid)
+	case TagGroupObj:
+		return e.Qualifier == fmt.Sprintf("%v", gid)
+	case TagMask:
+		return true
+	case TagOther:
+		return true
+	}
+	return false
+}
+
+func (e Entry) isAllowed(perm os.FileMode) bool {
+	return e.Perms&perm != 0
+}
+
+func IsReadAllowed(fi os.FileInfo, uid, gid uint32) bool {
+	fiuser := fi.Sys().(*syscall.Stat_t).Uid
+	figroup := fi.Sys().(*syscall.Stat_t).Gid
+
+	switch {
+	case fiuser == uid:
+		if fi.Mode()&0400 != 0 {
+			return true
+		}
+	case figroup == gid:
+		if fi.Mode()&0040 != 0 {
+			return true
+		}
+	default:
+		if fi.Mode()&0004 != 0 {
+			return true
+		}
+	}
+	return false
+}

backend/backend.go

@@ -18,9 +18,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 	"github.com/versity/versitygw/s3select"
@@ -32,46 +32,38 @@ type Backend interface {
 	Shutdown()
 
 	// bucket operations
-	ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
+	ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
 	HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 	GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
 	CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
 	PutBucketAcl(_ context.Context, bucket string, data []byte) error
-	DeleteBucket(_ context.Context, bucket string) error
-	PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
-	GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
+	DeleteBucket(context.Context, *s3.DeleteBucketInput) error
+	PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
+	GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
 	PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
 	GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
-	DeleteBucketPolicy(_ context.Context, bucket string) error
-	PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
-	GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
-	DeleteBucketOwnershipControls(_ context.Context, bucket string) error
-	PutBucketCors(context.Context, []byte) error
-	GetBucketCors(_ context.Context, bucket string) ([]byte, error)
-	DeleteBucketCors(_ context.Context, bucket string) error
-
 	// multipart operations
-	CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
+	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
 	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
 	ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
 	ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
-	UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
-	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
+	UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
+	UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
 
 	// standard object operations
-	PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
+	PutObject(context.Context, *s3.PutObjectInput) (string, error)
 	HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
-	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
-	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
-	CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
-	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
-	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
-	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
-	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
+	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
+	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
+	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
+	DeleteObject(context.Context, *s3.DeleteObjectInput) error
+	DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
 	PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
-	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
+	ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
 
 	// special case object operations
 	RestoreObject(context.Context, *s3.RestoreObjectInput) error
@@ -87,23 +79,11 @@ type Backend interface {
 	PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
 	DeleteObjectTagging(_ context.Context, bucket, object string) error
 
-	// object lock operations
-	PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
-	GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
-	PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
-	GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
-	PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
-	GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
-
 	// non AWS actions
-	ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
+	ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
 	ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
 }
 
-// InterfaceVersion tracks changes to the Backend interface for plugins.
-// Increment this when the Backend interface changes.
-const InterfaceVersion = 1
-
 type BackendUnsupported struct{}
 
 var _ Backend = &BackendUnsupported{}
@@ -115,7 +95,7 @@ func (BackendUnsupported) Shutdown() {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
 	return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -130,14 +110,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
 func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
+func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
+func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
-	return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -145,30 +125,9 @@ func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, poli
 func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
-	return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutBucketCors(context.Context, []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byte, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
 
-func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
-	return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -182,42 +141,42 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
 func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
 	return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
-	return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
-	return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
+	return "", s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
-	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
-	return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
-	return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
-	return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
+	return s3err.GetAPIError(s3err.ErrNotImplemented)
+}
+func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -241,8 +200,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
 	}
 }
 
-func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
-	return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
+func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
+	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
@@ -265,26 +224,7 @@ func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
-func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
-	return s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
-	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-}
-
-func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
+func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {

backend/common.go

@@ -18,8 +18,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"io"
-	"os"
+	"io/fs"
 	"strconv"
 	"strings"
 	"time"
@@ -29,13 +28,9 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
-const (
-	// this is the media type for directories in AWS and Nextcloud
-	DirContentType     = "application/x-directory"
-	DefaultContentType = "binary/octet-stream"
-
-	// this is the minimum allowed size for mp parts
-	MinPartSize = 5 * 1024 * 1024
+var (
+	// RFC3339TimeFormat RFC3339 time format
+	RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
 )
 
 func IsValidBucketName(name string) bool { return true }
@@ -52,196 +47,65 @@ func (d ByObjectName) Len() int           { return len(d) }
 func (d ByObjectName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
 
-func GetPtrFromString(str string) *string {
-	if str == "" {
-		return nil
-	}
-	return &str
-}
-
-func GetStringFromPtr(str *string) string {
-	if str == nil {
-		return ""
-	}
-	return *str
+func GetStringPtr(s string) *string {
+	return &s
 }
 
 func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
 
-func TrimEtag(etag *string) *string {
-	if etag == nil {
-		return nil
-	}
-
-	return GetPtrFromString(strings.Trim(*etag, "\""))
-}
-
 var (
-	errInvalidRange           = s3err.GetAPIError(s3err.ErrInvalidRange)
-	errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
+	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
 )
 
-// ParseGetObjectRange parses input range header and returns startoffset, length, isValid
-// and error. If no endoffset specified, then length is set to the object size
-// for invalid inputs, it returns no error, but isValid=false
-// `InvalidRange` error is returnd, only if startoffset is greater than the object size
-func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
+// ParseRange parses input range header and returns startoffset, length, and
+// error. If no endoffset specified, then length is set to -1.
+func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
-		return 0, size, false, nil
+		return 0, fi.Size(), nil
 	}
 
 	rangeKv := strings.Split(acceptRange, "=")
 
-	if len(rangeKv) != 2 {
-		return 0, size, false, nil
-	}
-
-	if rangeKv[0] != "bytes" {
-		return 0, size, false, nil
+	if len(rangeKv) < 2 {
+		return 0, 0, errInvalidRange
 	}
 
 	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) != 2 {
-		return 0, size, false, nil
+	if len(bRange) < 1 || len(bRange) > 2 {
+		return 0, 0, errInvalidRange
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
 	if err != nil {
-		return 0, size, false, nil
+		return 0, 0, errInvalidRange
 	}
 
-	if startOffset >= size {
-		return 0, 0, false, errInvalidRange
+	endOffset := int64(-1)
+	if len(bRange) == 1 || bRange[1] == "" {
+		return startOffset, endOffset, nil
 	}
 
-	if bRange[1] == "" {
-		return startOffset, size - startOffset, true, nil
-	}
-
-	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
 	if err != nil {
-		return 0, size, false, nil
+		return 0, 0, errInvalidRange
 	}
 
 	if endOffset < startOffset {
-		return 0, size, false, nil
-	}
-
-	if endOffset >= size {
-		return startOffset, size - startOffset, true, nil
-	}
-
-	return startOffset, endOffset - startOffset + 1, true, nil
-}
-
-// ParseCopySourceRange parses input range header and returns startoffset, length
-// and error. If no endoffset specified, then length is set to the object size
-func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
-	if acceptRange == "" {
-		return 0, size, nil
-	}
-
-	rangeKv := strings.Split(acceptRange, "=")
-
-	if len(rangeKv) != 2 {
-		return 0, 0, errInvalidCopySourceRange
-	}
-
-	if rangeKv[0] != "bytes" {
-		return 0, 0, errInvalidCopySourceRange
-	}
-
-	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) != 2 {
-		return 0, 0, errInvalidCopySourceRange
-	}
-
-	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
-	if err != nil {
-		return 0, 0, errInvalidCopySourceRange
-	}
-
-	if startOffset >= size {
-		return 0, 0, s3err.CreateExceedingRangeErr(size)
-	}
-
-	if bRange[1] == "" {
-		return startOffset, size - startOffset + 1, nil
-	}
-
-	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
-	if err != nil {
-		return 0, 0, errInvalidCopySourceRange
-	}
-
-	if endOffset < startOffset {
-		return 0, 0, errInvalidCopySourceRange
-	}
-
-	if endOffset >= size {
-		return 0, 0, s3err.CreateExceedingRangeErr(size)
+		return 0, 0, errInvalidRange
 	}
 
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
-// ParseCopySource parses x-amz-copy-source header and returns source bucket,
-// source object, versionId, error respectively
-func ParseCopySource(copySourceHeader string) (string, string, string, error) {
-	if copySourceHeader[0] == '/' {
-		copySourceHeader = copySourceHeader[1:]
-	}
-
-	var copySource, versionId string
-	i := strings.LastIndex(copySourceHeader, "?versionId=")
-	if i == -1 {
-		copySource = copySourceHeader
-	} else {
-		copySource = copySourceHeader[:i]
-		versionId = copySourceHeader[i+11:]
-	}
-
-	srcBucket, srcObject, ok := strings.Cut(copySource, "/")
-	if !ok {
-		return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
-	}
-
-	return srcBucket, srcObject, versionId, nil
-}
-
-// ParseObjectTags parses the url encoded input string into
-// map[string]string key-value tag set
-func ParseObjectTags(t string) (map[string]string, error) {
-	if t == "" {
-		return nil, nil
-	}
-
-	tagging := make(map[string]string)
-
-	tagParts := strings.Split(t, "&")
-	for _, prt := range tagParts {
-		p := strings.Split(prt, "=")
-		if len(p) != 2 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
-		}
-		if len(p[0]) > 128 || len(p[1]) > 256 {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
-		}
-		tagging[p[0]] = p[1]
-	}
-
-	return tagging, nil
-}
-
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
 		partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
 	}
-
-	return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
+	s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
+	return s3MD5
 }
 
 func getEtagBytes(etag string) []byte {
@@ -256,16 +120,3 @@ func md5String(data []byte) string {
 	sum := md5.Sum(data)
 	return hex.EncodeToString(sum[:])
 }
-
-type FileSectionReadCloser struct {
-	R io.Reader
-	F *os.File
-}
-
-func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
-	return f.R.Read(p)
-}
-
-func (f *FileSectionReadCloser) Close() error {
-	return f.F.Close()
-}

backend/meta/meta.go

@@ -1,42 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package meta
-
-import "os"
-
-// MetadataStorer defines the interface for managing metadata.
-// When object == "", the operation is on the bucket.
-type MetadataStorer interface {
-	// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
-	// Returns the value of the attribute, or an error if the attribute does not exist.
-	RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
-
-	// StoreAttribute stores the value of a specific attribute for an object or a bucket.
-	// If attribute already exists, new attribute should replace existing.
-	// Returns an error if the operation fails.
-	StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
-
-	// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
-	// Returns an error if the operation fails.
-	DeleteAttribute(bucket, object, attribute string) error
-
-	// ListAttributes lists all attributes for an object or a bucket.
-	// Returns list of attribute names, or an error if the operation fails.
-	ListAttributes(bucket, object string) ([]string, error)
-
-	// DeleteAttributes removes all attributes for an object or a bucket.
-	// Returns an error if the operation fails.
-	DeleteAttributes(bucket, object string) error
-}

backend/meta/none.go

@@ -1,54 +0,0 @@
-// Copyright 2025 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package meta
-
-import (
-	"os"
-)
-
-// NoMeta is a metadata storer that does not store metadata.
-// This can be useful for read only mounts where attempting to store metadata
-// would fail.
-type NoMeta struct{}
-
-// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
-// always returns ErrNoSuchKey
-func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
-	return nil, ErrNoSuchKey
-}
-
-// StoreAttribute stores the value of a specific attribute for an object or a bucket.
-// always returns nil without storing the attribute
-func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
-	return nil
-}
-
-// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
-// always returns nil without deleting the attribute
-func (NoMeta) DeleteAttribute(_, _, _ string) error {
-	return nil
-}
-
-// ListAttributes lists all attributes for an object or a bucket.
-// always returns an empty list of attributes
-func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
-	return []string{}, nil
-}
-
-// DeleteAttributes removes all attributes for an object or a bucket.
-// always returns nil without deleting any attributes
-func (NoMeta) DeleteAttributes(bucket, object string) error {
-	return nil
-}

backend/meta/sidecar.go

@@ -1,139 +0,0 @@
-// Copyright 2025 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package meta
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-)
-
-// SideCar is a metadata storer that uses sidecar files to store metadata.
-type SideCar struct {
-	dir string
-}
-
-const (
-	sidecarmeta = "meta"
-)
-
-// NewSideCar creates a new SideCar metadata storer.
-func NewSideCar(dir string) (SideCar, error) {
-	fi, err := os.Lstat(dir)
-	if err != nil {
-		return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
-	}
-	if !fi.IsDir() {
-		return SideCar{}, fmt.Errorf("not a directory")
-	}
-
-	return SideCar{dir: dir}, nil
-}
-
-// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
-func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
-	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
-	if object == "" {
-		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
-	}
-	attr := filepath.Join(metadir, attribute)
-
-	value, err := os.ReadFile(attr)
-	if errors.Is(err, os.ErrNotExist) {
-		return nil, ErrNoSuchKey
-	}
-	if err != nil {
-		return nil, fmt.Errorf("failed to read attribute: %v", err)
-	}
-
-	return value, nil
-}
-
-// StoreAttribute stores the value of a specific attribute for an object or a bucket.
-func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
-	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
-	if object == "" {
-		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
-	}
-	err := os.MkdirAll(metadir, 0777)
-	if err != nil {
-		return fmt.Errorf("failed to create metadata directory: %v", err)
-	}
-
-	attr := filepath.Join(metadir, attribute)
-	err = os.WriteFile(attr, value, 0666)
-	if err != nil {
-		return fmt.Errorf("failed to write attribute: %v", err)
-	}
-
-	return nil
-}
-
-// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
-func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
-	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
-	if object == "" {
-		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
-	}
-	attr := filepath.Join(metadir, attribute)
-
-	err := os.Remove(attr)
-	if errors.Is(err, os.ErrNotExist) {
-		return ErrNoSuchKey
-	}
-	if err != nil {
-		return fmt.Errorf("failed to remove attribute: %v", err)
-	}
-
-	return nil
-}
-
-// ListAttributes lists all attributes for an object or a bucket.
-func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
-	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
-	if object == "" {
-		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
-	}
-
-	ents, err := os.ReadDir(metadir)
-	if errors.Is(err, os.ErrNotExist) {
-		return []string{}, nil
-	}
-	if err != nil {
-		return nil, fmt.Errorf("failed to list attributes: %v", err)
-	}
-
-	var attrs []string
-	for _, ent := range ents {
-		attrs = append(attrs, ent.Name())
-	}
-
-	return attrs, nil
-}
-
-// DeleteAttributes removes all attributes for an object or a bucket.
-func (s SideCar) DeleteAttributes(bucket, object string) error {
-	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
-	if object == "" {
-		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
-	}
-
-	err := os.RemoveAll(metadir)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return fmt.Errorf("failed to remove attributes: %v", err)
-	}
-	return nil
-}

backend/meta/xattr.go

@@ -1,126 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package meta
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"syscall"
-
-	"github.com/pkg/xattr"
-	"github.com/versity/versitygw/s3err"
-)
-
-const (
-	xattrPrefix = "user."
-)
-
-var (
-	// ErrNoSuchKey is returned when the key does not exist.
-	ErrNoSuchKey = errors.New("no such key")
-)
-
-type XattrMeta struct{}
-
-// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
-	if f != nil {
-		b, err := xattr.FGet(f, xattrPrefix+attribute)
-		if errors.Is(err, xattr.ENOATTR) {
-			return nil, ErrNoSuchKey
-		}
-		return b, err
-	}
-
-	b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
-	if errors.Is(err, xattr.ENOATTR) {
-		return nil, ErrNoSuchKey
-	}
-	return b, err
-}
-
-// StoreAttribute stores the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
-	if f != nil {
-		err := xattr.FSet(f, xattrPrefix+attribute, value)
-		if errors.Is(err, syscall.EROFS) {
-			return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
-		}
-		return err
-	}
-
-	err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
-	if errors.Is(err, syscall.EROFS) {
-		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
-	}
-	return err
-}
-
-// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
-func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
-	err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
-	if errors.Is(err, xattr.ENOATTR) {
-		return ErrNoSuchKey
-	}
-	if errors.Is(err, syscall.EROFS) {
-		return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
-	}
-	return err
-}
-
-// DeleteAttributes is not implemented for xattr since xattrs
-// are automatically removed when the file is deleted.
-func (x XattrMeta) DeleteAttributes(bucket, object string) error {
-	return nil
-}
-
-// ListAttributes lists all attributes for an object in a bucket.
-func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
-	attrs, err := xattr.List(filepath.Join(bucket, object))
-	if err != nil {
-		return nil, err
-	}
-	attributes := make([]string, 0, len(attrs))
-	for _, attr := range attrs {
-		if !isUserAttr(attr) {
-			continue
-		}
-		attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
-	}
-	return attributes, nil
-}
-
-func isUserAttr(attr string) bool {
-	return strings.HasPrefix(attr, xattrPrefix)
-}
-
-// Test is a helper function to test if xattrs are supported.
-func (x XattrMeta) Test(path string) error {
-	// check for platform support
-	if !xattr.XATTR_SUPPORTED {
-		return fmt.Errorf("xattrs are not supported on this platform")
-	}
-
-	// check if the filesystem supports xattrs
-	_, err := xattr.Get(path, "user.test")
-	if errors.Is(err, syscall.ENOTSUP) {
-		return fmt.Errorf("xattrs are not supported on this filesystem")
-	}
-
-	return nil
-}

backend/mkdir.go

@@ -1,77 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-// Copyright 2024 Versity Software
-
-// MkdirAll borrowed from stdlib to add ability to set ownership
-// as directories are created
-
-package backend
-
-import (
-	"io/fs"
-	"os"
-
-	"github.com/versity/versitygw/s3err"
-)
-
-// MkdirAll is similar to os.MkdirAll but it will return
-// ErrObjectParentIsFile when appropriate
-// MkdirAll creates a directory named path,
-// along with any necessary parents, and returns nil,
-// or else returns an error.
-// The permission bits perm (before umask) are used for all
-// directories that MkdirAll creates.
-// Any newly created directory is set to provided uid/gid ownership.
-// If path is already a directory, MkdirAll does nothing
-// and returns nil.
-// Any directory created will be set to provided uid/gid ownership
-// if doChown is true.
-func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent.
-		err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, dirPerm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return err
-	}
-	if doChown {
-		err = os.Chown(path, uid, gid)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

backend/plugin/plugin.go

@@ -1,516 +0,0 @@
-// Copyright 2025 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package vgwplugin
-
-import (
-	"bufio"
-	"context"
-	"fmt"
-	"plugin"
-	"reflect"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3response"
-)
-
-// The plugin backend is used to dynamically load a Go plugin at runtime.
-// It loads the plugin and calls the InitPlugin function to initialize it.
-// A config string option is passed to init the plugin, it is expected that the
-// plugin will handle its own configuration and initialization from this.
-// If the plugin cannot be loaded or initialized, it returns an error.
-// The InitPlugin function should be defined in the plugin and should have
-// the signature func(configfile string) (version int, err error).
-// The plugin should also implement the backend.Backend interface functions.
-// However, the plugin does not need to implement all functions of the
-// backend.Backend interface. It can implement only the functions it needs.
-// Any non-implemented functions will return an error indicating that
-// the function is not implemented.
-// The plugin file should be compiled with the same Go version as the
-// application using it. The plugin file should be built with the
-// -buildmode=plugin flag.
-// Example: go build -buildmode=plugin -o myplugin.so myplugin.go
-// See the following for caveats and details:
-// https://pkg.go.dev/plugin#hdr-Warnings
-
-// PluginBackend implements the backend.Backend interface using Go plugins.
-type PluginBackend struct {
-	p *plugin.Plugin
-}
-
-// NewPluginBackend creates a new PluginBackend. The path parameter should
-// point to the compiled plugin file (e.g., .so file).
-func NewPluginBackend(path, config string) (*PluginBackend, error) {
-	p, err := plugin.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open plugin: %w", err)
-	}
-
-	initSymbol, err := p.Lookup("InitPlugin")
-	if err != nil {
-		return nil, fmt.Errorf("failed to lookup InitPlugin symbol: %w", err)
-	}
-
-	initFunc, ok := initSymbol.(func(string) (int, error))
-	if !ok {
-		return nil, fmt.Errorf("InitPlugin symbol is not a func() (int, error)")
-	}
-
-	version, err := initFunc(config)
-	if err != nil {
-		return nil, fmt.Errorf("InitPlugin failed: %w", err)
-	}
-
-	if version != backend.InterfaceVersion {
-		return nil, fmt.Errorf("plugin interface version mismatch: gateway %v, plugin %v",
-			backend.InterfaceVersion, version)
-	}
-
-	return &PluginBackend{p: p}, nil
-}
-
-func (p *PluginBackend) callPluginFunc(name string, args []any) ([]reflect.Value, error) {
-	symbol, err := p.p.Lookup(name)
-	if err != nil {
-		return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
-	}
-
-	symbolValue := reflect.ValueOf(symbol)
-	if symbolValue.Kind() != reflect.Func {
-		return nil, fmt.Errorf("symbol %s is not a function", name)
-	}
-
-	numIn := symbolValue.Type().NumIn()
-	if len(args) != numIn {
-		return nil, fmt.Errorf("incorrect number of arguments for function %s, expected %d, got %d", name, numIn, len(args))
-	}
-
-	in := make([]reflect.Value, len(args))
-	for i := range args {
-		in[i] = reflect.ValueOf(args[i])
-	}
-
-	return symbolValue.Call(in), nil
-}
-
-func (p *PluginBackend) String() string { return "Plugin Gateway" }
-func (p *PluginBackend) Shutdown()      {}
-
-func (p *PluginBackend) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
-	results, err := p.callPluginFunc("ListBuckets", []any{ctx, input})
-	if err != nil {
-		return s3response.ListAllMyBucketsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListAllMyBucketsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
-	results, err := p.callPluginFunc("HeadBucket", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.HeadBucketOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
-	results, err := p.callPluginFunc("GetBucketAcl", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, defaultACL []byte) error {
-	_, err := p.callPluginFunc("CreateBucket", []any{ctx, input, defaultACL})
-	return err
-}
-
-func (p *PluginBackend) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
-	_, err := p.callPluginFunc("PutBucketAcl", []any{ctx, bucket, data})
-	return err
-}
-
-func (p *PluginBackend) DeleteBucket(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucket", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
-	_, err := p.callPluginFunc("PutBucketVersioning", []any{ctx, bucket, status})
-	return err
-}
-
-func (p *PluginBackend) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
-	results, err := p.callPluginFunc("GetBucketVersioning", []any{ctx, bucket})
-	if err != nil {
-		return s3response.GetBucketVersioningOutput{}, err
-	}
-
-	return results[0].Interface().(s3response.GetBucketVersioningOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
-	_, err := p.callPluginFunc("PutBucketPolicy", []any{ctx, bucket, policy})
-	return err
-}
-
-func (p *PluginBackend) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetBucketPolicy", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteBucketPolicy(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketPolicy", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
-	_, err := p.callPluginFunc("PutBucketOwnershipControls", []any{ctx, bucket, ownership})
-	return err
-}
-
-func (p *PluginBackend) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
-	results, err := p.callPluginFunc("GetBucketOwnershipControls", []any{ctx, bucket})
-	if err != nil {
-		return "", err
-	}
-
-	return results[0].Interface().(types.ObjectOwnership), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketOwnershipControls", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) PutBucketCors(ctx context.Context, data []byte) error {
-	_, err := p.callPluginFunc("PutBucketCors", []any{ctx, data})
-	return err
-}
-
-func (p *PluginBackend) GetBucketCors(ctx context.Context, bucket string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetBucketCors", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteBucketCors(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketCors", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
-	results, err := p.callPluginFunc("CreateMultipartUpload", []any{ctx, input})
-	if err != nil {
-		return s3response.InitiateMultipartUploadResult{}, err
-	}
-
-	return results[0].Interface().(s3response.InitiateMultipartUploadResult), convertError(results[1])
-}
-
-func (p *PluginBackend) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	results, err := p.callPluginFunc("CompleteMultipartUpload", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.CompleteMultipartUploadOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
-	_, err := p.callPluginFunc("AbortMultipartUpload", []any{ctx, input})
-	return err
-}
-
-func (p *PluginBackend) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
-	results, err := p.callPluginFunc("ListMultipartUploads", []any{ctx, input})
-	if err != nil {
-		return s3response.ListMultipartUploadsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListMultipartUploadsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
-	results, err := p.callPluginFunc("ListParts", []any{ctx, input})
-	if err != nil {
-		return s3response.ListPartsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListPartsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
-	results, err := p.callPluginFunc("UploadPart", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.UploadPartOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
-	results, err := p.callPluginFunc("UploadPartCopy", []any{ctx, input})
-	if err != nil {
-		return s3response.CopyPartResult{}, err
-	}
-
-	return results[0].Interface().(s3response.CopyPartResult), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObject(ctx context.Context, input s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
-	results, err := p.callPluginFunc("PutObject", []any{ctx, input})
-	if err != nil {
-		return s3response.PutObjectOutput{}, err
-	}
-
-	return results[0].Interface().(s3response.PutObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	results, err := p.callPluginFunc("HeadObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.HeadObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
-	results, err := p.callPluginFunc("GetObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.GetObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetObjectAcl(ctx context.Context, input *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
-	results, err := p.callPluginFunc("GetObjectAcl", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.GetObjectAclOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
-	results, err := p.callPluginFunc("GetObjectAttributes", []any{ctx, input})
-	if err != nil {
-		return s3response.GetObjectAttributesResponse{}, err
-	}
-
-	return results[0].Interface().(s3response.GetObjectAttributesResponse), convertError(results[1])
-}
-
-func (p *PluginBackend) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	results, err := p.callPluginFunc("CopyObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.CopyObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
-	results, err := p.callPluginFunc("ListObjects", []any{ctx, input})
-	if err != nil {
-		return s3response.ListObjectsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListObjectsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
-	results, err := p.callPluginFunc("ListObjectsV2", []any{ctx, input})
-	if err != nil {
-		return s3response.ListObjectsV2Result{}, err
-	}
-
-	return results[0].Interface().(s3response.ListObjectsV2Result), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
-	results, err := p.callPluginFunc("DeleteObject", []any{ctx, input})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(*s3.DeleteObjectOutput), convertError(results[1])
-}
-
-func (p *PluginBackend) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
-	results, err := p.callPluginFunc("DeleteObjects", []any{ctx, input})
-	if err != nil {
-		return s3response.DeleteResult{}, err
-	}
-
-	return results[0].Interface().(s3response.DeleteResult), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectAcl(ctx context.Context, input *s3.PutObjectAclInput) error {
-	_, err := p.callPluginFunc("PutObjectAcl", []any{ctx, input})
-	return err
-}
-
-func (p *PluginBackend) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
-	results, err := p.callPluginFunc("ListObjectVersions", []any{ctx, input})
-	if err != nil {
-		return s3response.ListVersionsResult{}, err
-	}
-
-	return results[0].Interface().(s3response.ListVersionsResult), convertError(results[1])
-}
-
-func (p *PluginBackend) RestoreObject(ctx context.Context, input *s3.RestoreObjectInput) error {
-	_, err := p.callPluginFunc("RestoreObject", []any{ctx, input})
-	return err
-}
-
-func (p *PluginBackend) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
-	results, err := p.callPluginFunc("SelectObjectContent", []any{ctx, input})
-	if err != nil {
-		return func(w *bufio.Writer) {}
-	}
-
-	return results[0].Interface().(func(w *bufio.Writer))
-}
-
-func (p *PluginBackend) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) {
-	results, err := p.callPluginFunc("GetBucketTagging", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(map[string]string), convertError(results[1])
-}
-
-func (p *PluginBackend) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
-	_, err := p.callPluginFunc("PutBucketTagging", []any{ctx, bucket, tags})
-	return err
-}
-
-func (p *PluginBackend) DeleteBucketTagging(ctx context.Context, bucket string) error {
-	_, err := p.callPluginFunc("DeleteBucketTagging", []any{ctx, bucket})
-	return err
-}
-
-func (p *PluginBackend) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
-	results, err := p.callPluginFunc("GetObjectTagging", []any{ctx, bucket, object})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().(map[string]string), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
-	_, err := p.callPluginFunc("PutObjectTagging", []any{ctx, bucket, object, tags})
-	return err
-}
-
-func (p *PluginBackend) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
-	_, err := p.callPluginFunc("DeleteObjectTagging", []any{ctx, bucket, object})
-	return err
-}
-
-func (p *PluginBackend) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
-	_, err := p.callPluginFunc("PutObjectLockConfiguration", []any{ctx, bucket, config})
-	return err
-}
-
-func (p *PluginBackend) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetObjectLockConfiguration", []any{ctx, bucket})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
-	_, err := p.callPluginFunc("PutObjectRetention", []any{ctx, bucket, object, versionId, bypass, retention})
-	return err
-}
-
-func (p *PluginBackend) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
-	results, err := p.callPluginFunc("GetObjectRetention", []any{ctx, bucket, object, versionId})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]byte), convertError(results[1])
-}
-
-func (p *PluginBackend) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
-	_, err := p.callPluginFunc("PutObjectLegalHold", []any{ctx, bucket, object, versionId, status})
-	return err
-}
-
-func (p *PluginBackend) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
-	results, err := p.callPluginFunc("GetObjectLegalHold", []any{ctx, bucket, object, versionId})
-	if err != nil {
-		return nil, err
-	}
-
-	val := results[0].Interface()
-
-	if val == nil {
-		return nil, convertError(results[1])
-	}
-
-	return val.(*bool), convertError(results[1])
-}
-
-func (p *PluginBackend) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
-	_, err := p.callPluginFunc("ChangeBucketOwner", []any{ctx, bucket, acl})
-	return err
-}
-
-func (p *PluginBackend) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
-	results, err := p.callPluginFunc("ListBucketsAndOwners", []any{ctx})
-	if err != nil {
-		return nil, err
-	}
-
-	return results[0].Interface().([]s3response.Bucket), convertError(results[1])
-}
-
-func convertError(result reflect.Value) error {
-	if result.IsNil() {
-		return nil
-	}
-
-	err, ok := result.Interface().(error)
-	if !ok {
-		return fmt.Errorf("expected error, got %T", result.Interface())
-	}
-
-	return err
-}
-
-var _ backend.Backend = &PluginBackend{}

backend/posix/posix_darwin.go

@@ -12,9 +12,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build !linux
-// +build !linux
-
 package posix
 
 import (
@@ -24,11 +21,6 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"syscall"
-
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3err"
 )
 
 type tmpfile struct {
@@ -38,42 +30,20 @@ type tmpfile struct {
 	size    int64
 }
 
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
-	uid, gid, doChown := p.getChownIDs(acct)
-
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// Create a temp file for upload while in progress (see link comments below).
-	var err error
-	err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+	err := os.MkdirAll(dir, 0700)
 	if err != nil {
-		if errors.Is(err, syscall.EROFS) {
-			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
-		}
 		return nil, fmt.Errorf("make temp dir: %w", err)
 	}
 	f, err := os.CreateTemp(dir,
 		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
 	if err != nil {
-		if errors.Is(err, syscall.EROFS) {
-			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
-		}
-		return nil, fmt.Errorf("create temp file: %w", err)
+		return nil, err
 	}
-
-	if doChown {
-		err := f.Chown(uid, gid)
-		if err != nil {
-			return nil, fmt.Errorf("set temp file ownership: %w", err)
-		}
-	}
-
 	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
 }
 
-var (
-	// TODO: make this configurable
-	defaultFilePerm fs.FileMode = 0644
-)
-
 func (tmp *tmpfile) link() error {
 	tempname := tmp.f.Name()
 	// cleanup in case anything goes wrong, if rename succeeds then
@@ -91,9 +61,6 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	// reset default file mode because CreateTemp uses 0600
-	tmp.f.Chmod(defaultFilePerm)
-
 	err = tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)
@@ -120,7 +87,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
-
-func (tmp *tmpfile) File() *os.File {
-	return tmp.f
-}

backend/posix/posix_linux.go

@@ -12,9 +12,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build linux
-// +build linux
-
 package posix
 
 import (
@@ -27,48 +24,30 @@ import (
 	"strconv"
 	"syscall"
 
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3err"
 	"golang.org/x/sys/unix"
 )
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f          *os.File
-	bucket     string
-	objname    string
-	isOTmp     bool
-	size       int64
-	needsChown bool
-	uid        int
-	gid        int
-	newDirPerm fs.FileMode
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
 }
 
-var (
-	// TODO: make this configurable
-	defaultFilePerm uint32 = 0644
-)
-
-func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
-	uid, gid, doChown := p.getChownIDs(acct)
-
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
 	// Not all filesystems support this, so fallback to CreateTemp for when
 	// this is not supported.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
 	if err != nil {
-		if errors.Is(err, syscall.EROFS) {
-			return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
-		}
-
 		// O_TMPFILE not supported, try fallback
-		err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
+		err := os.MkdirAll(dir, 0700)
 		if err != nil {
 			return nil, fmt.Errorf("make temp dir: %w", err)
 		}
@@ -77,27 +56,11 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 		if err != nil {
 			return nil, err
 		}
-		tmp := &tmpfile{
-			f:          f,
-			bucket:     bucket,
-			objname:    obj,
-			size:       size,
-			needsChown: doChown,
-			uid:        uid,
-			gid:        gid,
-		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
 		// falloc is best effort, its fine if this fails
-		if size > 0 && dofalloc {
+		if size > 0 {
 			tmp.falloc()
 		}
-
-		if doChown {
-			err := f.Chown(uid, gid)
-			if err != nil {
-				return nil, fmt.Errorf("set temp file ownership: %w", err)
-			}
-		}
-
 		return tmp, nil
 	}
 
@@ -105,30 +68,11 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{
-		f:          f,
-		bucket:     bucket,
-		objname:    obj,
-		isOTmp:     true,
-		size:       size,
-		needsChown: doChown,
-		uid:        uid,
-		gid:        gid,
-		newDirPerm: p.newDirPerm,
-	}
-
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
 	// falloc is best effort, its fine if this fails
-	if size > 0 && dofalloc {
+	if size > 0 {
 		tmp.falloc()
 	}
-
-	if doChown {
-		err := f.Chown(uid, gid)
-		if err != nil {
-			return nil, fmt.Errorf("set temp file ownership: %w", err)
-		}
-	}
-
 	return tmp, nil
 }
 
@@ -141,9 +85,6 @@ func (tmp *tmpfile) falloc() error {
 }
 
 func (tmp *tmpfile) link() error {
-	// make sure this is cleaned up in all error cases
-	defer tmp.f.Close()
-
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
 	// with any other simultaneous uploads. The final operation is to move the
@@ -156,13 +97,6 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	dir := filepath.Dir(objPath)
-
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
-	if err != nil {
-		return fmt.Errorf("make parent dir: %w", err)
-	}
-
 	if !tmp.isOTmp {
 		// O_TMPFILE not suported, use fallback
 		return tmp.fallbackLink()
@@ -174,27 +108,17 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dirf, err := os.Open(dir)
+	dir, err := os.Open(filepath.Dir(objPath))
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dirf.Close()
+	defer dir.Close()
 
-	for {
-		err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-			int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
-		if errors.Is(err, syscall.EEXIST) {
-			err := os.Remove(objPath)
-			if err != nil && !errors.Is(err, fs.ErrNotExist) {
-				return fmt.Errorf("remove stale path: %w", err)
-			}
-			continue
-		}
-		if err != nil {
-			return fmt.Errorf("link tmpfile (fd %q as %q): %w",
-				filepath.Base(tmp.f.Name()), objPath, err)
-		}
-		break
+	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+	if err != nil {
+		return fmt.Errorf("link tmpfile (%q in %q): %w",
+			filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
 	}
 
 	err = tmp.f.Close()
@@ -211,9 +135,6 @@ func (tmp *tmpfile) fallbackLink() error {
 	// this will no longer exist
 	defer os.Remove(tempname)
 
-	// reset default file mode because CreateTemp uses 0600
-	tmp.f.Chmod(fs.FileMode(defaultFilePerm))
-
 	err := tmp.f.Close()
 	if err != nil {
 		return fmt.Errorf("close tmpfile: %w", err)
@@ -241,7 +162,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
-
-func (tmp *tmpfile) File() *os.File {
-	return tmp.f
-}

backend/posix/posix_windows.go

@@ -0,0 +1,89 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type tmpfile struct {
+	f       *os.File
+	bucket  string
+	objname string
+	size    int64
+}
+
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
+	// Create a temp file for upload while in progress (see link comments below).
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return nil, fmt.Errorf("make temp dir: %w", err)
+	}
+	f, err := os.CreateTemp(dir,
+		fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+	if err != nil {
+		return nil, err
+	}
+	return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
+}
+
+func (tmp *tmpfile) link() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	// We use Rename as the atomic operation for object puts. The upload is
+	// written to a temp file to not conflict with any other simultaneous
+	// uploads. The final operation is to move the temp file into place for
+	// the object. This ensures the object semantics of last upload completed
+	// wins and is not some combination of writes from simultaneous uploads.
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err := os.Remove(objPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("remove stale path: %w", err)
+	}
+
+	err = tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
+func (tmp *tmpfile) Write(b []byte) (int, error) {
+	if int64(len(b)) > tmp.size {
+		return 0, fmt.Errorf("write exceeds content length")
+	}
+
+	n, err := tmp.f.Write(b)
+	tmp.size -= int64(n)
+	return n, err
+}
+
+func (tmp *tmpfile) cleanup() {
+	tmp.f.Close()
+}

backend/s3proxy/client.go

@@ -33,12 +33,6 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
 		return nil, err
 	}
 
-	if s.endpoint != "" {
-		return s3.NewFromConfig(cfg, func(o *s3.Options) {
-			o.BaseEndpoint = &s.endpoint
-		}), nil
-	}
-
 	return s3.NewFromConfig(cfg), nil
 }
 
@@ -56,6 +50,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 		config.WithHTTPClient(client),
 	}
 
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
 	if s.disableChecksum {
 		opts = append(opts,
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -68,3 +67,13 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 
 	return config.LoadDefaultConfig(ctx, opts...)
 }
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}

backend/scoutfs/scoutfs_compat.go

@@ -17,31 +17,23 @@
 package scoutfs
 
 import (
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 
 	"golang.org/x/sys/unix"
 
 	"github.com/versity/scoutfs-go"
-	"github.com/versity/versitygw/auth"
-	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
-func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
-	metastore := meta.XattrMeta{}
-
-	p, err := posix.New(rootdir, metastore, posix.PosixOpts{
-		ChownUID:    opts.ChownUID,
-		ChownGID:    opts.ChownGID,
-		BucketLinks: opts.BucketLinks,
-		NewDirPerm:  opts.NewDirPerm,
-	})
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
+	p, err := posix.New(rootdir)
 	if err != nil {
 		return nil, err
 	}
@@ -51,73 +43,71 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
-	return &ScoutFS{
-		Posix:            p,
-		rootfd:           f,
-		rootdir:          rootdir,
-		meta:             metastore,
-		chownuid:         opts.ChownUID,
-		chowngid:         opts.ChownGID,
-		glaciermode:      opts.GlacierMode,
-		newDirPerm:       opts.NewDirPerm,
-		disableNoArchive: opts.DisableNoArchive,
-	}, nil
+	s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
+	for _, opt := range opts {
+		opt(s)
+	}
+
+	return s, nil
 }
 
 const procfddir = "/proc/self/fd"
 
 type tmpfile struct {
-	f          *os.File
-	bucket     string
-	objname    string
-	size       int64
-	needsChown bool
-	uid        int
-	gid        int
-	newDirPerm fs.FileMode
+	f       *os.File
+	bucket  string
+	objname string
+	isOTmp  bool
+	size    int64
 }
 
-var (
-	defaultFilePerm uint32 = 0644
-)
-
-func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
-	uid, gid, doChown := s.getChownIDs(acct)
-
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
 	// This can help reduce contention within the namespace (parent directories),
 	// etc. And will auto cleanup the inode on close if we never link this
 	// file descriptor into the namespace.
-	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
+	// Not all filesystems support this, so fallback to CreateTemp for when
+	// this is not supported.
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
 	if err != nil {
-		return nil, err
+		// O_TMPFILE not supported, try fallback
+		err := os.MkdirAll(dir, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("make temp dir: %w", err)
+		}
+		f, err := os.CreateTemp(dir,
+			fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
+		if err != nil {
+			return nil, err
+		}
+		tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
+		// falloc is best effort, its fine if this fails
+		if size > 0 {
+			tmp.falloc()
+		}
+		return tmp, nil
 	}
 
 	// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
 	// later to link file into namespace
 	f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
 
-	tmp := &tmpfile{
-		f:          f,
-		bucket:     bucket,
-		objname:    obj,
-		size:       size,
-		needsChown: doChown,
-		uid:        uid,
-		gid:        gid,
-		newDirPerm: s.newDirPerm,
+	tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
+	// falloc is best effort, its fine if this fails
+	if size > 0 {
+		tmp.falloc()
 	}
-
-	if doChown {
-		err := f.Chown(uid, gid)
-		if err != nil {
-			return nil, fmt.Errorf("set temp file ownership: %w", err)
-		}
-	}
-
 	return tmp, nil
 }
 
+func (tmp *tmpfile) falloc() error {
+	err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
+	if err != nil {
+		return fmt.Errorf("fallocate: %v", err)
+	}
+	return nil
+}
+
 func (tmp *tmpfile) link() error {
 	// We use Linkat/Rename as the atomic operation for object puts. The
 	// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
@@ -131,11 +121,9 @@ func (tmp *tmpfile) link() error {
 		return fmt.Errorf("remove stale path: %w", err)
 	}
 
-	dir := filepath.Dir(objPath)
-
-	err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
-	if err != nil {
-		return fmt.Errorf("make parent dir: %w", err)
+	if !tmp.isOTmp {
+		// O_TMPFILE not suported, use fallback
+		return tmp.fallbackLink()
 	}
 
 	procdir, err := os.Open(procfddir)
@@ -144,14 +132,14 @@ func (tmp *tmpfile) link() error {
 	}
 	defer procdir.Close()
 
-	dirf, err := os.Open(dir)
+	dir, err := os.Open(filepath.Dir(objPath))
 	if err != nil {
 		return fmt.Errorf("open parent dir: %w", err)
 	}
-	defer dirf.Close()
+	defer dir.Close()
 
 	err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
-		int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
+		int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
 	if err != nil {
 		return fmt.Errorf("link tmpfile: %w", err)
 	}
@@ -164,6 +152,26 @@ func (tmp *tmpfile) link() error {
 	return nil
 }
 
+func (tmp *tmpfile) fallbackLink() error {
+	tempname := tmp.f.Name()
+	// cleanup in case anything goes wrong, if rename succeeds then
+	// this will no longer exist
+	defer os.Remove(tempname)
+
+	err := tmp.f.Close()
+	if err != nil {
+		return fmt.Errorf("close tmpfile: %w", err)
+	}
+
+	objPath := filepath.Join(tmp.bucket, tmp.objname)
+	err = os.Rename(tempname, objPath)
+	if err != nil {
+		return fmt.Errorf("rename tmpfile: %w", err)
+	}
+
+	return nil
+}
+
 func (tmp *tmpfile) Write(b []byte) (int, error) {
 	if int64(len(b)) > tmp.size {
 		return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
@@ -178,10 +186,6 @@ func (tmp *tmpfile) cleanup() {
 	tmp.f.Close()
 }
 
-func (tmp *tmpfile) File() *os.File {
-	return tmp.f
-}
-
 func moveData(from *os.File, to *os.File) error {
 	return scoutfs.MoveData(from, to)
 }

backend/scoutfs/scoutfs_incompat.go

@@ -20,26 +20,21 @@ import (
 	"errors"
 	"fmt"
 	"os"
-
-	"github.com/versity/versitygw/auth"
 )
 
-func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
+func New(rootdir string, opts ...Option) (*ScoutFS, error) {
 	return nil, fmt.Errorf("scoutfs only available on linux")
 }
 
-type tmpfile struct{}
+type tmpfile struct {
+	f *os.File
+}
 
 var (
 	errNotSupported = errors.New("not supported")
 )
 
-func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile, error) {
-	// make these look used for static check
-	_ = s.chownuid
-	_ = s.chowngid
-	_ = s.euid
-	_ = s.egid
+func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
 	return nil, errNotSupported
 }
 
@@ -54,14 +49,10 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
 func (tmp *tmpfile) cleanup() {
 }
 
-func (tmp *tmpfile) File() *os.File {
-	return nil
-}
-
-func moveData(_, _ *os.File) error {
+func moveData(from *os.File, to *os.File) error {
 	return errNotSupported
 }
 
-func statMore(_ string) (stat, error) {
+func statMore(path string) (stat, error) {
 	return stat{}, errNotSupported
 }

backend/walk.go

@@ -15,66 +15,46 @@
 package backend
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"io/fs"
+	"os"
 	"sort"
 	"strings"
-	"syscall"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/versity/versitygw/s3response"
 )
 
 type WalkResults struct {
 	CommonPrefixes []types.CommonPrefix
-	Objects        []s3response.Object
+	Objects        []types.Object
 	Truncated      bool
 	NextMarker     string
 }
 
-type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
+type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
 
 var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
-	var objects []s3response.Object
-
-	// if max is 0, it should return empty non-truncated result
-	if max == 0 {
-		return WalkResults{
-			Truncated: false,
-		}, nil
-	}
+	var objects []types.Object
 
 	var pastMarker bool
 	if marker == "" {
 		pastMarker = true
 	}
 
-	var pastMax bool
+	pastMax := max == 0
 	var newMarker string
 	var truncated bool
 
-	root := "."
-	if strings.Contains(prefix, "/") {
-		idx := strings.LastIndex(prefix, "/")
-		if idx > 0 {
-			root = prefix[:idx]
-		}
-	}
-
-	err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
 		// Ignore the root directory
 		if path == "." {
 			return nil
@@ -83,9 +63,14 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			return fs.SkipDir
 		}
 
-		// After this point, return skipflag instead of nil
-		// so we can skip a directory without an early return
-		var skipflag error
+		if pastMax {
+			if len(objects) != 0 {
+				newMarker = *objects[len(objects)-1].Key
+				truncated = true
+			}
+			return fs.SkipAll
+		}
+
 		if d.IsDir() {
 			// If prefix is defined and the directory does not match prefix,
 			// do not descend into the directory because nothing will
@@ -95,66 +80,44 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// building to match. So only skip if path isn't a prefix of prefix
 			// and prefix isn't a prefix of path.
 			if prefix != "" &&
-				!strings.HasPrefix(path+"/", prefix) &&
-				!strings.HasPrefix(prefix, path+"/") {
+				!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
+				!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
 				return fs.SkipDir
 			}
 
-			// Don't recurse into subdirectories which contain the delimiter
-			// after reaching the prefix
-			if delimiter != "" &&
-				strings.HasPrefix(path+"/", prefix) &&
-				strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
-				skipflag = fs.SkipDir
-			} else {
-				if delimiter == "" {
-					dirobj, err := getObj(path+"/", d)
-					if err == ErrSkipObj {
-						return skipflag
-					}
-					if err != nil {
-						return fmt.Errorf("directory to object %q: %w", path, err)
-					}
-					if pastMax {
-						truncated = true
-						return fs.SkipAll
-					}
-					objects = append(objects, dirobj)
-					if (len(objects) + len(cpmap)) == int(max) {
-						newMarker = path
-						pastMax = true
-					}
-
-					return skipflag
-				}
-
-				// TODO: can we do better here rather than a second readdir
-				// per directory?
-				ents, err := fs.ReadDir(fileSystem, path)
-				if err != nil {
-					return fmt.Errorf("readdir %q: %w", path, err)
-				}
-
-				if len(ents) != 0 {
-					return skipflag
-				}
+			// TODO: can we do better here rather than a second readdir
+			// per directory?
+			ents, err := fs.ReadDir(fileSystem, path)
+			if err != nil {
+				return fmt.Errorf("readdir %q: %w", path, err)
 			}
-			path += "/"
+			if len(ents) == 0 {
+				dirobj, err := getObj(path, d)
+				if err == ErrSkipObj {
+					return nil
+				}
+				if err != nil {
+					return fmt.Errorf("directory to object %q: %w", path, err)
+				}
+				objects = append(objects, dirobj)
+			}
+
+			return nil
 		}
 
 		if !pastMarker {
 			if path == marker {
 				pastMarker = true
-				return skipflag
+				return nil
 			}
 			if path < marker {
-				return skipflag
+				return nil
 			}
 		}
 
 		// If object doesn't have prefix, don't include in results.
 		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return skipflag
+			return nil
 		}
 
 		if delimiter == "" {
@@ -162,24 +125,18 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 			// prefix are included in results
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return skipflag
+				return nil
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
 			}
-			if pastMax {
-				truncated = true
-				return fs.SkipAll
-			}
-
 			objects = append(objects, obj)
 
-			if (len(objects) + len(cpmap)) == int(max) {
-				newMarker = path
+			if max > 0 && (len(objects)+len(cpmap)) == int(max) {
 				pastMax = true
 			}
 
-			return skipflag
+			return nil
 		}
 
 		// Since delimiter is specified, we only want results that
@@ -208,55 +165,29 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		if !found {
 			obj, err := getObj(path, d)
 			if err == ErrSkipObj {
-				return skipflag
+				return nil
 			}
 			if err != nil {
 				return fmt.Errorf("file to object %q: %w", path, err)
 			}
-			if pastMax {
-				truncated = true
-				return fs.SkipAll
-			}
 			objects = append(objects, obj)
 			if (len(objects) + len(cpmap)) == int(max) {
-				newMarker = path
 				pastMax = true
 			}
-			return skipflag
+			return nil
 		}
 
 		// Common prefixes are a set, so should not have duplicates.
 		// These are abstractly a "directory", so need to include the
-		// delimiter at the end when we add to the map.
-		cprefNoDelim := prefix + before
-		cpref := prefix + before + delimiter
-		if cpref == marker {
-			pastMarker = true
-			return skipflag
-		}
-
-		if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
-			// skip common prefixes that are before the marker
-			return skipflag
-		}
-
-		if pastMax {
-			truncated = true
-			return fs.SkipAll
-		}
-		cpmap[cpref] = struct{}{}
+		// delimiter at the end.
+		cpmap[prefix+before+delimiter] = struct{}{}
 		if (len(objects) + len(cpmap)) == int(max) {
-			newMarker = cpref
 			pastMax = true
 		}
 
-		return skipflag
+		return nil
 	})
 	if err != nil {
-		// suppress file not found caused by user's prefix
-		if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
-			return WalkResults{}, nil
-		}
 		return WalkResults{}, err
 	}
 
@@ -273,10 +204,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
 		})
 	}
 
-	if !truncated {
-		newMarker = ""
-	}
-
 	return WalkResults{
 		CommonPrefixes: commonPrefixes,
 		Objects:        objects,
@@ -293,213 +220,3 @@ func contains(a string, strs []string) bool {
 	}
 	return false
 }
-
-type WalkVersioningResults struct {
-	CommonPrefixes      []types.CommonPrefix
-	ObjectVersions      []types.ObjectVersion
-	DelMarkers          []types.DeleteMarkerEntry
-	Truncated           bool
-	NextMarker          string
-	NextVersionIdMarker string
-}
-
-type ObjVersionFuncResult struct {
-	ObjectVersions      []types.ObjectVersion
-	DelMarkers          []types.DeleteMarkerEntry
-	NextVersionIdMarker string
-	Truncated           bool
-}
-
-type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
-
-// WalkVersions walks the supplied fs.FS and returns results compatible with
-// ListObjectVersions action response
-func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
-	cpmap := make(map[string]struct{})
-	var objects []types.ObjectVersion
-	var delMarkers []types.DeleteMarkerEntry
-
-	var pastMarker bool
-	if keyMarker == "" {
-		pastMarker = true
-	}
-	var nextMarker string
-	var nextVersionIdMarker string
-	var truncated bool
-
-	pastVersionIdMarker := versionIdMarker == ""
-
-	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
-		// Ignore the root directory
-		if path == "." {
-			return nil
-		}
-		if contains(d.Name(), skipdirs) {
-			return fs.SkipDir
-		}
-
-		if !pastMarker {
-			if path == keyMarker {
-				pastMarker = true
-			}
-			if path < keyMarker {
-				return nil
-			}
-		}
-
-		if d.IsDir() {
-			// If prefix is defined and the directory does not match prefix,
-			// do not descend into the directory because nothing will
-			// match this prefix. Make sure to append the / at the end of
-			// directories since this is implied as a directory path name.
-			// If path is a prefix of prefix, then path could still be
-			// building to match. So only skip if path isn't a prefix of prefix
-			// and prefix isn't a prefix of path.
-			if prefix != "" &&
-				!strings.HasPrefix(path+"/", prefix) &&
-				!strings.HasPrefix(prefix, path+"/") {
-				return fs.SkipDir
-			}
-
-			// Don't recurse into subdirectories when listing with delimiter.
-			if delimiter == "/" &&
-				prefix != path+"/" &&
-				strings.HasPrefix(path+"/", prefix) {
-				cpmap[path+"/"] = struct{}{}
-				return fs.SkipDir
-			}
-
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
-			if err == ErrSkipObj {
-				return nil
-			}
-			if err != nil {
-				return fmt.Errorf("directory to object %q: %w", path, err)
-			}
-			objects = append(objects, res.ObjectVersions...)
-			delMarkers = append(delMarkers, res.DelMarkers...)
-			if res.Truncated {
-				truncated = true
-				nextMarker = path
-				nextVersionIdMarker = res.NextVersionIdMarker
-				return fs.SkipAll
-			}
-
-			return nil
-		}
-
-		// If object doesn't have prefix, don't include in results.
-		if prefix != "" && !strings.HasPrefix(path, prefix) {
-			return nil
-		}
-
-		if delimiter == "" {
-			// If no delimiter specified, then all files with matching
-			// prefix are included in results
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
-			if err == ErrSkipObj {
-				return nil
-			}
-			if err != nil {
-				return fmt.Errorf("file to object %q: %w", path, err)
-			}
-			objects = append(objects, res.ObjectVersions...)
-			delMarkers = append(delMarkers, res.DelMarkers...)
-			if res.Truncated {
-				truncated = true
-				nextMarker = path
-				nextVersionIdMarker = res.NextVersionIdMarker
-				return fs.SkipAll
-			}
-
-			return nil
-		}
-
-		// Since delimiter is specified, we only want results that
-		// do not contain the delimiter beyond the prefix.  If the
-		// delimiter exists past the prefix, then the substring
-		// between the prefix and delimiter is part of common prefixes.
-		//
-		// For example:
-		// prefix = A/
-		// delimiter = /
-		// and objects:
-		// A/file
-		// A/B/file
-		// B/C
-		// would return:
-		// objects: A/file
-		// common prefix: A/B/
-		//
-		// Note: No objects are included past the common prefix since
-		// these are all rolled up into the common prefix.
-		// Note: The delimiter can be anything, so we have to operate on
-		// the full path without any assumptions on posix directory hierarchy
-		// here.  Usually the delimiter will be "/", but thats not required.
-		suffix := strings.TrimPrefix(path, prefix)
-		before, _, found := strings.Cut(suffix, delimiter)
-		if !found {
-			res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
-			if err == ErrSkipObj {
-				return nil
-			}
-			if err != nil {
-				return fmt.Errorf("file to object %q: %w", path, err)
-			}
-			objects = append(objects, res.ObjectVersions...)
-			delMarkers = append(delMarkers, res.DelMarkers...)
-
-			if res.Truncated {
-				truncated = true
-				nextMarker = path
-				nextVersionIdMarker = res.NextVersionIdMarker
-				return fs.SkipAll
-			}
-			return nil
-		}
-
-		// Common prefixes are a set, so should not have duplicates.
-		// These are abstractly a "directory", so need to include the
-		// delimiter at the end.
-		cpmap[prefix+before+delimiter] = struct{}{}
-		if (len(objects) + len(cpmap)) == int(max) {
-			nextMarker = path
-			truncated = true
-
-			return fs.SkipAll
-		}
-
-		return nil
-	})
-	if err != nil {
-		return WalkVersioningResults{}, err
-	}
-
-	var commonPrefixStrings []string
-	for k := range cpmap {
-		commonPrefixStrings = append(commonPrefixStrings, k)
-	}
-	sort.Strings(commonPrefixStrings)
-	commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
-	for _, cp := range commonPrefixStrings {
-		pfx := cp
-		commonPrefixes = append(commonPrefixes, types.CommonPrefix{
-			Prefix: &pfx,
-		})
-	}
-
-	return WalkVersioningResults{
-		CommonPrefixes:      commonPrefixes,
-		ObjectVersions:      objects,
-		DelMarkers:          delMarkers,
-		Truncated:           truncated,
-		NextMarker:          nextMarker,
-		NextVersionIdMarker: nextVersionIdMarker,
-	}, nil
-}

backend/walk_test.go

@@ -15,50 +15,36 @@
 package backend_test
 
 import (
-	"context"
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
 	"io/fs"
-	"sync"
 	"testing"
 	"testing/fstest"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3response"
 )
 
 type walkTest struct {
-	fsys   fs.FS
-	getobj backend.GetObjFunc
-	cases  []testcase
+	fsys     fs.FS
+	expected backend.WalkResults
+	getobj   backend.GetObjFunc
 }
 
-type testcase struct {
-	name      string
-	prefix    string
-	delimiter string
-	marker    string
-	maxObjs   int32
-	expected  backend.WalkResults
-}
-
-func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
+func getObj(path string, d fs.DirEntry) (types.Object, error) {
 	if d.IsDir() {
 		etag := getMD5(path)
 
 		fi, err := d.Info()
 		if err != nil {
-			return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
-		mtime := fi.ModTime()
 
-		return s3response.Object{
+		return types.Object{
 			ETag:         &etag,
 			Key:          &path,
-			LastModified: &mtime,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
 		}, nil
 	}
 
@@ -66,16 +52,15 @@ func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
 
 	fi, err := d.Info()
 	if err != nil {
-		return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 	}
 
 	size := fi.Size()
-	mtime := fi.ModTime()
 
-	return s3response.Object{
+	return types.Object{
 		ETag:         &etag,
 		Key:          &path,
-		LastModified: &mtime,
+		LastModified: backend.GetTimePtr(fi.ModTime()),
 		Size:         &size,
 	}, nil
 }
@@ -97,154 +82,50 @@ func TestWalk(t *testing.T) {
 				"photos/2006/February/sample3.jpg": {},
 				"photos/2006/February/sample4.jpg": {},
 			},
-			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "aws example",
-					delimiter: "/",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photos/"),
-						}},
-						Objects: []s3response.Object{{
-							Key: backend.GetPtrFromString("sample.jpg"),
-						}},
-					},
-				},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("photos/"),
+				}},
+				Objects: []types.Object{{
+					Key: backend.GetStringPtr("sample.jpg"),
+				}},
 			},
+			getobj: getObj,
 		},
 		{
 			// test case single dir/single file
 			fsys: fstest.MapFS{
 				"test/file": {},
 			},
-			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "single dir single file",
-					delimiter: "/",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("test/"),
-						}},
-						Objects: []s3response.Object{},
-					},
-				},
-			},
-		},
-		{
-			// non-standard delimiter
-			fsys: fstest.MapFS{
-				"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
-				"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
-				"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
+			expected: backend.WalkResults{
+				CommonPrefixes: []types.CommonPrefix{{
+					Prefix: backend.GetStringPtr("test/"),
+				}},
+				Objects: []types.Object{},
 			},
 			getobj: getObj,
-			cases: []testcase{
-				{
-					name:      "different delimiter 1",
-					delimiter: "|",
-					maxObjs:   1000,
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 2",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 3",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 4",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 5",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|6/Januar|",
-					expected: backend.WalkResults{
-						CommonPrefixes: []types.CommonPrefix{{
-							Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
-						}},
-					},
-				},
-				{
-					name:      "different delimiter 6",
-					delimiter: "|",
-					maxObjs:   1000,
-					prefix:    "photo|s/200|6/Januar|y/sampl|",
-					expected: backend.WalkResults{
-						Objects: []s3response.Object{
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
-							},
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
-							},
-							{
-								Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
-							},
-						},
-					},
-				},
-			},
 		},
 	}
 
 	for _, tt := range tests {
-		for _, tc := range tt.cases {
-			res, err := backend.Walk(context.Background(),
-				tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
-				tt.getobj, []string{})
-			if err != nil {
-				t.Errorf("tc.name: walk: %v", err)
-			}
-
-			compareResults(tc.name, res, tc.expected, t)
+		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
+		if err != nil {
+			t.Fatalf("walk: %v", err)
 		}
+
+		compareResults(res, tt.expected, t)
 	}
 }
 
-func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
+func compareResults(got, wanted backend.WalkResults, t *testing.T) {
 	if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
-		t.Errorf("%v: unexpected common prefix, got %v wanted %v",
-			name,
+		t.Errorf("unexpected common prefix, got %v wanted %v",
 			printCommonPrefixes(got.CommonPrefixes),
 			printCommonPrefixes(wanted.CommonPrefixes))
 	}
 
 	if !compareObjects(got.Objects, wanted.Objects) {
-		t.Errorf("%v: unexpected object, got %v wanted %v",
-			name,
+		t.Errorf("unexpected object, got %v wanted %v",
 			printObjects(got.Objects),
 			printObjects(wanted.Objects))
 	}
@@ -287,7 +168,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
 	return res + "]"
 }
 
-func compareObjects(a, b []s3response.Object) bool {
+func compareObjects(a, b []types.Object) bool {
 	if len(a) == 0 && len(b) == 0 {
 		return true
 	}
@@ -303,7 +184,7 @@ func compareObjects(a, b []s3response.Object) bool {
 	return false
 }
 
-func containsObject(c s3response.Object, list []s3response.Object) bool {
+func containsObject(c types.Object, list []types.Object) bool {
 	for _, cp := range list {
 		if *c.Key == *cp.Key {
 			return true
@@ -312,67 +193,14 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
 	return false
 }
 
-func printObjects(list []s3response.Object) string {
+func printObjects(list []types.Object) string {
 	res := "["
 	for _, cp := range list {
-		var key string
-		if cp.Key == nil {
-			key = "<nil>"
-		} else {
-			key = *cp.Key
-		}
 		if res == "[" {
-			res = res + key
+			res = res + *cp.Key
 		} else {
-			res = res + ", " + key
+			res = res + ", " + *cp.Key
 		}
 	}
 	return res + "]"
 }
-
-type slowFS struct {
-	fstest.MapFS
-}
-
-const (
-	readDirPause = 100 * time.Millisecond
-
-	// walkTimeOut should be less than the tree traversal time
-	// which is the readdirPause time * the number of directories
-	walkTimeOut = 500 * time.Millisecond
-)
-
-func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
-	time.Sleep(readDirPause)
-	return s.MapFS.ReadDir(name)
-}
-
-func TestWalkStop(t *testing.T) {
-	s := &slowFS{MapFS: fstest.MapFS{
-		"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
-	}}
-
-	ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
-	defer cancel()
-
-	var err error
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		_, err = backend.Walk(ctx, s, "", "/", "", 1000,
-			func(path string, d fs.DirEntry) (s3response.Object, error) {
-				return s3response.Object{}, nil
-			}, []string{})
-	}()
-
-	select {
-	case <-time.After(1 * time.Second):
-		t.Fatalf("walk is not terminated in time")
-	case <-ctx.Done():
-	}
-	wg.Wait()
-	if err != ctx.Err() {
-		t.Fatalf("unexpected error: %v", err)
-	}
-}

cmd/versitygw/admin.go

@@ -17,9 +17,8 @@ package main
 import (
 	"bytes"
 	"crypto/sha256"
-	"crypto/tls"
 	"encoding/hex"
-	"encoding/xml"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,7 +28,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/aws/smithy-go"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3response"
@@ -38,9 +36,7 @@ import (
 var (
 	adminAccess   string
 	adminSecret   string
-	adminRegion   string
 	adminEndpoint string
-	allowInsecure bool
 )
 
 func adminCommand() *cli.Command {
@@ -82,33 +78,10 @@ func adminCommand() *cli.Command {
 						Usage:   "groupID for the new user",
 						Aliases: []string{"gi"},
 					},
-				},
-			},
-			{
-				Name:   "update-user",
-				Usage:  "Updates a user account",
-				Action: updateUser,
-				Flags: []cli.Flag{
-					&cli.StringFlag{
-						Name:     "access",
-						Usage:    "user access key id to be updated",
-						Required: true,
-						Aliases:  []string{"a"},
-					},
-					&cli.StringFlag{
-						Name:    "secret",
-						Usage:   "secret access key for the new user",
-						Aliases: []string{"s"},
-					},
 					&cli.IntFlag{
-						Name:    "user-id",
-						Usage:   "userID for the new user",
-						Aliases: []string{"ui"},
-					},
-					&cli.IntFlag{
-						Name:    "group-id",
-						Usage:   "groupID for the new user",
-						Aliases: []string{"gi"},
+						Name:    "project-id",
+						Usage:   "projectID for the new user",
+						Aliases: []string{"pi"},
 					},
 				},
 			},
@@ -173,14 +146,6 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminSecret,
 			},
-			&cli.StringFlag{
-				Name:        "region",
-				Usage:       "admin s3 region string",
-				EnvVars:     []string{"ADMIN_REGION"},
-				Value:       "us-east-1",
-				Destination: &adminRegion,
-				Aliases:     []string{"r"},
-			},
 			&cli.StringFlag{
 				Name:        "endpoint-url",
 				Usage:       "admin apis endpoint url",
@@ -189,80 +154,65 @@ func adminCommand() *cli.Command {
 				Required:    true,
 				Destination: &adminEndpoint,
 			},
-			&cli.BoolFlag{
-				Name:        "allow-insecure",
-				Usage:       "disable tls certificate verification for the admin endpoint",
-				EnvVars:     []string{"ADMIN_ALLOW_INSECURE"},
-				Aliases:     []string{"ai"},
-				Destination: &allowInsecure,
-			},
 		},
 	}
 }
 
-func initHTTPClient() *http.Client {
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
-	}
-	return &http.Client{Transport: tr}
-}
-
 func createUser(ctx *cli.Context) error {
 	access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
-	userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
+	userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
 	if access == "" || secret == "" {
-		return fmt.Errorf("invalid input parameters for the new user access/secret keys")
+		return fmt.Errorf("invalid input parameters for the new user")
 	}
 	if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
 		return fmt.Errorf("invalid input parameter for role: %v", role)
 	}
 
 	acc := auth.Account{
-		Access:  access,
-		Secret:  secret,
-		Role:    auth.Role(role),
-		UserID:  userID,
-		GroupID: groupID,
+		Access:    access,
+		Secret:    secret,
+		Role:      auth.Role(role),
+		UserID:    userID,
+		GroupID:   groupID,
+		ProjectID: projectID,
 	}
 
-	accxml, err := xml.Marshal(acc)
+	accJson, err := json.Marshal(acc)
 	if err != nil {
 		return fmt.Errorf("failed to parse user data: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
 
 	signer := v4.NewSigner()
 
-	hashedPayload := sha256.Sum256(accxml)
+	hashedPayload := sha256.Sum256(accJson)
 	hexPayload := hex.EncodeToString(hashedPayload[:])
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
+	fmt.Printf("%s\n", body)
 
 	return nil
 }
@@ -270,7 +220,7 @@ func createUser(ctx *cli.Context) error {
 func deleteUser(ctx *cli.Context) error {
 	access := ctx.String("access")
 	if access == "" {
-		return fmt.Errorf("invalid input parameter for the user access key")
+		return fmt.Errorf("invalid input parameter for the new user")
 	}
 
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
@@ -285,82 +235,25 @@ func deleteUser(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
-
-	return nil
-}
-
-func updateUser(ctx *cli.Context) error {
-	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
-	props := auth.MutableProps{}
-	if ctx.IsSet("secret") {
-		props.Secret = &secret
-	}
-	if ctx.IsSet("user-id") {
-		props.UserID = &userId
-	}
-	if ctx.IsSet("group-id") {
-		props.GroupID = &groupId
-	}
-
-	propsxml, err := xml.Marshal(props)
-	if err != nil {
-		return fmt.Errorf("failed to parse user attributes: %w", err)
-	}
-
-	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
-	if err != nil {
-		return fmt.Errorf("failed to send the request: %w", err)
-	}
-
-	signer := v4.NewSigner()
-
-	hashedPayload := sha256.Sum256(propsxml)
-	hexPayload := hex.EncodeToString(hashedPayload[:])
-
-	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
-
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
-	if signErr != nil {
-		return fmt.Errorf("failed to sign the request: %w", err)
-	}
-
-	client := initHTTPClient()
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to send the request: %w", err)
-	}
 	defer resp.Body.Close()
 
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
+	fmt.Printf("%s\n", body)
 
 	return nil
 }
@@ -378,34 +271,34 @@ func listUsers(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
-	var accs auth.ListUserAccountsResult
-	if err := xml.Unmarshal(body, &accs); err != nil {
+	var accs []auth.Account
+	if err := json.Unmarshal(body, &accs); err != nil {
 		return err
 	}
 
-	printAcctTable(accs.Accounts)
+	printAcctTable(accs)
 
 	return nil
 }
@@ -422,10 +315,10 @@ const (
 func printAcctTable(accs []auth.Account) {
 	w := new(tabwriter.Writer)
 	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
-	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
-	fmt.Fprintln(w, "-------\t----\t------\t-------")
+	fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
+	fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
 	for _, acc := range accs {
-		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
 	}
 	fmt.Fprintln(w)
 	w.Flush()
@@ -445,27 +338,25 @@ func changeBucketOwner(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
-	if resp.StatusCode >= 400 {
-		return parseApiError(body)
-	}
+	fmt.Println(string(body))
 
 	return nil
 }
@@ -495,45 +386,34 @@ func listBuckets(ctx *cli.Context) error {
 
 	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
 
-	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
 	if signErr != nil {
 		return fmt.Errorf("failed to sign the request: %w", err)
 	}
 
-	client := initHTTPClient()
+	client := http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to send the request: %w", err)
 	}
-	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
-		return parseApiError(body)
+		return fmt.Errorf("%s", body)
 	}
 
-	var result s3response.ListBucketsResult
-	if err := xml.Unmarshal(body, &result); err != nil {
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
 		return err
 	}
 
-	printBuckets(result.Buckets)
+	printBuckets(buckets)
 
 	return nil
 }
-
-func parseApiError(body []byte) error {
-	var apiErr smithy.GenericAPIError
-	err := xml.Unmarshal(body, &apiErr)
-	if err != nil {
-		apiErr.Code = "InternalServerError"
-		apiErr.Message = err.Error()
-	}
-
-	return &apiErr
-}

cmd/versitygw/gateway_test.go

@@ -7,11 +7,9 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
-	"time"
 
-	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
-	"github.com/versity/versitygw/tests/integration"
+	"github.com/versity/versitygw/integration"
 )
 
 const (
@@ -58,9 +56,7 @@ func initPosix(ctx context.Context) {
 		log.Fatalf("make temp directory: %v", err)
 	}
 
-	be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
-		NewDirPerm: 0755,
-	})
+	be, err := posix.New(tempdir)
 	if err != nil {
 		log.Fatalf("init posix: %v", err)
 	}
@@ -78,9 +74,6 @@ func initPosix(ctx context.Context) {
 		}
 		wg.Done()
 	}()
-
-	// wait for server to start
-	time.Sleep(1 * time.Second)
 }
 
 func TestIntegration(t *testing.T) {

cmd/versitygw/main.go

@@ -19,17 +19,12 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
-	"net"
-	"net/http"
-	_ "net/http/pprof"
 	"os"
-	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api"
 	"github.com/versity/versitygw/s3api/middlewares"
 	"github.com/versity/versitygw/s3event"
@@ -37,46 +32,30 @@ import (
 )
 
 var (
-	port, admPort                            string
-	rootUserAccess                           string
-	rootUserSecret                           string
-	region                                   string
-	admCertFile, admKeyFile                  string
-	certFile, keyFile                        string
-	kafkaURL, kafkaTopic, kafkaKey           string
-	natsURL, natsTopic                       string
-	eventWebhookURL                          string
-	eventConfigFilePath                      string
-	logWebhookURL, accessLog                 string
-	adminLogFile                             string
-	healthPath                               string
-	debug                                    bool
-	pprof                                    string
-	quiet                                    bool
-	readonly                                 bool
-	iamDir                                   string
-	ldapURL, ldapBindDN, ldapPassword        string
-	ldapQueryBase, ldapObjClasses            string
-	ldapAccessAtr, ldapSecAtr, ldapRoleAtr   string
-	ldapUserIdAtr, ldapGroupIdAtr            string
-	vaultEndpointURL, vaultSecretStoragePath string
-	vaultMountPath, vaultRootToken           string
-	vaultRoleId, vaultRoleSecret             string
-	vaultServerCert, vaultClientCert         string
-	vaultClientCertKey                       string
-	s3IamAccess, s3IamSecret                 string
-	s3IamRegion, s3IamBucket                 string
-	s3IamEndpoint                            string
-	s3IamSslNoVerify, s3IamDebug             bool
-	iamCacheDisable                          bool
-	iamCacheTTL                              int
-	iamCachePrune                            int
-	metricsService                           string
-	statsdServers                            string
-	dogstatsServers                          string
-	ipaHost, ipaVaultName                    string
-	ipaUser, ipaPassword                     string
-	ipaInsecure, ipaDebug                    bool
+	port, admPort                          string
+	rootUserAccess                         string
+	rootUserSecret                         string
+	region                                 string
+	admCertFile, admKeyFile                string
+	certFile, keyFile                      string
+	kafkaURL, kafkaTopic, kafkaKey         string
+	natsURL, natsTopic                     string
+	logWebhookURL                          string
+	accessLog                              string
+	healthPath                             string
+	debug                                  bool
+	quiet                                  bool
+	iamDir                                 string
+	ldapURL, ldapBindDN, ldapPassword      string
+	ldapQueryBase, ldapObjClasses          string
+	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	s3IamAccess, s3IamSecret               string
+	s3IamRegion, s3IamBucket               string
+	s3IamEndpoint                          string
+	s3IamSslNoVerify, s3IamDebug           bool
+	iamCacheDisable                        bool
+	iamCacheTTL                            int
+	iamCachePrune                          int
 )
 
 var (
@@ -98,10 +77,8 @@ func main() {
 		scoutfsCommand(),
 		s3Command(),
 		azureCommand(),
-		pluginCommand(),
 		adminCommand(),
 		testCommand(),
-		utilsCommand(),
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -118,13 +95,10 @@ func main() {
 
 func initApp() *cli.App {
 	return &cli.App{
-		Usage: "Versity S3 Gateway",
-		Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
-to access the supported backend storage as if it was a native S3 service.
-VersityGW is an open-source project licensed under the Apache 2.0 License. The
-source code is hosted on GitHub at https://github.com/versity/versitygw, and
-documentation can be found in the GitHub wiki.`,
-		Copyright: "Copyright (c) 2023-2024 Versity Software",
+		Name:  "versitygw",
+		Usage: "Start S3 gateway service with specified backend storage.",
+		Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
+to access the supported backend storage as if it was a native S3 service.`,
 		Action: func(ctx *cli.Context) error {
 			return ctx.App.Command("help").Run(ctx)
 		},
@@ -210,16 +184,9 @@ func initFlags() []cli.Flag {
 		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "enable debug output",
-			Value:       false,
 			EnvVars:     []string{"VGW_DEBUG"},
 			Destination: &debug,
 		},
-		&cli.StringFlag{
-			Name:        "pprof",
-			Usage:       "enable pprof debug on specified port",
-			EnvVars:     []string{"VGW_PPROF"},
-			Destination: &pprof,
-		},
 		&cli.BoolFlag{
 			Name:        "quiet",
 			Usage:       "silence stdout request logging output",
@@ -230,19 +197,13 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
-			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
+			EnvVars:     []string{"LOGFILE"},
 			Destination: &accessLog,
 		},
-		&cli.StringFlag{
-			Name:        "admin-access-log",
-			Usage:       "enable admin server access logging to specified file",
-			EnvVars:     []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
-			Destination: &adminLogFile,
-		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
-			EnvVars:     []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"},
+			EnvVars:     []string{"WEBHOOK"},
 			Destination: &logWebhookURL,
 		},
 		&cli.StringFlag{
@@ -280,20 +241,6 @@ func initFlags() []cli.Flag {
 			Destination: &natsTopic,
 			Aliases:     []string{"ent"},
 		},
-		&cli.StringFlag{
-			Name:        "event-webhook-url",
-			Usage:       "webhook url to send bucket notifications",
-			EnvVars:     []string{"VGW_EVENT_WEBHOOK_URL"},
-			Destination: &eventWebhookURL,
-			Aliases:     []string{"ewu"},
-		},
-		&cli.StringFlag{
-			Name:        "event-filter",
-			Usage:       "bucket event notifications filters configuration file path",
-			EnvVars:     []string{"VGW_EVENT_FILTER"},
-			Destination: &eventConfigFilePath,
-			Aliases:     []string{"ef"},
-		},
 		&cli.StringFlag{
 			Name:        "iam-dir",
 			Usage:       "if defined, run internal iam service within this directory",
@@ -348,72 +295,6 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
-		&cli.StringFlag{
-			Name:        "iam-ldap-user-id-atr",
-			Usage:       "ldap server user id attribute name",
-			EnvVars:     []string{"VGW_IAM_LDAP_USER_ID_ATR"},
-			Destination: &ldapUserIdAtr,
-		},
-		&cli.StringFlag{
-			Name:        "iam-ldap-group-id-atr",
-			Usage:       "ldap server user group id attribute name",
-			EnvVars:     []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
-			Destination: &ldapGroupIdAtr,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-endpoint-url",
-			Usage:       "vault server url",
-			EnvVars:     []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
-			Destination: &vaultEndpointURL,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-secret-storage-path",
-			Usage:       "vault server secret storage path",
-			EnvVars:     []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
-			Destination: &vaultSecretStoragePath,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-mount-path",
-			Usage:       "vault server mount path",
-			EnvVars:     []string{"VGW_IAM_VAULT_MOUNT_PATH"},
-			Destination: &vaultMountPath,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-root-token",
-			Usage:       "vault server root token",
-			EnvVars:     []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
-			Destination: &vaultRootToken,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-role-id",
-			Usage:       "vault server user role id",
-			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_ID"},
-			Destination: &vaultRoleId,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-role-secret",
-			Usage:       "vault server user role secret",
-			EnvVars:     []string{"VGW_IAM_VAULT_ROLE_SECRET"},
-			Destination: &vaultRoleSecret,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-server_cert",
-			Usage:       "vault server TLS certificate",
-			EnvVars:     []string{"VGW_IAM_VAULT_SERVER_CERT"},
-			Destination: &vaultServerCert,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-client_cert",
-			Usage:       "vault client TLS certificate",
-			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT"},
-			Destination: &vaultClientCert,
-		},
-		&cli.StringFlag{
-			Name:        "iam-vault-client_cert_key",
-			Usage:       "vault client TLS certificate key",
-			EnvVars:     []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
-			Destination: &vaultClientCertKey,
-		},
 		&cli.StringFlag{
 			Name:        "s3-iam-access",
 			Usage:       "s3 IAM access key",
@@ -484,92 +365,15 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_HEALTH"},
 			Destination: &healthPath,
 		},
-		&cli.BoolFlag{
-			Name:        "readonly",
-			Usage:       "allow only read operations across all the gateway",
-			EnvVars:     []string{"VGW_READ_ONLY"},
-			Destination: &readonly,
-		},
-		&cli.StringFlag{
-			Name:        "metrics-service-name",
-			Usage:       "service name tag for metrics, hostname if blank",
-			EnvVars:     []string{"VGW_METRICS_SERVICE_NAME"},
-			Aliases:     []string{"msn"},
-			Destination: &metricsService,
-		},
-		&cli.StringFlag{
-			Name:        "metrics-statsd-servers",
-			Usage:       "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
-			EnvVars:     []string{"VGW_METRICS_STATSD_SERVERS"},
-			Aliases:     []string{"mss"},
-			Destination: &statsdServers,
-		},
-		&cli.StringFlag{
-			Name:        "metrics-dogstatsd-servers",
-			Usage:       "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
-			EnvVars:     []string{"VGW_METRICS_DOGSTATS_SERVERS"},
-			Aliases:     []string{"mds"},
-			Destination: &dogstatsServers,
-		},
-		&cli.StringFlag{
-			Name:        "ipa-host",
-			Usage:       "FreeIPA server url e.g. https://ipa.example.test",
-			EnvVars:     []string{"VGW_IPA_HOST"},
-			Destination: &ipaHost,
-		},
-		&cli.StringFlag{
-			Name:        "ipa-vault-name",
-			Usage:       "A name of the user vault containing their secret",
-			EnvVars:     []string{"VGW_IPA_VAULT_NAME"},
-			Destination: &ipaVaultName,
-		},
-		&cli.StringFlag{
-			Name:        "ipa-user",
-			Usage:       "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
-			EnvVars:     []string{"VGW_IPA_USER"},
-			Destination: &ipaUser,
-		},
-		&cli.StringFlag{
-			Name:        "ipa-password",
-			Usage:       "Password of the user used to connect to FreeIPA.",
-			EnvVars:     []string{"VGW_IPA_PASSWORD"},
-			Destination: &ipaPassword,
-		},
-		&cli.BoolFlag{
-			Name:        "ipa-insecure",
-			Usage:       "Verify TLS certificate of FreeIPA server. Default is 'true'.",
-			EnvVars:     []string{"VGW_IPA_INSECURE"},
-			Destination: &ipaInsecure,
-		},
-		&cli.BoolFlag{
-			Name:        "ipa-debug",
-			Usage:       "FreeIPA IAM debug output",
-			EnvVars:     []string{"VGW_IPA_DEBUG"},
-			Destination: &ipaDebug,
-		},
 	}
 }
 
 func runGateway(ctx context.Context, be backend.Backend) error {
-	if rootUserAccess == "" || rootUserSecret == "" {
-		return fmt.Errorf("root user access and secret key must be provided")
-	}
-
-	if pprof != "" {
-		// listen on specified port for pprof debug
-		// point browser to http://<ip:port>/debug/pprof/
-		go func() {
-			log.Fatal(http.ListenAndServe(pprof, nil))
-		}()
-	}
-
 	app := fiber.New(fiber.Config{
-		AppName:               "versitygw",
-		ServerHeader:          "VERSITYGW",
-		StreamRequestBody:     true,
-		DisableKeepalive:      true,
-		Network:               fiber.NetworkTCP,
-		DisableStartupMessage: true,
+		AppName:           "versitygw",
+		ServerHeader:      "VERSITYGW",
+		StreamRequestBody: true,
+		DisableKeepalive:  true,
 	})
 
 	var opts []s3api.Option
@@ -600,15 +404,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if healthPath != "" {
 		opts = append(opts, s3api.WithHealth(healthPath))
 	}
-	if readonly {
-		opts = append(opts, s3api.WithReadOnly())
-	}
 
 	admApp := fiber.New(fiber.Config{
-		AppName:               "versitygw",
-		ServerHeader:          "VERSITYGW",
-		Network:               fiber.NetworkTCP,
-		DisableStartupMessage: true,
+		AppName:      "versitygw",
+		ServerHeader: "VERSITYGW",
 	})
 
 	var admOpts []s3api.AdminOpt
@@ -629,96 +428,58 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
-		RootAccount: auth.Account{
-			Access: rootUserAccess,
-			Secret: rootUserSecret,
-			Role:   auth.RoleAdmin,
-		},
-		Dir:                    iamDir,
-		LDAPServerURL:          ldapURL,
-		LDAPBindDN:             ldapBindDN,
-		LDAPPassword:           ldapPassword,
-		LDAPQueryBase:          ldapQueryBase,
-		LDAPObjClasses:         ldapObjClasses,
-		LDAPAccessAtr:          ldapAccessAtr,
-		LDAPSecretAtr:          ldapSecAtr,
-		LDAPRoleAtr:            ldapRoleAtr,
-		LDAPUserIdAtr:          ldapUserIdAtr,
-		LDAPGroupIdAtr:         ldapGroupIdAtr,
-		VaultEndpointURL:       vaultEndpointURL,
-		VaultSecretStoragePath: vaultSecretStoragePath,
-		VaultMountPath:         vaultMountPath,
-		VaultRootToken:         vaultRootToken,
-		VaultRoleId:            vaultRoleId,
-		VaultRoleSecret:        vaultRoleSecret,
-		VaultServerCert:        vaultServerCert,
-		VaultClientCert:        vaultClientCert,
-		VaultClientCertKey:     vaultClientCertKey,
-		S3Access:               s3IamAccess,
-		S3Secret:               s3IamSecret,
-		S3Region:               s3IamRegion,
-		S3Bucket:               s3IamBucket,
-		S3Endpoint:             s3IamEndpoint,
-		S3DisableSSlVerfiy:     s3IamSslNoVerify,
-		S3Debug:                s3IamDebug,
-		CacheDisable:           iamCacheDisable,
-		CacheTTL:               iamCacheTTL,
-		CachePrune:             iamCachePrune,
-		IpaHost:                ipaHost,
-		IpaVaultName:           ipaVaultName,
-		IpaUser:                ipaUser,
-		IpaPassword:            ipaPassword,
-		IpaInsecure:            ipaInsecure,
-		IpaDebug:               ipaDebug,
+		Dir:                iamDir,
+		LDAPServerURL:      ldapURL,
+		LDAPBindDN:         ldapBindDN,
+		LDAPPassword:       ldapPassword,
+		LDAPQueryBase:      ldapQueryBase,
+		LDAPObjClasses:     ldapObjClasses,
+		LDAPAccessAtr:      ldapAccessAtr,
+		LDAPSecretAtr:      ldapSecAtr,
+		LDAPRoleAtr:        ldapRoleAtr,
+		S3Access:           s3IamAccess,
+		S3Secret:           s3IamSecret,
+		S3Region:           s3IamRegion,
+		S3Bucket:           s3IamBucket,
+		S3Endpoint:         s3IamEndpoint,
+		S3DisableSSlVerfiy: s3IamSslNoVerify,
+		S3Debug:            s3IamDebug,
+		CacheDisable:       iamCacheDisable,
+		CacheTTL:           iamCacheTTL,
+		CachePrune:         iamCachePrune,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)
 	}
 
-	loggers, err := s3log.InitLogger(&s3log.LogConfig{
-		LogFile:      accessLog,
-		WebhookURL:   logWebhookURL,
-		AdminLogFile: adminLogFile,
+	logger, err := s3log.InitLogger(&s3log.LogConfig{
+		LogFile:    accessLog,
+		WebhookURL: logWebhookURL,
 	})
 	if err != nil {
 		return fmt.Errorf("setup logger: %w", err)
 	}
 
-	metricsManager, err := metrics.NewManager(ctx, metrics.Config{
-		ServiceName:      metricsService,
-		StatsdServers:    statsdServers,
-		DogStatsdServers: dogstatsServers,
-	})
-	if err != nil {
-		return fmt.Errorf("init metrics manager: %w", err)
-	}
-
 	evSender, err := s3event.InitEventSender(&s3event.EventConfig{
-		KafkaURL:             kafkaURL,
-		KafkaTopic:           kafkaTopic,
-		KafkaTopicKey:        kafkaKey,
-		NatsURL:              natsURL,
-		NatsTopic:            natsTopic,
-		WebhookURL:           eventWebhookURL,
-		FilterConfigFilePath: eventConfigFilePath,
+		KafkaURL:      kafkaURL,
+		KafkaTopic:    kafkaTopic,
+		KafkaTopicKey: kafkaKey,
+		NatsURL:       natsURL,
+		NatsTopic:     natsTopic,
 	})
 	if err != nil {
-		return fmt.Errorf("init bucket event notifications: %w", err)
+		return fmt.Errorf("unable to connect to the message broker: %w", err)
 	}
 
 	srv, err := s3api.New(app, be, middlewares.RootUserConfig{
 		Access: rootUserAccess,
 		Secret: rootUserSecret,
-	}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
+	}, port, region, iam, logger, evSender, opts...)
 	if err != nil {
 		return fmt.Errorf("init gateway: %v", err)
 	}
 
-	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
-
-	if !quiet {
-		printBanner(port, admPort, certFile != "", admCertFile != "")
-	}
+	admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
 
 	c := make(chan error, 2)
 	go func() { c <- srv.Serve() }()
@@ -731,21 +492,15 @@ Loop:
 	for {
 		select {
 		case <-ctx.Done():
+			err = ctx.Err()
 			break Loop
 		case err = <-c:
 			break Loop
 		case <-sigHup:
-			if loggers.S3Logger != nil {
-				err = loggers.S3Logger.HangUp()
+			if logger != nil {
+				err = logger.HangUp()
 				if err != nil {
-					err = fmt.Errorf("HUP s3 logger: %w", err)
-					break Loop
-				}
-			}
-			if loggers.AdminLogger != nil {
-				err = loggers.AdminLogger.HangUp()
-				if err != nil {
-					err = fmt.Errorf("HUP admin logger: %w", err)
+					err = fmt.Errorf("HUP logger: %w", err)
 					break Loop
 				}
 			}
@@ -757,218 +512,15 @@ Loop:
 
 	err = iam.Shutdown()
 	if err != nil {
-		if saveErr == nil {
-			saveErr = err
-		}
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
-	if loggers.S3Logger != nil {
-		err := loggers.S3Logger.Shutdown()
+	if logger != nil {
+		err := logger.Shutdown()
 		if err != nil {
-			if saveErr == nil {
-				saveErr = err
-			}
-			fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
+			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
 		}
 	}
-	if loggers.AdminLogger != nil {
-		err := loggers.AdminLogger.Shutdown()
-		if err != nil {
-			if saveErr == nil {
-				saveErr = err
-			}
-			fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
-		}
-	}
-
-	if evSender != nil {
-		err := evSender.Close()
-		if err != nil {
-			if saveErr == nil {
-				saveErr = err
-			}
-			fmt.Fprintf(os.Stderr, "close event sender: %v\n", err)
-		}
-	}
-
-	if metricsManager != nil {
-		metricsManager.Close()
-	}
 
 	return saveErr
 }
-
-func printBanner(port, admPort string, ssl, admSsl bool) {
-	interfaces, err := getMatchingIPs(port)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
-		return
-	}
-
-	var admInterfaces []string
-	if admPort != "" {
-		admInterfaces, err = getMatchingIPs(admPort)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
-			return
-		}
-	}
-
-	title := "VersityGW"
-	version := fmt.Sprintf("Version %v, Build %v", Version, Build)
-	urls := []string{}
-
-	hst, prt, err := net.SplitHostPort(port)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
-		return
-	}
-
-	for _, ip := range interfaces {
-		url := fmt.Sprintf("http://%s:%s", ip, prt)
-		if ssl {
-			url = fmt.Sprintf("https://%s:%s", ip, prt)
-		}
-		urls = append(urls, url)
-	}
-
-	if hst == "" {
-		hst = "0.0.0.0"
-	}
-
-	boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
-
-	lines := []string{
-		centerText(title),
-		centerText(version),
-		centerText(boundHost),
-		centerText(""),
-	}
-
-	if len(admInterfaces) > 0 {
-		lines = append(lines,
-			leftText("S3 service listening on:"),
-		)
-	} else {
-		lines = append(lines,
-			leftText("Admin/S3 service listening on:"),
-		)
-	}
-
-	for _, url := range urls {
-		lines = append(lines, leftText("  "+url))
-	}
-
-	if len(admInterfaces) > 0 {
-		lines = append(lines,
-			centerText(""),
-			leftText("Admin service listening on:"),
-		)
-
-		_, prt, err := net.SplitHostPort(admPort)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
-			return
-		}
-
-		for _, ip := range admInterfaces {
-			url := fmt.Sprintf("http://%s:%s", ip, prt)
-			if admSsl {
-				url = fmt.Sprintf("https://%s:%s", ip, prt)
-			}
-			lines = append(lines, leftText("  "+url))
-		}
-	}
-
-	// Print the top border
-	fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
-
-	// Print each line
-	for _, line := range lines {
-		fmt.Printf("│%-*s│\n", columnWidth-2, line)
-	}
-
-	// Print the bottom border
-	fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
-}
-
-// getMatchingIPs returns all IP addresses for local system interfaces that
-// match the input address specification.
-func getMatchingIPs(spec string) ([]string, error) {
-	// Split the input spec into IP and port
-	host, _, err := net.SplitHostPort(spec)
-	if err != nil {
-		return nil, fmt.Errorf("parse address/port: %v", err)
-	}
-
-	// Handle cases where IP is omitted (e.g., ":1234")
-	if host == "" {
-		host = "0.0.0.0"
-	}
-
-	ipaddr, err := net.ResolveIPAddr("ip", host)
-	if err != nil {
-		return nil, err
-	}
-
-	parsedInputIP := ipaddr.IP
-
-	var result []string
-
-	// Get all network interfaces
-	interfaces, err := net.Interfaces()
-	if err != nil {
-		return nil, err
-	}
-
-	for _, iface := range interfaces {
-		// Get all addresses associated with the interface
-		addrs, err := iface.Addrs()
-		if err != nil {
-			return nil, err
-		}
-
-		for _, addr := range addrs {
-			// Parse the address to get the IP part
-			ipAddr, _, err := net.ParseCIDR(addr.String())
-			if err != nil {
-				return nil, err
-			}
-
-			if ipAddr.IsLinkLocalUnicast() {
-				continue
-			}
-			if ipAddr.IsInterfaceLocalMulticast() {
-				continue
-			}
-			if ipAddr.IsLinkLocalMulticast() {
-				continue
-			}
-
-			// Check if the IP matches the input specification
-			if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
-				result = append(result, ipAddr.String())
-			}
-		}
-	}
-
-	return result, nil
-}
-
-const columnWidth = 70
-
-func centerText(text string) string {
-	padding := (columnWidth - 2 - len(text)) / 2
-	if padding < 0 {
-		padding = 0
-	}
-	return strings.Repeat(" ", padding) + text
-}
-
-func leftText(text string) string {
-	if len(text) > columnWidth-2 {
-		return text
-	}
-	return text + strings.Repeat(" ", columnWidth-2-len(text))
-}

cmd/versitygw/plugin.go

@@ -1,64 +0,0 @@
-// Copyright 2025 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package main
-
-import (
-	"fmt"
-
-	"github.com/urfave/cli/v2"
-	vgwplugin "github.com/versity/versitygw/backend/plugin"
-)
-
-var (
-	pluginPath   string
-	pluginConfig string
-)
-
-func pluginCommand() *cli.Command {
-	return &cli.Command{
-		Name:        "plugin",
-		Usage:       "plugin storage backend",
-		Description: `This tells the gateway to load the backend from a dynamic runtime plugin.`,
-		Action:      runPlugin,
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:        "file",
-				Usage:       "path to plugin shared object file",
-				Value:       "",
-				Required:    true,
-				EnvVars:     []string{"VGW_PLUGIN_FILE"},
-				Destination: &pluginPath,
-				Aliases:     []string{"f"},
-			},
-			&cli.StringFlag{
-				Name:        "config",
-				Usage:       "configuration option for the plugin",
-				Value:       "",
-				Required:    true,
-				EnvVars:     []string{"VGW_PLUGIN_CONFIG"},
-				Destination: &pluginConfig,
-				Aliases:     []string{"c"},
-			},
-		},
-	}
-}
-
-func runPlugin(ctx *cli.Context) error {
-	be, err := vgwplugin.NewPluginBackend(pluginPath, pluginConfig)
-	if err != nil {
-		return fmt.Errorf("init plugin backend: %w", err)
-	}
-	return runGateway(ctx.Context, be)
-}

cmd/versitygw/posix.go

@@ -16,23 +16,11 @@ package main
 
 import (
 	"fmt"
-	"io/fs"
-	"math"
 
 	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/backend/meta"
 	"github.com/versity/versitygw/backend/posix"
 )
 
-var (
-	chownuid, chowngid bool
-	bucketlinks        bool
-	versioningDir      string
-	dirPerms           uint
-	sidecar            string
-	nometa             bool
-)
-
 func posixCommand() *cli.Command {
 	return &cli.Command{
 		Name:  "posix",
@@ -48,52 +36,6 @@ bucket: mybucket
 object: a/b/c/myobject
 will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 		Action: runPosix,
-		Flags: []cli.Flag{
-			&cli.BoolFlag{
-				Name:        "chuid",
-				Usage:       "chown newly created files and directories to client account UID",
-				EnvVars:     []string{"VGW_CHOWN_UID"},
-				Destination: &chownuid,
-			},
-			&cli.BoolFlag{
-				Name:        "chgid",
-				Usage:       "chown newly created files and directories to client account GID",
-				EnvVars:     []string{"VGW_CHOWN_GID"},
-				Destination: &chowngid,
-			},
-			&cli.BoolFlag{
-				Name:        "bucketlinks",
-				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
-				EnvVars:     []string{"VGW_BUCKET_LINKS"},
-				Destination: &bucketlinks,
-			},
-			&cli.StringFlag{
-				Name:        "versioning-dir",
-				Usage:       "the directory path to enable bucket versioning",
-				EnvVars:     []string{"VGW_VERSIONING_DIR"},
-				Destination: &versioningDir,
-			},
-			&cli.UintFlag{
-				Name:        "dir-perms",
-				Usage:       "default directory permissions for new directories",
-				EnvVars:     []string{"VGW_DIR_PERMS"},
-				Destination: &dirPerms,
-				DefaultText: "0755",
-				Value:       0755,
-			},
-			&cli.StringFlag{
-				Name:        "sidecar",
-				Usage:       "use provided sidecar directory to store metadata",
-				EnvVars:     []string{"VGW_META_SIDECAR"},
-				Destination: &sidecar,
-			},
-			&cli.BoolFlag{
-				Name:        "nometa",
-				Usage:       "disable metadata storage",
-				EnvVars:     []string{"VGW_META_NONE"},
-				Destination: &nometa,
-			},
-		},
 	}
 }
 
@@ -102,46 +44,9 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	gwroot := (ctx.Args().Get(0))
-
-	if dirPerms > math.MaxUint32 {
-		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
-	}
-
-	if nometa && sidecar != "" {
-		return fmt.Errorf("cannot use both nometa and sidecar metadata")
-	}
-
-	opts := posix.PosixOpts{
-		ChownUID:      chownuid,
-		ChownGID:      chowngid,
-		BucketLinks:   bucketlinks,
-		VersioningDir: versioningDir,
-		NewDirPerm:    fs.FileMode(dirPerms),
-	}
-
-	var ms meta.MetadataStorer
-	switch {
-	case sidecar != "":
-		sc, err := meta.NewSideCar(sidecar)
-		if err != nil {
-			return fmt.Errorf("failed to init sidecar metadata: %w", err)
-		}
-		ms = sc
-		opts.SideCarDir = sidecar
-	case nometa:
-		ms = meta.NoMeta{}
-	default:
-		ms = meta.XattrMeta{}
-		err := meta.XattrMeta{}.Test(gwroot)
-		if err != nil {
-			return fmt.Errorf("xattr check failed: %w", err)
-		}
-	}
-
-	be, err := posix.New(gwroot, ms, opts)
+	be, err := posix.New(ctx.Args().Get(0))
 	if err != nil {
-		return fmt.Errorf("failed to init posix backend: %w", err)
+		return fmt.Errorf("init posix: %v", err)
 	}
 
 	return runGateway(ctx.Context, be)

cmd/versitygw/s3.go

@@ -44,7 +44,6 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server access key id",
 				Value:       "",
 				Required:    true,
-				EnvVars:     []string{"VGW_S3_ACCESS_KEY"},
 				Destination: &s3proxyAccess,
 				Aliases:     []string{"a"},
 			},
@@ -53,7 +52,6 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server secret access key",
 				Value:       "",
 				Required:    true,
-				EnvVars:     []string{"VGW_S3_SECRET_KEY"},
 				Destination: &s3proxySecret,
 				Aliases:     []string{"s"},
 			},
@@ -61,27 +59,23 @@ to an s3 storage backend service.`,
 				Name:        "endpoint",
 				Usage:       "s3 service endpoint, default AWS if not specified",
 				Value:       "",
-				EnvVars:     []string{"VGW_S3_ENDPOINT"},
 				Destination: &s3proxyEndpoint,
 			},
 			&cli.StringFlag{
 				Name:        "region",
 				Usage:       "s3 service region, default 'us-east-1' if not specified",
 				Value:       "us-east-1",
-				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
 				Value:       false,
-				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
-				EnvVars:     []string{"VGW_S3_SSL_SKIP_VERIFY"},
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
@@ -89,7 +83,6 @@ to an s3 storage backend service.`,
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
 				Value:       false,
-				EnvVars:     []string{"VGW_S3_DEBUG"},
 				Destination: &s3proxyDebug,
 			},
 		},

cmd/versitygw/scoutfs.go

@@ -16,16 +16,13 @@ package main
 
 import (
 	"fmt"
-	"io/fs"
-	"math"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/scoutfs"
 )
 
 var (
-	glacier          bool
-	disableNoArchive bool
+	glacier bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -51,41 +48,8 @@ move interfaces as well as support for tiered filesystems.`,
 				Name:        "glacier",
 				Usage:       "enable glacier emulation mode",
 				Aliases:     []string{"g"},
-				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
-			&cli.BoolFlag{
-				Name:        "chuid",
-				Usage:       "chown newly created files and directories to client account UID",
-				EnvVars:     []string{"VGW_CHOWN_UID"},
-				Destination: &chownuid,
-			},
-			&cli.BoolFlag{
-				Name:        "chgid",
-				Usage:       "chown newly created files and directories to client account GID",
-				EnvVars:     []string{"VGW_CHOWN_GID"},
-				Destination: &chowngid,
-			},
-			&cli.BoolFlag{
-				Name:        "bucketlinks",
-				Usage:       "allow symlinked directories at bucket level to be treated as buckets",
-				EnvVars:     []string{"VGW_BUCKET_LINKS"},
-				Destination: &bucketlinks,
-			},
-			&cli.UintFlag{
-				Name:        "dir-perms",
-				Usage:       "default directory permissions for new directories",
-				EnvVars:     []string{"VGW_DIR_PERMS"},
-				Destination: &dirPerms,
-				DefaultText: "0755",
-				Value:       0755,
-			},
-			&cli.BoolFlag{
-				Name:        "disable-noarchive",
-				Usage:       "disable setting noarchive for multipart part uploads",
-				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
-				Destination: &disableNoArchive,
-			},
 		},
 	}
 }
@@ -95,19 +59,12 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("no directory provided for operation")
 	}
 
-	if dirPerms > math.MaxUint32 {
-		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
+	var opts []scoutfs.Option
+	if glacier {
+		opts = append(opts, scoutfs.WithGlacierEmulation())
 	}
 
-	var opts scoutfs.ScoutfsOpts
-	opts.GlacierMode = glacier
-	opts.ChownUID = chownuid
-	opts.ChownGID = chowngid
-	opts.BucketLinks = bucketlinks
-	opts.NewDirPerm = fs.FileMode(dirPerms)
-	opts.DisableNoArchive = disableNoArchive
-
-	be, err := scoutfs.New(ctx.Args().Get(0), opts)
+	be, err := scoutfs.New(ctx.Args().Get(0), opts...)
 	if err != nil {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}

cmd/versitygw/test.go

@@ -1,44 +1,27 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package main
 
 import (
 	"fmt"
 
 	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/tests/integration"
+	"github.com/versity/versitygw/integration"
 )
 
 var (
-	awsID             string
-	awsSecret         string
-	endpoint          string
-	prefix            string
-	dstBucket         string
-	partSize          int64
-	objSize           int64
-	concurrency       int
-	files             int
-	totalReqs         int
-	upload            bool
-	download          bool
-	pathStyle         bool
-	checksumDisable   bool
-	versioningEnabled bool
-	azureTests        bool
-	tlsStatus         bool
+	awsID           string
+	awsSecret       string
+	endpoint        string
+	prefix          string
+	dstBucket       string
+	partSize        int64
+	objSize         int64
+	concurrency     int
+	files           int
+	totalReqs       int
+	upload          bool
+	download        bool
+	pathStyle       bool
+	checksumDisable bool
 )
 
 func testCommand() *cli.Command {
@@ -80,12 +63,6 @@ func initTestFlags() []cli.Flag {
 			Aliases:     []string{"d"},
 			Destination: &debug,
 		},
-		&cli.BoolFlag{
-			Name:        "allow-insecure",
-			Usage:       "skip tls verification",
-			Aliases:     []string{"ai"},
-			Destination: &tlsStatus,
-		},
 	}
 }
 
@@ -96,44 +73,17 @@ func initTestCommands() []*cli.Command {
 			Usage:       "Tests the full flow of gateway.",
 			Description: `Runs all the available tests to test the full flow of the gateway.`,
 			Action:      getAction(integration.TestFullFlow),
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test the bucket object versioning, if the versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-				&cli.BoolFlag{
-					Name:        "azure-test-mode",
-					Usage:       "Skips tests that are not supported by Azure",
-					Destination: &azureTests,
-					Aliases:     []string{"azure"},
-				},
-			},
 		},
 		{
 			Name:   "posix",
 			Usage:  "Tests posix specific features",
 			Action: getAction(integration.TestPosix),
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test posix when versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-			},
 		},
 		{
 			Name:   "iam",
 			Usage:  "Tests iam service",
 			Action: getAction(integration.TestIAM),
 		},
-		{
-			Name:   "access-control",
-			Usage:  "Tests gateway access control with bucket ACLs and Policies",
-			Action: getAction(integration.TestAccessControl),
-		},
 		{
 			Name:  "bench",
 			Usage: "Runs download/upload performance test on the gateway",
@@ -218,7 +168,6 @@ func initTestCommands() []*cli.Command {
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
 					integration.WithPartSize(partSize),
-					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -279,7 +228,6 @@ func initTestCommands() []*cli.Command {
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
 					integration.WithConcurrency(concurrency),
-					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
@@ -305,17 +253,10 @@ func getAction(tf testFunc) func(*cli.Context) error {
 			integration.WithSecret(awsSecret),
 			integration.WithRegion(region),
 			integration.WithEndpoint(endpoint),
-			integration.WithTLSStatus(tlsStatus),
 		}
 		if debug {
 			opts = append(opts, integration.WithDebug())
 		}
-		if versioningEnabled {
-			opts = append(opts, integration.WithVersioningEnabled())
-		}
-		if azureTests {
-			opts = append(opts, integration.WithAzureMode())
-		}
 
 		s := integration.NewS3Conf(opts...)
 		tf(s)
@@ -343,27 +284,15 @@ func extractIntTests() (commands []*cli.Command) {
 					integration.WithSecret(awsSecret),
 					integration.WithRegion(region),
 					integration.WithEndpoint(endpoint),
-					integration.WithTLSStatus(tlsStatus),
 				}
 				if debug {
 					opts = append(opts, integration.WithDebug())
 				}
-				if versioningEnabled {
-					opts = append(opts, integration.WithVersioningEnabled())
-				}
 
 				s := integration.NewS3Conf(opts...)
 				err := testFunc(s)
 				return err
 			},
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:        "versioning-enabled",
-					Usage:       "Test the bucket object versioning, if the versioning is enabled",
-					Destination: &versioningEnabled,
-					Aliases:     []string{"vs"},
-				},
-			},
 		})
 	}
 	return

cmd/versitygw/utils.go

@@ -1,91 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/urfave/cli/v2"
-	"github.com/versity/versitygw/s3event"
-)
-
-func utilsCommand() *cli.Command {
-	return &cli.Command{
-		Name:  "utils",
-		Usage: "utility helper CLI tool",
-		Subcommands: []*cli.Command{
-			{
-				Name:    "gen-event-filter-config",
-				Aliases: []string{"gefc"},
-				Usage:   "Create a new configuration file for bucket event notifications filter.",
-				Action:  generateEventFiltersConfig,
-				Flags: []cli.Flag{
-					&cli.StringFlag{
-						Name:    "path",
-						Usage:   "the path where the config file has to be created",
-						Aliases: []string{"p"},
-					},
-				},
-			},
-		},
-	}
-}
-
-func generateEventFiltersConfig(ctx *cli.Context) error {
-	pathFlag := ctx.String("path")
-	path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
-	if err != nil {
-		return err
-	}
-
-	config := s3event.EventFilter{
-		s3event.EventObjectCreated:              true,
-		s3event.EventObjectCreatedPut:           true,
-		s3event.EventObjectCreatedPost:          true,
-		s3event.EventObjectCreatedCopy:          true,
-		s3event.EventCompleteMultipartUpload:    true,
-		s3event.EventObjectRemoved:              true,
-		s3event.EventObjectRemovedDelete:        true,
-		s3event.EventObjectRemovedDeleteObjects: true,
-		s3event.EventObjectTagging:              true,
-		s3event.EventObjectTaggingPut:           true,
-		s3event.EventObjectTaggingDelete:        true,
-		s3event.EventObjectAclPut:               true,
-		s3event.EventObjectRestore:              true,
-		s3event.EventObjectRestorePost:          true,
-		s3event.EventObjectRestoreCompleted:     true,
-	}
-
-	configBytes, err := json.MarshalIndent(config, "", "  ")
-	if err != nil {
-		return fmt.Errorf("parse event config: %w", err)
-	}
-
-	file, err := os.Create(path)
-	if err != nil {
-		return fmt.Errorf("create config file: %w", err)
-	}
-	defer file.Close()
-
-	_, err = file.Write(configBytes)
-	if err != nil {
-		return fmt.Errorf("write config file: %w", err)
-	}
-
-	return nil
-}

docker-compose.yml

@@ -1,8 +1,9 @@
+version: "3"
 services:
   posix:
     build: 
       context: .
-      dockerfile: tests/Dockerfile.dev
+      dockerfile: ./Dockerfile.dev
       args:
         - IAM_DIR=${IAM_DIR}
         - SETUP_DIR=${SETUP_DIR}
@@ -14,7 +15,7 @@ services:
   proxy:
     build: 
       context: .
-      dockerfile: tests/Dockerfile.dev
+      dockerfile: ./Dockerfile.dev
     volumes:
       - ./:/app
     ports:
@@ -28,15 +29,15 @@ services:
       - "10002:10002"
     restart: always
     hostname: azurite
-    command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
+    command: "azurite --oauth basic --cert /certs/azurite.pem --key /certs/azurite-key.pem --blobHost 0.0.0.0"
     volumes:
-      - ./tests/certs:/tests/certs
+      - ./certs:/certs
   azuritegw:
     build:
       context: .
-      dockerfile: tests/Dockerfile.dev
+      dockerfile: ./Dockerfile.dev
     volumes:
       - ./:/app
     ports:
       - 7070:7070
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]

extra/dashboard/README.md

@@ -1,19 +0,0 @@
-# Versity Gateway Dashboard
-
-This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana. 
-
-The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
-
-This implementation uses the influxql query language. 
-
-## Usage
-
-From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
-
-To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`. 
-
-The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
-
-The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with  username: `admin` and password: `admin`.
-
-To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

extra/dashboard/aws_env_setup.sh

@@ -1,4 +0,0 @@
-export AWS_SECRET_ACCESS_KEY=password
-export AWS_ACCESS_KEY_ID=user
-export AWS_ENDPOINT_URL=http://127.0.0.1:7070
-export AWS_REGION=us-east-1

extra/dashboard/docker-compose-metrics.yml

@@ -1,64 +0,0 @@
-services:
-  telegraf:
-    image: telegraf
-    container_name: telegraf
-    restart: always
-    volumes:
-    - ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
-    depends_on:
-      - influxdb
-    links:
-      - influxdb
-    ports:
-    - '8125:8125/udp'
-
-  influxdb:
-    image: influxdb
-    container_name: influxdb
-    restart: always
-    environment:
-      - DOCKER_INFLUXDB_INIT_MODE=setup
-      - DOCKER_INFLUXDB_INIT_USERNAME=admin
-      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
-      - DOCKER_INFLUXDB_INIT_ORG=myorg
-      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
-      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
-    ports:
-      - '8086:8086'
-    volumes:
-      - influxdb_data:/var/lib/influxdb
-
-  grafana:
-    image: grafana/grafana
-    container_name: grafana-server
-    restart: always
-    depends_on:
-      - influxdb
-    environment:
-      - GF_SECURITY_ADMIN_USER=admin
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-      - GF_INSTALL_PLUGINS=
-    links:
-      - influxdb
-    ports:
-      - '3000:3000'
-    volumes:
-      - ./metrics-exploration/grafana_data:/var/lib/grafana
-      - ./metrics-exploration/provisioning:/etc/grafana/provisioning
-
-  versitygw:
-    image: versity/versitygw:latest
-    container_name: versitygw
-    ports:
-      - "7070:7070"
-    environment:
-      - ROOT_ACCESS_KEY=user
-      - ROOT_SECRET_KEY=password
-      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
-    depends_on:
-      - telegraf
-    command: >
-      posix /tmp/vgw
-
-volumes:
-  influxdb_data: {}

extra/dashboard/docker-compose.yml

@@ -1,64 +0,0 @@
-services:
-  telegraf:
-    image: telegraf
-    container_name: telegraf
-    restart: always
-    volumes:
-    - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
-    depends_on:
-      - influxdb
-    links:
-      - influxdb
-    ports:
-    - '8125:8125/udp'
-
-  influxdb:
-    image: influxdb
-    container_name: influxdb
-    restart: always
-    environment:
-      - DOCKER_INFLUXDB_INIT_MODE=setup
-      - DOCKER_INFLUXDB_INIT_USERNAME=admin
-      - DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
-      - DOCKER_INFLUXDB_INIT_ORG=myorg
-      - DOCKER_INFLUXDB_INIT_BUCKET=metrics
-      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
-    ports:
-      - '8086:8086'
-    volumes:
-      - influxdb_data:/var/lib/influxdb
-
-  grafana:
-    image: grafana/grafana
-    container_name: grafana-server
-    restart: always
-    depends_on:
-      - influxdb
-    environment:
-      - GF_SECURITY_ADMIN_USER=admin
-      - GF_SECURITY_ADMIN_PASSWORD=admin
-      - GF_INSTALL_PLUGINS=
-    links:
-      - influxdb
-    ports:
-      - '3000:3000'
-    volumes:
-      - ./grafana_data:/var/lib/grafana
-      - ./provisioning:/etc/grafana/provisioning
-
-  versitygw:
-    image: versity/versitygw:latest
-    container_name: versitygw
-    ports:
-      - "7070:7070"
-    environment:
-      - ROOT_ACCESS_KEY=user
-      - ROOT_SECRET_KEY=password
-      - VGW_METRICS_STATSD_SERVERS=telegraf:8125
-    depends_on:
-      - telegraf
-    command: >
-      posix /tmp/vgw
-
-volumes:
-  influxdb_data: {}

extra/dashboard/provisioning/dashboards/influxql.yaml

@@ -1,25 +0,0 @@
-apiVersion: 1
-
-providers:
-  # <string> an unique provider name. Required
-  - name: 'influxql'
-    # <int> Org id. Default to 1
-    orgId: 1
-    # <string> name of the dashboard folder.
-    folder: 'influxql'
-    # <string> folder UID. will be automatically generated if not specified
-    folderUid: ''
-    # <string> provider type. Default to 'file'
-    type: file
-    # <bool> disable dashboard deletion
-    disableDeletion: false
-    # <int> how often Grafana will scan for changed dashboards
-    updateIntervalSeconds: 10
-    # <bool> allow updating provisioned dashboards from the UI
-    allowUiUpdates: true
-    options:
-      # <string, required> path to dashboard files on disk. Required when using the 'file' type
-      path: /etc/grafana/provisioning/dashboards/influxql
-      # <bool> use folder names from filesystem to create folders in Grafana
-      foldersFromFilesStructure: true
-

extra/dashboard/provisioning/datasources/influxdb.yml

@@ -1,13 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: influxdb
-    type: influxdb
-    isDefault: true
-    access: proxy
-    url: http://influxdb:8086
-    jsonData:
-      dbName: 'metrics'
-      httpHeaderName1: 'Authorization'
-    secureJsonData:
-      httpHeaderValue1: 'Token my-super-secret-auth-token'

extra/dashboard/telegraf.conf

@@ -1,34 +0,0 @@
-[global_tags]
-
-[agent]
-  debug = true
-  quiet = false
-  interval = "60s"
-  round_interval = true
-  metric_batch_size = 1000
-  metric_buffer_limit = 10000
-  collection_jitter = "0s"
-  flush_interval = "10s"
-  flush_jitter = "0s"
-  precision = ""
-  hostname = "versitygw"
-  omit_hostname = false
-
-[[outputs.file]]
-  files = ["stdout"]
-
-[[outputs.influxdb_v2]]
-  urls = ["http://influxdb:8086"]
-  timeout = "5s"
-  token = "my-super-secret-auth-token"
-  organization = "myorg"
-  bucket = "metrics"
-
-[[inputs.statsd]]
-  protocol = "udp4"
-  service_address = ":8125"
-  percentiles = [90]
-  metric_separator = "_"
-  datadog_extensions = false
-  allowed_pending_messages = 10000
-  percentile_limit = 1000

extra/dashboard/test.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-. ./aws_env_setup.sh
-
-aws s3 mb s3://test
-aws s3 cp docker-compose.yml s3://test/test.yaml

extra/example.conf

@@ -1,389 +0,0 @@
-###################################
-# VersityGW systemd configuration #
-###################################
-
-# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
-# For example, if the service name is "mygateway", then the file should be
-# named /etc/versitygw.d/mygateway.conf.
-# The systemd template file /lib/systemd/system/versitygw@.service will
-# automatically load the configuration file for the service name.
-# To start the gateway, use the following command:
-# systemctl start versitygw@mygateway
-# To enable the gateway to start on boot, use the following command:
-# systemctl enable versitygw@mygateway
-# To stop the gateway, use the following command:
-# systemctl stop versitygw@mygateway
-
-# There can be multiple gateway services running on the same host. Each
-# gateway service must have a unique service name with a unique configuration
-# file in /etc/versitygw.d/. They must also listen on different ports and/or
-# interfaces using the VGW_PORT option.
-
-##############################
-# VersityGW Required Options #
-##############################
-
-# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3
-# This defines the backend that the VGW will use for data access.
-VGW_BACKEND=posix
-
-# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
-# as the the top level directory for the gateway.
-# All sub directories of the top level directory are treated as buckets,
-# and all files/directories below the "bucket directory" are treated as
-# the objects. The object name is split on "/" separator to translate
-# to posix storage.
-# For example:
-# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
-# bucket: mybucket
-# object: a/b/c/myobject
-# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
-VGW_BACKEND_ARG=
-
-############################
-# VersityGW Global Options #
-############################
-
-# commented options are the default values
-
-# The following must be set, and do not have default values
-# The access and secret options will specify the root account credentials.
-# The root account is granted full authorization to all API requests after
-# authentication.
-ROOT_ACCESS_KEY_ID=
-ROOT_SECRET_ACCESS_KEY=
-
-# The following are optional, and have the default values as listed
-
-# The VGW_PORT option will specify the listening port for the S3 server.
-# This option can use either the form <ip>:<port> which will listen only
-# on the network interface that matches the IP on the specified port, or
-# :<port> which will listen on all network interfaces on the specified port.
-# The <ip> spec can either be IP dotted notation or a resolvable hostname.
-# The <port> spec can either be a numeric port or the service name typically
-# in /etc/services.
-#VGW_PORT=:7070
-
-# The VGW_REGION option will specify the region that the S3 server will
-# report to clients. This option is optional, and defaults to "us-east-1".
-#VGW_REGION=us-east-1
-
-# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
-# private key that the S3 server will use for SSL connections. This option
-# is optional, and defaults to not using SSL.
-#VGW_CERT=
-#VGW_KEY=
-
-# The VGW_ADMIN_PORT option will specify the listening port for the admin
-# server. The admin server endpoint can optionally be set to listen on a
-# different interface or port than the S3 service. This allows for better
-# control of firewall restrictions to the admin endpoint. The certs for this
-# can be different certs than specified for the S3 service. The default when
-# these are not specified is to have the admin server listen on the same
-# endpoint as the S3 service.
-# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
-# server will use SSL.
-#VGW_ADMIN_PORT=
-#VGW_ADMIN_CERT=
-#VGW_ADMIN_CERT_KEY=
-
-# The VGW_QUIET option when set will supress the S3 server request summary
-# logging to stdout.
-#VGW_QUIET=false
-
-# The VGW_HEALTH option when set will specify the URL to accept health checks
-# on. The health check endpoint is often used for load balancers to verify
-# gateway is alive. The health endpoint masks any bucket with this setting.
-# For example, if the health endpoint is set to /health, the gateway will not
-# allow creating or listing contents of a bucket called "health". The health
-# endpoint is unauthenticated, and returns a 200 status for GET.
-#VGW_HEALTH=
-
-###############
-# Access Logs #
-###############
-
-# The VGW_ACCESS_LOG option when set will specify the file to log all S3
-# server requests to. This option is optional, and defaults to not logging.
-# It is suggested to use absolute paths for the server log file because the
-# server may chdir into the backend root directory and change locations for
-# relative paths.
-# The log file format follows the AWS S3 access log format documented in
-# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
-#VGW_ACCESS_LOG=
-
-# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
-# S3 server request access logs to. The access logs are JSON encoded when
-# sent to the webhook.
-#VGW_LOG_WEBHOOK_URL=
-
-##############
-# Event Logs #
-##############
-
-# The gateway events are similar to AWS S3 events, and are documented in the
-# wiki:
-# https://github.com/versity/versitygw/wiki/Events-Notifications.
-
-# The VGW_EVENT_FILTER option specifies a config file that contains the
-# event filter rules. The event filter rules are used to determine which
-# events are sent to the configured event services.
-# Use the following to generate a default rules file in /etc/versitygw.d/:
-# versitygw utils gen-event-filter-config -p /etc/versitygw.d
-# The resulting file, /etc/versitygw.d/event_config.json, can be modified and
-# specified in the VGW_EVENT_FILTER option.
-# When VGW_EVENT_FILTER is not specified, all events are sent to the configured
-# event service.
-#VGW_EVENT_FILTER=
-
-# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
-# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
-# configured bucket events will be sent to the kafka service.
-#VGW_EVENT_KAFKA_URL=
-#VGW_EVENT_KAFKA_TOPIC=
-#VGW_EVENT_KAFKA_KEY=
-
-# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
-# and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be
-# sent to the the NATS messaging service.
-#VGW_EVENT_NATS_URL=
-#VGW_EVENT_NATS_TOPIC=
-
-# Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is
-# specified, all configured bucket events will be sent to the webhook.
-#VGW_EVENT_WEBHOOK_URL=
-
-# Bucket events can be filtered for any of the above event types. The
-# VGW_EVENT_FILTER option specifies a config file that contains the event
-# filter rules. The event filter rules are used to determine which events are
-# sent to the configured event services. Run:
-# versitygw utils gen-event-filter-config --path .
-# to generate a default rules file "event_config.json" in the current directory.
-#VGW_EVENT_FILTER=
-
-#######################
-# Debug / Diagnostics #
-#######################
-
-# The VGW_DEBUG option enables verbose debug log output to stdout. This output
-# includes details for signature verification steps. This is generally only
-# useful for debugging the S3 server, and should not be used in production.
-#VGW_DEBUG=false
-
-# The VGW_PPROF option enables the pprof HTTP server for profiling the S3
-# server. See the following for more information:
-# https://pkg.go.dev/net/http/pprof
-# To enable, set the VGW_PPROF option to the listening address for the pprof
-# server. For example, to listen on localhost port 6060, set the option to
-# "localhost:6060".
-#VGW_PPROF=
-
-################
-# IAM services #
-################
-
-# The VGW_IAM_DIR option will enable the internal IAM service with accounts
-# stored in a file under the specified directory. This is provided to minimize
-# dependencies on outside services for basic functionality. The local account
-# files are plain text and only protected with file permissions. This IAM
-# service is added for convenience, but is not considered as secure or scalable
-# as a dedicated IAM service.
-#VGW_IAM_DIR=
-
-# The Vault options will enable the Vault IAM service with accounts stored in
-# the HashiCorp Vault service. The Vault URL is the address and port of the
-# Vault server with the format <IP/host>:<port>. A root taken can be used for
-# testing, but it is recommended to use the role based authentication in
-# production. The Vault server certificate, client certificate, and client
-# certificate key are optional, and will default to not verifying the server
-# certificate and not using client certificates. The Vault server certificate
-# is used to verify the Vault server, and the client certificate and key are
-# used to authenticate the gateway to the Vault server. See wiki documentation
-# for an example of using Vault in dev mode with the gateway.
-#VGW_IAM_VAULT_ENDPOINT_URL=
-#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
-#VGW_IAM_VAULT_MOUNT_PATH=
-#VGW_IAM_VAULT_ROOT_TOKEN=
-#VGW_IAM_VAULT_ROLE_ID=
-#VGW_IAM_VAULT_ROLE_SECRET=
-#VGW_IAM_VAULT_SERVER_CERT=
-#VGW_IAM_VAULT_CLIENT_CERT=
-#VGW_IAM_VAULT_CLIENT_CERT_KEY=
-
-# The VGW_S3 IAM service is similar to the internal IAM service, but instead
-# stores the account information JSON encoded in an S3 object. This should use
-# a bucket that is not accessible to general users when using s3 backend to
-# prevent access to account credentials. This IAM service is added for
-# convenience, but is not considered as secure or scalable as a dedicated IAM
-# service.
-#VGW_S3_IAM_ACCESS_KEY=
-#VGW_S3_IAM_SECRET_KEY=
-#VGW_S3_IAM_REGION=
-#VGW_S3_IAM_ENDPOINT=
-#VGW_S3_IAM_BUCKET=
-#VGW_S3_IAM_NO_VERIFY=
-
-# The LDAP options will enable the LDAP IAM service with accounts stored in an
-# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
-# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
-# secret credentials and role respectively. The other options are used to
-# connect to the LDAP service.
-#VGW_IAM_LDAP_URL=
-#VGW_IAM_LDAP_BASE_DN=
-#VGW_IAM_LDAP_BIND_DN=
-#VGW_IAM_LDAP_BIND_PASS=
-#VGW_IAM_LDAP_QUERY_BASE=
-#VGW_IAM_LDAP_OBJECT_CLASSES=
-#VGW_IAM_LDAP_ACCESS_ATR=
-#VGW_IAM_LDAP_SECRET_ATR=
-#VGW_IAM_LDAP_ROLE_ATR=
-#VGW_IAM_LDAP_USER_ID_ATR=
-#VGW_IAM_LDAP_GROUP_ID_ATR=
-
-###############
-# IAM caching #
-###############
-
-# The IAM cache is intended to ease the load on the IAM service and increase
-# the Gateway performance by caching accounts and credentials for the TTL time
-# interval. Disabling this will cause a request to the configured IAM service
-# for each incoming request to retrieve the corresponding account credentials.
-# The cache is enabled by default. The TTL specifies how long to cache
-# credentials, and the prune value determines the interval for expired entries
-# to be removed from the cache. Increasing the TTL may lessen the load on the
-# IAM service backend, but may have out of date account info until the next
-# interval. Increasing the prune value may reduce memory use at the cost of
-# added CPU to check cache expirations.
-#VGW_IAM_CACHE_DISABLE=false
-#VGW_IAM_CACHE_TTL=120
-#VGW_IAM_CACHE_PRUNE=3600
-
-###########
-# Metrics #
-###########
-
-# The metrics service name is a tag that is added to all metrics to help
-# identify the source of the metrics. This is especially useful when multiple
-# gateways are running. The default is the hostname of the system.
-#VGW_METRICS_SERVICE_NAME=$HOSTNAME
-
-# The metrics service will send metrics to the configured statsd servers. The
-# servers are specified as a comma separated list of host:port pairs. The
-# default is to not send metrics to any statsd servers. The gateway uses
-# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
-#VGW_METRICS_STATSD_SERVERS=
-
-# The metrics service will send metrics to the configured dogstatsd servers.
-# The servers are specified as a comma separated list of host:port pairs. The
-# default is to not send metrics to any dogstatsd servers. Generally
-# DataDog recommends installing a local agent to collect metrics and forward
-# them to the DataDog service. In this case the option value would be the
-# local agent address: 127.0.0.1:8125.
-#VGW_METRICS_DOGSTATS_SERVERS=
-
-######################################
-# VersityGW Backend Specific Options #
-######################################
-
-#########
-# posix #
-#########
-
-# The posix backend translates S3 requests to file access in a local filesystem.
-# The posix backend requires a filesystem that supports extended attributes.
-# The top level directory for the gateway must be provided. All sub directories
-# of the top level directory are treated as buckets, and all files/directories
-# below the "bucket directory" are treated as the objects. The object
-# name is split on "/" separator to translate to posix storage.
-# For example:
-# top level (VGW_BACKEND_ARG): /mnt/fs/gwroot
-# bucket: mybucket
-# object: a/b/c/myobject
-# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
-
-# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
-# change the ownership of newly created files and directories to the IAM
-# account UID/GID.
-#VGW_CHOWN_UID=false
-#VGW_CHOWN_GID=false
-
-# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
-# to directories at the top level gateway directory as buckets.
-#VGW_BUCKET_LINKS=false
-
-# The default permissions mode when creating new directories is 0755. Use
-# VGW_DIR_PERMS option to set a different mode for any new directory that the
-# gateway creates. This applies to buckets created through the gateway as well
-# as any parent directories automatically created with object uploads.
-#VGW_DIR_PERMS=0755
-
-###########
-# scoutfs #
-###########
-
-# The scoutfs backend requires a ScoutFS filesystem type for the backend
-# path. The object to posix name mappings follow the same rules as posix for
-# scoutfs. The glacier mode functionality requires ScoutAM to be configured
-# for tiering data from the ScoutFS filesystem to a mass stroage system.
-# The mass storage system is often one or more tape libraries. Due to the
-# high latency of tape, the glacier mode functionality is designed to
-# give feedback to clients about object state and offer ability to request
-# data to be staged back to disk without the client dealing with long
-# request timeout settings.
-
-# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
-# GET object:  if file offline, return invalid object state
-# HEAD object: if file offline, set obj storage class to GLACIER
-#              if file offline and staging, x-amz-restore: ongoing-request="true"
-#              if file offline and not staging, x-amz-restore: ongoing-request="false"
-#              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
-#              note: this expiry-date is not used but provided for client glacier compatibility
-# ListObjects: if file offline, set obj storage class to GLACIER
-# RestoreObject: add batch stage request to file
-#VGW_SCOUTFS_GLACIER=false
-
-# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
-# change the ownership of newly created files and directories to the IAM
-# account UID/GID.
-#VGW_CHOWN_UID=false
-#VGW_CHOWN_GID=false
-
-# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
-# to directories at the top level gateway directory as buckets.
-#VGW_BUCKET_LINKS=false
-
-# The default permissions mode when creating new directories is 0755. Use
-# VGW_DIR_PERMS option to set a different mode for any new directory that the
-# gateway creates. This applies to buckets created through the gateway as well
-# as any parent directories automatically created with object uploads.
-#VGW_DIR_PERMS=0755
-
-# The default behavior of the gateway is to automatically set the noarchive
-# flag on the multipart upload parts while the multipart upload is in progress.
-# This is to prevent the parts from being archived since they are temporary
-# and will be deleted after the multipart upload is completed or aborted. The
-# VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
-#VGW_DISABLE_NOARCHIVE=false
-
-######
-# s3 #
-######
-
-# The s3 backend allows the gateway to forward requests to an S3 compatible
-# service. This allows the gateway to act as a proxy for another S3 service.
-# The backend S3 access is all done with a single configured account. The
-# gateway will manage incoming multi-tenant access with the gateway configured
-# IAM service. This gives stroage admins the ability to manage local gateway
-# accounts while maintaining full control and a single account for the backend
-# S3 service.
-
-# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
-# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
-# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
-#VGW_S3_ACCESS_KEY=
-#VGW_S3_SECRET_KEY=
-#VGW_S3_REGION=
-#VGW_S3_ENDPOINT=
-#VGW_S3_DISABLE_CHECKSUM=false
-#VGW_S3_SSL_SKIP_VERIFY=false
-#VGW_S3_DEBUG=false

extra/posttrans.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-systemctl daemon-reload

extra/versitygw@.service

@@ -1,33 +0,0 @@
-[Unit]
-Description=VersityGW
-Documentation=https://github.com/versity/versitygw/wiki
-Wants=network-online.target
-After=network-online.target remote-fs.target
-AssertFileIsExecutable=/usr/bin/versitygw
-AssertPathExists=/etc/versitygw.d/%i.conf
-
-[Service]
-WorkingDirectory=/root
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=versitygw-%i
-
-User=root
-Group=root
-
-EnvironmentFile=/etc/versitygw.d/%i.conf
-
-ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"'
-
-# Let systemd restart this service always
-Restart=always
-
-# Specifies the maximum file descriptor number that can be opened by this process
-LimitNOFILE=65536
-
-# Specifies the maximum number of threads this process can create
-TasksMax=infinity
-
-[Install]
-WantedBy=multi-user.target
-

go.mod

@@ -1,83 +1,70 @@
 module github.com/versity/versitygw
 
-go 1.23.0
-
-toolchain go1.24.1
+go 1.21
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
-	github.com/DataDog/datadog-go/v5 v5.6.0
-	github.com/aws/aws-sdk-go-v2 v1.36.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1
-	github.com/aws/smithy-go v1.22.3
-	github.com/go-ldap/ldap/v3 v3.4.10
-	github.com/gofiber/fiber/v2 v2.52.6
-	github.com/google/go-cmp v0.7.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
+	github.com/aws/aws-sdk-go-v2 v1.25.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2
+	github.com/aws/smithy-go v1.20.1
+	github.com/go-ldap/ldap/v3 v3.4.6
+	github.com/gofiber/fiber/v2 v2.52.2
+	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.41.0
-	github.com/oklog/ulid/v2 v2.1.0
-	github.com/pkg/xattr v0.4.10
+	github.com/nats-io/nats.go v1.33.1
+	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
-	github.com/smira/go-statsd v1.3.4
-	github.com/urfave/cli/v2 v2.27.6
-	github.com/valyala/fasthttp v1.60.0
-	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
-	golang.org/x/sync v0.13.0
-	golang.org/x/sys v0.32.0
+	github.com/urfave/cli/v2 v2.27.1
+	github.com/valyala/fasthttp v1.52.0
+	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
+	golang.org/x/sys v0.18.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
-	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
-	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
-	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.2 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/nats-io/nkeys v0.4.10 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.37.0 // indirect
-	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/text v0.24.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 )
 
 require (
-	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.29.13
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.66
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
-	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.5
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.5
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 )

go.sum

@@ -1,276 +1,182 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2/go.mod h1:SqINnQ9lVVdRlyC8cd1lCI0SdX4n2paeABd2K8ggfnE=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 h1:Bg8m3nq/X1DeePkAbCfb6ml6F3F0IunEhE8TMh+lY48=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
-github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
-github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
-github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
-github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
-github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
-github.com/aws/aws-sdk-go-v2/config v1.29.13 h1:RgdPqWoE8nPpIekpVpDJsBckbqT4Liiaq9f35pbTh1Y=
-github.com/aws/aws-sdk-go-v2/config v1.29.13/go.mod h1:NI28qs/IOUIRhsR7GQ/JdexoqRN9tDxkIrYZq0SOF44=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.66 h1:aKpEKaTy6n4CEJeYI1MNj97oSDLi4xro3UzQfwf5RWE=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.66/go.mod h1:xQ5SusDmHb/fy55wU0QqTy0yNfLqxzec59YcsRZB+rI=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 h1:s43gLuY+zGmtpx+KybfFP4IckopmTfDOPdlf/L++N5I=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71/go.mod h1:KH6wWmY3O3c/jVAjHk0MGzVAFDxkOSt42Eoe4ZO4ge0=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 h1:2Ku1xwAohSSXHR1tpAnyVDSQSxoDMA+/NZBytW+f4qg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
-github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3QasqBfuAFHY4Pk5g=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
-github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
-github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
-github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w=
+github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
+github.com/aws/aws-sdk-go-v2/config v1.27.5 h1:brBPsyRFQn97M1ZhQ9tLXkO7Zytiar0NS06FGmEJBdg=
+github.com/aws/aws-sdk-go-v2/config v1.27.5/go.mod h1:I53uvsfddRRTG5YcC4n5Z3aOD1BU8hYCoIG7iEJG4wM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.5 h1:yn3zSvIKC2NZIs40cY3kckcy9Zma96PrRR07N54PCvY=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.5/go.mod h1:8JcKPAGZVnDWuR5lusAwmrSDtZnDIAnpQWaDC9RFt2g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7 h1:/r2O0R/JAD1Y1iCxxz7nClKntXqB9CLTrxu7csrAsSA=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7/go.mod h1:TbQoOduGh1PZbTNRqaEemgj/e+mmFC3hScHEQDTcUoQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 h1:bNo4LagzUKbjdxE0tIcR9pMzLR2U/Tgie1Hq1HQ3iH8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 h1:EtOU5jsPdIQNP+6Q2C5e3d65NKT1PeCiQk+9OdzO12Q=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 h1:en92G0Z7xlksoOylkUhuBSfJgijC7rHVLRdnIlHEs0E=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2/go.mod h1:HgtQ/wN5G+8QSlK62lbOtNwQ3wTSByJ4wH2rCkPt+AE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3 h1:fpFzBoro/MetYBk+8kxoQGMeKSkXbymnbUh2gy6nVgk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3/go.mod h1:qmQPbMe5NQk/nEmpkl8iHyCSREJjEbRUrnqHpHabLlM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3 h1:x0N5ftQzgcfRpCpTiyZC40pvNUJYhzf4UgCsAyO6/P8=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 h1:1oY1AVEisRI4HNuFoLdRUB0hC63ylDAN6Me3MrfclEg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2/go.mod h1:KZ03VgvZwSjkT7fOetQ/wF3MZUvYFirlI1H5NklUNsY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2 h1:ukAaTX8n/pX0Essg9CxW8VCjACv75vnNo2GRONR1w1Q=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2/go.mod h1:wt4wZz/CBlJJwY0L7X6vPQ9njh2aHi59knqpJ6B/2cM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 h1:utEGkfdQ4L6YW/ietH7111ZYglLJvS+sLriHJ1NBJEQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 h1:9/GylMS45hGGFCcMrUZDVayQE1jYSIN6da9jo7RAYIw=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.2 h1:0YjXuWdYHvsm0HnT4vO8XpwG1D+i2roxSCBoN6deJ7M=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.2/go.mod h1:jI+FWmYkSMn+4APWmZiZTgt0oM0TrvymD51FMqCnWgA=
+github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
+github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
-github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
-github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
-github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
-github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
-github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
-github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
+github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
+github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
+github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo=
+github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
+github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
-github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
-github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
-github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
-github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
-github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
-github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
-github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
-github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
-github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
-github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
-github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
-github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
-github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
-github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
-github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
-github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
-github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
-github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
-github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
-github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
-github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
-github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
-github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
-github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
-github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.41.0 h1:PzxEva7fflkd+n87OtQTXqCTyLfIIMFJBpyccHLE2Ko=
-github.com/nats-io/nats.go v1.41.0/go.mod h1:wV73x0FSI/orHPSYoyMeJB+KajMDoWyXmFaRrrYaaTo=
-github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc=
-github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
+github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
-github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
-github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
-github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
-github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
+github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
-github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
-github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
 github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
-github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
-github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
+github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.60.0 h1:kBRYS0lOhVJ6V+bYN8PqAHELKHtXqwq9zNMLKx1MBsw=
-github.com/valyala/fasthttp v1.60.0/go.mod h1:iY4kDgV3Gc6EqhRZ8icqcmlG6bqhcDXfuHgTO4FXCvc=
-github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
-github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
+github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
+github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
+github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
-github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
-golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
-golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
-golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -278,25 +184,17 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
-golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
-golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
-golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration/bench.go

@@ -1,17 +1,3 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package integration
 
 import (

integration/data-io.go

@@ -1,17 +1,3 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package integration
 
 import (

integration/group-tests.go

@@ -0,0 +1,455 @@
+package integration
+
+func TestAuthentication(s *S3Conf) {
+	Authentication_empty_auth_header(s)
+	Authentication_invalid_auth_header(s)
+	Authentication_unsupported_signature_version(s)
+	Authentication_malformed_credentials(s)
+	Authentication_malformed_credentials_invalid_parts(s)
+	Authentication_credentials_terminated_string(s)
+	Authentication_credentials_incorrect_service(s)
+	Authentication_credentials_incorrect_region(s)
+	Authentication_credentials_invalid_date(s)
+	Authentication_credentials_future_date(s)
+	Authentication_credentials_past_date(s)
+	Authentication_credentials_non_existing_access_key(s)
+	Authentication_invalid_signed_headers(s)
+	Authentication_missing_date_header(s)
+	Authentication_invalid_date_header(s)
+	Authentication_date_mismatch(s)
+	Authentication_incorrect_payload_hash(s)
+	Authentication_incorrect_md5(s)
+	Authentication_signature_error_incorrect_secret_key(s)
+}
+
+func TestPresignedAuthentication(s *S3Conf) {
+	PresignedAuth_missing_algo_query_param(s)
+	PresignedAuth_unsupported_algorithm(s)
+	PresignedAuth_missing_credentials_query_param(s)
+	PresignedAuth_malformed_creds_invalid_parts(s)
+	PresignedAuth_malformed_creds_invalid_parts(s)
+	PresignedAuth_creds_incorrect_service(s)
+	PresignedAuth_creds_incorrect_region(s)
+	PresignedAuth_creds_invalid_date(s)
+	PresignedAuth_missing_date_query(s)
+	PresignedAuth_dates_mismatch(s)
+	PresignedAuth_non_existing_access_key_id(s)
+	PresignedAuth_missing_signed_headers_query_param(s)
+	PresignedAuth_missing_expiration_query_param(s)
+	PresignedAuth_invalid_expiration_query_param(s)
+	PresignedAuth_negative_expiration_query_param(s)
+	PresignedAuth_exceeding_expiration_query_param(s)
+	PresignedAuth_expired_request(s)
+	PresignedAuth_incorrect_secret_key(s)
+	PresignedAuth_PutObject_success(s)
+	PresignedAuth_Put_GetObject_with_data(s)
+	PresignedAuth_UploadPart(s)
+}
+
+func TestCreateBucket(s *S3Conf) {
+	CreateBucket_invalid_bucket_name(s)
+	CreateBucket_existing_bucket(s)
+	CreateBucket_as_user(s)
+	CreateBucket_default_acl(s)
+	CreateBucket_non_default_acl(s)
+	CreateDeleteBucket_success(s)
+}
+
+func TestHeadBucket(s *S3Conf) {
+	HeadBucket_non_existing_bucket(s)
+	HeadBucket_success(s)
+}
+
+func TestListBuckets(s *S3Conf) {
+	ListBuckets_as_user(s)
+	ListBuckets_as_admin(s)
+	ListBuckets_success(s)
+}
+
+func TestDeleteBucket(s *S3Conf) {
+	DeleteBucket_non_existing_bucket(s)
+	DeleteBucket_non_empty_bucket(s)
+	DeleteBucket_success_status_code(s)
+}
+
+func TestPutBucketTagging(s *S3Conf) {
+	PutBucketTagging_non_existing_bucket(s)
+	PutBucketTagging_long_tags(s)
+	PutBucketTagging_success(s)
+}
+
+func TestGetBucketTagging(s *S3Conf) {
+	GetBucketTagging_non_existing_bucket(s)
+	GetBucketTagging_success(s)
+}
+
+func TestDeleteBucketTagging(s *S3Conf) {
+	DeleteBucketTagging_non_existing_object(s)
+	DeleteBucketTagging_success_status(s)
+	DeleteBucketTagging_success(s)
+}
+
+func TestPutObject(s *S3Conf) {
+	PutObject_non_existing_bucket(s)
+	PutObject_special_chars(s)
+	PutObject_invalid_long_tags(s)
+	PutObject_success(s)
+	PutObject_invalid_credentials(s)
+}
+
+func TestHeadObject(s *S3Conf) {
+	HeadObject_non_existing_object(s)
+	HeadObject_success(s)
+}
+
+func TestGetObject(s *S3Conf) {
+	GetObject_non_existing_key(s)
+	GetObject_invalid_ranges(s)
+	GetObject_with_meta(s)
+	GetObject_success(s)
+	GetObject_by_range_success(s)
+}
+
+func TestListObjects(s *S3Conf) {
+	ListObjects_non_existing_bucket(s)
+	ListObjects_with_prefix(s)
+	ListObject_truncated(s)
+	ListObjects_invalid_max_keys(s)
+	ListObjects_max_keys_0(s)
+	ListObjects_delimiter(s)
+	ListObjects_max_keys_none(s)
+	ListObjects_marker_not_from_obj_list(s)
+}
+
+func TestListObjectsV2(s *S3Conf) {
+	ListObjectsV2_start_after(s)
+	ListObjectsV2_both_start_after_and_continuation_token(s)
+	ListObjectsV2_start_after_not_in_list(s)
+	ListObjectsV2_start_after_empty_result(s)
+}
+
+func TestDeleteObject(s *S3Conf) {
+	DeleteObject_non_existing_object(s)
+	DeleteObject_success(s)
+	DeleteObject_success_status_code(s)
+}
+
+func TestDeleteObjects(s *S3Conf) {
+	DeleteObjects_empty_input(s)
+	DeleteObjects_non_existing_objects(s)
+	DeleteObjects_success(s)
+}
+
+func TestCopyObject(s *S3Conf) {
+	CopyObject_non_existing_dst_bucket(s)
+	CopyObject_not_owned_source_bucket(s)
+	CopyObject_copy_to_itself(s)
+	CopyObject_to_itself_with_new_metadata(s)
+	CopyObject_success(s)
+}
+
+func TestPutObjectTagging(s *S3Conf) {
+	PutObjectTagging_non_existing_object(s)
+	PutObjectTagging_long_tags(s)
+	PutObjectTagging_success(s)
+}
+
+func TestGetObjectTagging(s *S3Conf) {
+	GetObjectTagging_non_existing_object(s)
+	GetObjectTagging_success(s)
+}
+
+func TestDeleteObjectTagging(s *S3Conf) {
+	DeleteObjectTagging_non_existing_object(s)
+	DeleteObjectTagging_success_status(s)
+	DeleteObjectTagging_success(s)
+}
+
+func TestCreateMultipartUpload(s *S3Conf) {
+	CreateMultipartUpload_non_existing_bucket(s)
+	CreateMultipartUpload_success(s)
+}
+
+func TestUploadPart(s *S3Conf) {
+	UploadPart_non_existing_bucket(s)
+	UploadPart_invalid_part_number(s)
+	UploadPart_non_existing_key(s)
+	UploadPart_non_existing_mp_upload(s)
+	UploadPart_success(s)
+}
+
+func TestUploadPartCopy(s *S3Conf) {
+	UploadPartCopy_non_existing_bucket(s)
+	UploadPartCopy_incorrect_uploadId(s)
+	UploadPartCopy_incorrect_object_key(s)
+	UploadPartCopy_invalid_part_number(s)
+	UploadPartCopy_invalid_copy_source(s)
+	UploadPartCopy_non_existing_source_bucket(s)
+	UploadPartCopy_non_existing_source_object_key(s)
+	UploadPartCopy_success(s)
+	UploadPartCopy_by_range_invalid_range(s)
+	UploadPartCopy_greater_range_than_obj_size(s)
+	UploadPartCopy_by_range_success(s)
+}
+
+func TestListParts(s *S3Conf) {
+	ListParts_incorrect_uploadId(s)
+	ListParts_incorrect_object_key(s)
+	ListParts_success(s)
+}
+
+func TestListMultipartUploads(s *S3Conf) {
+	ListMultipartUploads_non_existing_bucket(s)
+	ListMultipartUploads_empty_result(s)
+	ListMultipartUploads_invalid_max_uploads(s)
+	ListMultipartUploads_max_uploads(s)
+	ListMultipartUploads_incorrect_next_key_marker(s)
+	ListMultipartUploads_ignore_upload_id_marker(s)
+	ListMultipartUploads_success(s)
+}
+
+func TestAbortMultipartUpload(s *S3Conf) {
+	AbortMultipartUpload_non_existing_bucket(s)
+	AbortMultipartUpload_incorrect_uploadId(s)
+	AbortMultipartUpload_incorrect_object_key(s)
+	AbortMultipartUpload_success(s)
+	AbortMultipartUpload_success_status_code(s)
+}
+
+func TestCompleteMultipartUpload(s *S3Conf) {
+	CompletedMultipartUpload_non_existing_bucket(s)
+	CompleteMultipartUpload_invalid_part_number(s)
+	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_success(s)
+}
+
+func TestPutBucketAcl(s *S3Conf) {
+	PutBucketAcl_non_existing_bucket(s)
+	PutBucketAcl_invalid_acl_canned_and_acp(s)
+	PutBucketAcl_invalid_acl_canned_and_grants(s)
+	PutBucketAcl_invalid_acl_acp_and_grants(s)
+	PutBucketAcl_invalid_owner(s)
+	PutBucketAcl_success_access_denied(s)
+	PutBucketAcl_success_grants(s)
+	PutBucketAcl_success_canned_acl(s)
+	PutBucketAcl_success_acp(s)
+}
+
+func TestGetBucketAcl(s *S3Conf) {
+	GetBucketAcl_non_existing_bucket(s)
+	GetBucketAcl_access_denied(s)
+	GetBucketAcl_success(s)
+}
+
+func TestFullFlow(s *S3Conf) {
+	TestAuthentication(s)
+	TestPresignedAuthentication(s)
+	TestCreateBucket(s)
+	TestHeadBucket(s)
+	TestListBuckets(s)
+	TestDeleteBucket(s)
+	TestPutBucketTagging(s)
+	TestGetBucketTagging(s)
+	TestDeleteBucketTagging(s)
+	TestPutObject(s)
+	TestHeadObject(s)
+	TestGetObject(s)
+	TestListObjects(s)
+	TestListObjectsV2(s)
+	TestDeleteObject(s)
+	TestDeleteObjects(s)
+	TestCopyObject(s)
+	TestPutObjectTagging(s)
+	TestDeleteObjectTagging(s)
+	TestCreateMultipartUpload(s)
+	TestUploadPart(s)
+	TestUploadPartCopy(s)
+	TestListParts(s)
+	TestListMultipartUploads(s)
+	TestAbortMultipartUpload(s)
+	TestCompleteMultipartUpload(s)
+	TestPutBucketAcl(s)
+	TestGetBucketAcl(s)
+}
+
+func TestPosix(s *S3Conf) {
+	PutObject_overwrite_dir_obj(s)
+	PutObject_overwrite_file_obj(s)
+	PutObject_dir_obj_with_data(s)
+	CreateMultipartUpload_dir_obj(s)
+}
+
+func TestIAM(s *S3Conf) {
+	IAM_user_access_denied(s)
+	IAM_userplus_access_denied(s)
+	IAM_userplus_CreateBucket(s)
+	IAM_admin_ChangeBucketOwner(s)
+}
+
+type IntTests map[string]func(s *S3Conf) error
+
+func GetIntTests() IntTests {
+	return IntTests{
+		"Authentication_empty_auth_header":                      Authentication_empty_auth_header,
+		"Authentication_invalid_auth_header":                    Authentication_invalid_auth_header,
+		"Authentication_unsupported_signature_version":          Authentication_unsupported_signature_version,
+		"Authentication_malformed_credentials":                  Authentication_malformed_credentials,
+		"Authentication_malformed_credentials_invalid_parts":    Authentication_malformed_credentials_invalid_parts,
+		"Authentication_credentials_terminated_string":          Authentication_credentials_terminated_string,
+		"Authentication_credentials_incorrect_service":          Authentication_credentials_incorrect_service,
+		"Authentication_credentials_incorrect_region":           Authentication_credentials_incorrect_region,
+		"Authentication_credentials_invalid_date":               Authentication_credentials_invalid_date,
+		"Authentication_credentials_future_date":                Authentication_credentials_future_date,
+		"Authentication_credentials_past_date":                  Authentication_credentials_past_date,
+		"Authentication_credentials_non_existing_access_key":    Authentication_credentials_non_existing_access_key,
+		"Authentication_invalid_signed_headers":                 Authentication_invalid_signed_headers,
+		"Authentication_missing_date_header":                    Authentication_missing_date_header,
+		"Authentication_invalid_date_header":                    Authentication_invalid_date_header,
+		"Authentication_date_mismatch":                          Authentication_date_mismatch,
+		"Authentication_incorrect_payload_hash":                 Authentication_incorrect_payload_hash,
+		"Authentication_incorrect_md5":                          Authentication_incorrect_md5,
+		"Authentication_signature_error_incorrect_secret_key":   Authentication_signature_error_incorrect_secret_key,
+		"PresignedAuth_missing_algo_query_param":                PresignedAuth_missing_algo_query_param,
+		"PresignedAuth_unsupported_algorithm":                   PresignedAuth_unsupported_algorithm,
+		"PresignedAuth_missing_credentials_query_param":         PresignedAuth_missing_credentials_query_param,
+		"PresignedAuth_malformed_creds_invalid_parts":           PresignedAuth_malformed_creds_invalid_parts,
+		"PresignedAuth_creds_invalid_terminator":                PresignedAuth_creds_invalid_terminator,
+		"PresignedAuth_creds_incorrect_service":                 PresignedAuth_creds_incorrect_service,
+		"PresignedAuth_creds_incorrect_region":                  PresignedAuth_creds_incorrect_region,
+		"PresignedAuth_creds_invalid_date":                      PresignedAuth_creds_invalid_date,
+		"PresignedAuth_missing_date_query":                      PresignedAuth_missing_date_query,
+		"PresignedAuth_dates_mismatch":                          PresignedAuth_dates_mismatch,
+		"PresignedAuth_non_existing_access_key_id":              PresignedAuth_non_existing_access_key_id,
+		"PresignedAuth_missing_signed_headers_query_param":      PresignedAuth_missing_signed_headers_query_param,
+		"PresignedAuth_missing_expiration_query_param":          PresignedAuth_missing_expiration_query_param,
+		"PresignedAuth_invalid_expiration_query_param":          PresignedAuth_invalid_expiration_query_param,
+		"PresignedAuth_negative_expiration_query_param":         PresignedAuth_negative_expiration_query_param,
+		"PresignedAuth_exceeding_expiration_query_param":        PresignedAuth_exceeding_expiration_query_param,
+		"PresignedAuth_expired_request":                         PresignedAuth_expired_request,
+		"PresignedAuth_incorrect_secret_key":                    PresignedAuth_incorrect_secret_key,
+		"PresignedAuth_PutObject_success":                       PresignedAuth_PutObject_success,
+		"PresignedAuth_Put_GetObject_with_data":                 PresignedAuth_Put_GetObject_with_data,
+		"PresignedAuth_UploadPart":                              PresignedAuth_UploadPart,
+		"CreateBucket_invalid_bucket_name":                      CreateBucket_invalid_bucket_name,
+		"CreateBucket_existing_bucket":                          CreateBucket_existing_bucket,
+		"CreateBucket_as_user":                                  CreateBucket_as_user,
+		"CreateDeleteBucket_success":                            CreateDeleteBucket_success,
+		"CreateBucket_default_acl":                              CreateBucket_default_acl,
+		"CreateBucket_non_default_acl":                          CreateBucket_non_default_acl,
+		"HeadBucket_non_existing_bucket":                        HeadBucket_non_existing_bucket,
+		"HeadBucket_success":                                    HeadBucket_success,
+		"ListBuckets_as_user":                                   ListBuckets_as_user,
+		"ListBuckets_as_admin":                                  ListBuckets_as_admin,
+		"ListBuckets_success":                                   ListBuckets_success,
+		"DeleteBucket_non_existing_bucket":                      DeleteBucket_non_existing_bucket,
+		"DeleteBucket_non_empty_bucket":                         DeleteBucket_non_empty_bucket,
+		"DeleteBucket_success_status_code":                      DeleteBucket_success_status_code,
+		"PutBucketTagging_non_existing_bucket":                  PutBucketTagging_non_existing_bucket,
+		"PutBucketTagging_long_tags":                            PutBucketTagging_long_tags,
+		"PutBucketTagging_success":                              PutBucketTagging_success,
+		"GetBucketTagging_non_existing_bucket":                  GetBucketTagging_non_existing_bucket,
+		"GetBucketTagging_success":                              GetBucketTagging_success,
+		"DeleteBucketTagging_non_existing_object":               DeleteBucketTagging_non_existing_object,
+		"DeleteBucketTagging_success_status":                    DeleteBucketTagging_success_status,
+		"DeleteBucketTagging_success":                           DeleteBucketTagging_success,
+		"PutObject_non_existing_bucket":                         PutObject_non_existing_bucket,
+		"PutObject_special_chars":                               PutObject_special_chars,
+		"PutObject_invalid_long_tags":                           PutObject_invalid_long_tags,
+		"PutObject_success":                                     PutObject_success,
+		"HeadObject_non_existing_object":                        HeadObject_non_existing_object,
+		"HeadObject_success":                                    HeadObject_success,
+		"GetObject_non_existing_key":                            GetObject_non_existing_key,
+		"GetObject_invalid_ranges":                              GetObject_invalid_ranges,
+		"GetObject_with_meta":                                   GetObject_with_meta,
+		"GetObject_success":                                     GetObject_success,
+		"GetObject_by_range_success":                            GetObject_by_range_success,
+		"ListObjects_non_existing_bucket":                       ListObjects_non_existing_bucket,
+		"ListObjects_with_prefix":                               ListObjects_with_prefix,
+		"ListObject_truncated":                                  ListObject_truncated,
+		"ListObjects_invalid_max_keys":                          ListObjects_invalid_max_keys,
+		"ListObjects_max_keys_0":                                ListObjects_max_keys_0,
+		"ListObjects_delimiter":                                 ListObjects_delimiter,
+		"ListObjects_max_keys_none":                             ListObjects_max_keys_none,
+		"ListObjects_marker_not_from_obj_list":                  ListObjects_marker_not_from_obj_list,
+		"ListObjectsV2_start_after":                             ListObjectsV2_start_after,
+		"ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token,
+		"ListObjectsV2_start_after_not_in_list":                 ListObjectsV2_start_after_not_in_list,
+		"ListObjectsV2_start_after_empty_result":                ListObjectsV2_start_after_empty_result,
+		"DeleteObject_non_existing_object":                      DeleteObject_non_existing_object,
+		"DeleteObject_success":                                  DeleteObject_success,
+		"DeleteObject_success_status_code":                      DeleteObject_success_status_code,
+		"DeleteObjects_empty_input":                             DeleteObjects_empty_input,
+		"DeleteObjects_non_existing_objects":                    DeleteObjects_non_existing_objects,
+		"DeleteObjects_success":                                 DeleteObjects_success,
+		"CopyObject_non_existing_dst_bucket":                    CopyObject_non_existing_dst_bucket,
+		"CopyObject_not_owned_source_bucket":                    CopyObject_not_owned_source_bucket,
+		"CopyObject_copy_to_itself":                             CopyObject_copy_to_itself,
+		"CopyObject_to_itself_with_new_metadata":                CopyObject_to_itself_with_new_metadata,
+		"CopyObject_success":                                    CopyObject_success,
+		"PutObjectTagging_non_existing_object":                  PutObjectTagging_non_existing_object,
+		"PutObjectTagging_long_tags":                            PutObjectTagging_long_tags,
+		"PutObjectTagging_success":                              PutObjectTagging_success,
+		"GetObjectTagging_non_existing_object":                  GetObjectTagging_non_existing_object,
+		"GetObjectTagging_success":                              GetObjectTagging_success,
+		"DeleteObjectTagging_non_existing_object":               DeleteObjectTagging_non_existing_object,
+		"DeleteObjectTagging_success_status":                    DeleteObjectTagging_success_status,
+		"DeleteObjectTagging_success":                           DeleteObjectTagging_success,
+		"CreateMultipartUpload_non_existing_bucket":             CreateMultipartUpload_non_existing_bucket,
+		"CreateMultipartUpload_success":                         CreateMultipartUpload_success,
+		"UploadPart_non_existing_bucket":                        UploadPart_non_existing_bucket,
+		"UploadPart_invalid_part_number":                        UploadPart_invalid_part_number,
+		"UploadPart_non_existing_key":                           UploadPart_non_existing_key,
+		"UploadPart_non_existing_mp_upload":                     UploadPart_non_existing_mp_upload,
+		"UploadPart_success":                                    UploadPart_success,
+		"UploadPartCopy_non_existing_bucket":                    UploadPartCopy_non_existing_bucket,
+		"UploadPartCopy_incorrect_uploadId":                     UploadPartCopy_incorrect_uploadId,
+		"UploadPartCopy_incorrect_object_key":                   UploadPartCopy_incorrect_object_key,
+		"UploadPartCopy_invalid_part_number":                    UploadPartCopy_invalid_part_number,
+		"UploadPartCopy_invalid_copy_source":                    UploadPartCopy_invalid_copy_source,
+		"UploadPartCopy_non_existing_source_bucket":             UploadPartCopy_non_existing_source_bucket,
+		"UploadPartCopy_non_existing_source_object_key":         UploadPartCopy_non_existing_source_object_key,
+		"UploadPartCopy_success":                                UploadPartCopy_success,
+		"UploadPartCopy_by_range_invalid_range":                 UploadPartCopy_by_range_invalid_range,
+		"UploadPartCopy_greater_range_than_obj_size":            UploadPartCopy_greater_range_than_obj_size,
+		"UploadPartCopy_by_range_success":                       UploadPartCopy_by_range_success,
+		"ListParts_incorrect_uploadId":                          ListParts_incorrect_uploadId,
+		"ListParts_incorrect_object_key":                        ListParts_incorrect_object_key,
+		"ListParts_success":                                     ListParts_success,
+		"ListMultipartUploads_non_existing_bucket":              ListMultipartUploads_non_existing_bucket,
+		"ListMultipartUploads_empty_result":                     ListMultipartUploads_empty_result,
+		"ListMultipartUploads_invalid_max_uploads":              ListMultipartUploads_invalid_max_uploads,
+		"ListMultipartUploads_max_uploads":                      ListMultipartUploads_max_uploads,
+		"ListMultipartUploads_incorrect_next_key_marker":        ListMultipartUploads_incorrect_next_key_marker,
+		"ListMultipartUploads_ignore_upload_id_marker":          ListMultipartUploads_ignore_upload_id_marker,
+		"ListMultipartUploads_success":                          ListMultipartUploads_success,
+		"AbortMultipartUpload_non_existing_bucket":              AbortMultipartUpload_non_existing_bucket,
+		"AbortMultipartUpload_incorrect_uploadId":               AbortMultipartUpload_incorrect_uploadId,
+		"AbortMultipartUpload_incorrect_object_key":             AbortMultipartUpload_incorrect_object_key,
+		"AbortMultipartUpload_success":                          AbortMultipartUpload_success,
+		"AbortMultipartUpload_success_status_code":              AbortMultipartUpload_success_status_code,
+		"CompletedMultipartUpload_non_existing_bucket":          CompletedMultipartUpload_non_existing_bucket,
+		"CompleteMultipartUpload_invalid_part_number":           CompleteMultipartUpload_invalid_part_number,
+		"CompleteMultipartUpload_invalid_ETag":                  CompleteMultipartUpload_invalid_ETag,
+		"CompleteMultipartUpload_success":                       CompleteMultipartUpload_success,
+		"PutBucketAcl_non_existing_bucket":                      PutBucketAcl_non_existing_bucket,
+		"PutBucketAcl_invalid_acl_canned_and_acp":               PutBucketAcl_invalid_acl_canned_and_acp,
+		"PutBucketAcl_invalid_acl_canned_and_grants":            PutBucketAcl_invalid_acl_canned_and_grants,
+		"PutBucketAcl_invalid_acl_acp_and_grants":               PutBucketAcl_invalid_acl_acp_and_grants,
+		"PutBucketAcl_invalid_owner":                            PutBucketAcl_invalid_owner,
+		"PutBucketAcl_success_access_denied":                    PutBucketAcl_success_access_denied,
+		"PutBucketAcl_success_grants":                           PutBucketAcl_success_grants,
+		"PutBucketAcl_success_canned_acl":                       PutBucketAcl_success_canned_acl,
+		"PutBucketAcl_success_acp":                              PutBucketAcl_success_acp,
+		"GetBucketAcl_non_existing_bucket":                      GetBucketAcl_non_existing_bucket,
+		"GetBucketAcl_access_denied":                            GetBucketAcl_access_denied,
+		"GetBucketAcl_success":                                  GetBucketAcl_success,
+		"PutObject_overwrite_dir_obj":                           PutObject_overwrite_dir_obj,
+		"PutObject_overwrite_file_obj":                          PutObject_overwrite_file_obj,
+		"PutObject_dir_obj_with_data":                           PutObject_dir_obj_with_data,
+		"CreateMultipartUpload_dir_obj":                         CreateMultipartUpload_dir_obj,
+		"IAM_user_access_denied":                                IAM_user_access_denied,
+		"IAM_userplus_access_denied":                            IAM_userplus_access_denied,
+		"IAM_userplus_CreateBucket":                             IAM_userplus_CreateBucket,
+		"IAM_admin_ChangeBucketOwner":                           IAM_admin_ChangeBucketOwner,
+	}
+}

integration/output.go

@@ -0,0 +1,31 @@
+package integration
+
+import "fmt"
+
+var (
+	colorReset = "\033[0m"
+	colorRed   = "\033[31m"
+	colorGreen = "\033[32m"
+	colorCyan  = "\033[36m"
+)
+
+var (
+	RunCount  = 0
+	PassCount = 0
+	FailCount = 0
+)
+
+func runF(format string, a ...interface{}) {
+	RunCount++
+	fmt.Printf(colorCyan+"RUN  "+colorReset+format+"\n", a...)
+}
+
+func failF(format string, a ...interface{}) {
+	FailCount++
+	fmt.Printf(colorRed+"FAIL "+colorReset+format+"\n", a...)
+}
+
+func passF(format string, a ...interface{}) {
+	PassCount++
+	fmt.Printf(colorGreen+"PASS "+colorReset+format+"\n", a...)
+}

integration/s3conf.go

@@ -1,22 +1,7 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 package integration
 
 import (
 	"context"
-	"crypto/tls"
 	"io"
 	"log"
 	"net/http"
@@ -32,19 +17,15 @@ import (
 )
 
 type S3Conf struct {
-	awsID             string
-	awsSecret         string
-	awsRegion         string
-	endpoint          string
-	checksumDisable   bool
-	pathStyle         bool
-	PartSize          int64
-	Concurrency       int
-	debug             bool
-	versioningEnabled bool
-	azureTests        bool
-	tlsStatus         bool
-	httpClient        *http.Client
+	awsID           string
+	awsSecret       string
+	awsRegion       string
+	endpoint        string
+	checksumDisable bool
+	pathStyle       bool
+	PartSize        int64
+	Concurrency     int
+	debug           bool
 }
 
 func NewS3Conf(opts ...Option) *S3Conf {
@@ -53,20 +34,6 @@ func NewS3Conf(opts ...Option) *S3Conf {
 	for _, opt := range opts {
 		opt(s)
 	}
-
-	customTransport := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: s.tlsStatus,
-		},
-	}
-
-	customHTTPClient := &http.Client{
-		Transport: customTransport,
-		Timeout:   shortTimeout,
-	}
-
-	s.httpClient = customHTTPClient
-
 	return s
 }
 
@@ -99,15 +66,6 @@ func WithConcurrency(c int) Option {
 func WithDebug() Option {
 	return func(s *S3Conf) { s.debug = true }
 }
-func WithVersioningEnabled() Option {
-	return func(s *S3Conf) { s.versioningEnabled = true }
-}
-func WithAzureMode() Option {
-	return func(s *S3Conf) { s.azureTests = true }
-}
-func WithTLSStatus(ts bool) Option {
-	return func(s *S3Conf) { s.tlsStatus = ts }
-}
 
 func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
 	// TODO support token/IAM
@@ -121,8 +79,13 @@ func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
 	return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "")
 }
 
-func (c *S3Conf) GetClient() *s3.Client {
-	return s3.NewFromConfig(c.Config())
+func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               c.endpoint,
+		SigningRegion:     c.awsRegion,
+		HostnameImmutable: true,
+	}, nil
 }
 
 func (c *S3Conf) Config() aws.Config {
@@ -135,10 +98,12 @@ func (c *S3Conf) Config() aws.Config {
 		config.WithRegion(c.awsRegion),
 		config.WithCredentialsProvider(creds),
 		config.WithHTTPClient(client),
-		config.WithRetryMaxAttempts(1),
 	}
 
-	opts = append(opts, config.WithHTTPClient(c.httpClient))
+	if c.endpoint != "" && c.endpoint != "aws" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(c))
+	}
 
 	if c.checksumDisable {
 		opts = append(opts,
@@ -156,15 +121,11 @@ func (c *S3Conf) Config() aws.Config {
 		log.Fatalln("error:", err)
 	}
 
-	if c.endpoint != "" && c.endpoint != "aws" {
-		cfg.BaseEndpoint = &c.endpoint
-	}
-
 	return cfg
 }
 
 func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
-	uploader := manager.NewUploader(c.GetClient())
+	uploader := manager.NewUploader(s3.NewFromConfig(c.Config()))
 	uploader.PartSize = c.PartSize
 	uploader.Concurrency = c.Concurrency
 
@@ -179,7 +140,7 @@ func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
 }
 
 func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
-	downloader := manager.NewDownloader(c.GetClient())
+	downloader := manager.NewDownloader(s3.NewFromConfig(c.Config()))
 	downloader.PartSize = c.PartSize
 	downloader.Concurrency = c.Concurrency
 
@@ -190,11 +151,3 @@ func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, erro
 
 	return downloader.Download(context.Background(), w, downinfo)
 }
-
-func (c *S3Conf) getAdminCommand(args ...string) []string {
-	if c.tlsStatus {
-		return append([]string{"admin", "--allow-insecure"}, args...)
-	}
-
-	return append([]string{"admin"}, args...)
-}

integration/utils.go

@@ -0,0 +1,605 @@
+package integration
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	rnd "math/rand"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+var (
+	bcktCount            = 0
+	succUsrCrt           = "The user has been created successfully"
+	failUsrCrt           = "failed to create a user: update iam data: account already exists"
+	adminAccessDeniedMsg = "access denied: only admin users have access to this resource"
+	succDeleteUserMsg    = "The user has been deleted successfully"
+)
+
+func getBucketName() string {
+	bcktCount++
+	return fmt.Sprintf("test-bucket-%v", bcktCount)
+}
+
+func setup(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func teardown(s *S3Conf, bucket string) error {
+	s3client := s3.NewFromConfig(s.Config())
+
+	deleteObject := func(bucket, key, versionId *string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
+			Bucket:    bucket,
+			Key:       key,
+			VersionId: versionId,
+		})
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to delete object %v: %w", *key, err)
+		}
+		return nil
+	}
+
+	in := &s3.ListObjectsV2Input{Bucket: &bucket}
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListObjectsV2(ctx, in)
+		cancel()
+		if err != nil {
+			return fmt.Errorf("failed to list objects: %w", err)
+		}
+
+		for _, item := range out.Contents {
+			err = deleteObject(&bucket, item.Key, nil)
+			if err != nil {
+				return err
+			}
+		}
+
+		if out.IsTruncated != nil && *out.IsTruncated {
+			in.ContinuationToken = out.ContinuationToken
+		} else {
+			break
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: &bucket,
+	})
+	cancel()
+	return err
+}
+
+func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) error {
+	runF(testName)
+	bucketName := getBucketName()
+	err := setup(s, bucketName)
+	if err != nil {
+		failF("%v: failed to create a bucket: %v", testName, err)
+		return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
+	}
+	client := s3.NewFromConfig(s.Config())
+	handlerErr := handler(client, bucketName)
+	if handlerErr != nil {
+		failF("%v: %v", testName, handlerErr)
+	}
+
+	err = teardown(s, bucketName)
+	if err != nil {
+		fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
+		if handlerErr == nil {
+			return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
+		}
+	}
+	if handlerErr == nil {
+		passF(testName)
+	}
+
+	return handlerErr
+}
+
+type authConfig struct {
+	testName string
+	path     string
+	method   string
+	body     []byte
+	service  string
+	date     time.Time
+}
+
+func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
+	runF(cfg.testName)
+	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err)
+		return fmt.Errorf("%v: %w", cfg.testName, err)
+	}
+
+	err = handler(req)
+	if err != nil {
+		failF("%v: %v", cfg.testName, err)
+		return fmt.Errorf("%v: %w", cfg.testName, err)
+	}
+	passF(cfg.testName)
+	return nil
+}
+
+func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error {
+	runF(testName)
+	clt := s3.NewPresignClient(s3.NewFromConfig(s.Config()))
+
+	err := handler(clt)
+	if err != nil {
+		failF("%v: %v", testName, err)
+		return fmt.Errorf("%v: %w", testName, err)
+	}
+
+	passF(testName)
+	return nil
+}
+
+func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
+	req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(body)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date)
+	if signErr != nil {
+		return nil, fmt.Errorf("failed to sign the request: %w", signErr)
+	}
+
+	return req, nil
+}
+
+func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	var errResp s3err.APIErrorResponse
+	err = xml.Unmarshal(body, &errResp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != apiErr.HTTPStatusCode {
+		return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
+	}
+	if errResp.Code != apiErr.Code {
+		return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
+	}
+	if errResp.Message != apiErr.Description {
+		return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
+	}
+
+	return nil
+}
+
+func checkApiErr(err error, apiErr s3err.APIError) error {
+	if err == nil {
+		return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
+			return nil
+		}
+
+		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+	}
+	return fmt.Errorf("expected aws api error, instead got: %w", err)
+}
+
+func checkSdkApiErr(err error, code string) error {
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		if ae.ErrorCode() != code {
+			return fmt.Errorf("expected %v, instead got %v", code, ae.ErrorCode())
+		}
+		return nil
+	}
+	return err
+}
+
+func putObjects(client *s3.Client, objs []string, bucket string) error {
+	for _, key := range objs {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Key:    &key,
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
+	data = make([]byte, lgth)
+	rand.Read(data)
+	csum = sha256.Sum256(data)
+	r := bytes.NewReader(data)
+	input.Body = r
+
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err = client.PutObject(ctx, input)
+	cancel()
+
+	return
+}
+
+func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: &bucket,
+		Key:    &key,
+	})
+	cancel()
+	return out, err
+}
+
+func isEqual(a, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i, d := range a {
+		if d != b[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+	for i, item := range list1 {
+		if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareParts(parts1, parts2 []types.Part) bool {
+	if len(parts1) != len(parts2) {
+		return false
+	}
+
+	for i, prt := range parts1 {
+		if *prt.PartNumber != *parts2[i].PartNumber {
+			return false
+		}
+		if *prt.ETag != *parts2[i].ETag {
+			return false
+		}
+	}
+	return true
+}
+
+func areTagsSame(tags1, tags2 []types.Tag) bool {
+	if len(tags1) != len(tags2) {
+		return false
+	}
+
+	for _, tag := range tags1 {
+		if !containsTag(tag, tags2) {
+			return false
+		}
+	}
+	return true
+}
+
+func containsTag(tag types.Tag, list []types.Tag) bool {
+	for _, item := range list {
+		if *item.Key == *tag.Key && *item.Value == *tag.Value {
+			return true
+		}
+	}
+	return false
+}
+
+func compareGrants(grts1, grts2 []types.Grant) bool {
+	if len(grts1) != len(grts2) {
+		return false
+	}
+
+	for i, grt := range grts1 {
+		if grt.Permission != grts2[i].Permission {
+			return false
+		}
+		if *grt.Grantee.ID != *grts2[i].Grantee.ID {
+			return false
+		}
+	}
+	return true
+}
+
+func execCommand(args ...string) ([]byte, error) {
+	cmd := exec.Command("./versitygw", args...)
+
+	return cmd.CombinedOutput()
+}
+
+func getString(str *string) string {
+	if str == nil {
+		return ""
+	}
+	return *str
+}
+
+func getPtr(str string) *string {
+	return &str
+}
+
+func areMapsSame(mp1, mp2 map[string]string) bool {
+	if len(mp1) != len(mp2) {
+		return false
+	}
+	for key, val := range mp1 {
+		if mp2[key] != val {
+			return false
+		}
+	}
+	return true
+}
+
+func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[*elem.Name] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[elem.Name]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareObjects(list1 []string, list2 []types.Object) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Prefix]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
+	if len(list1) != len(list2) {
+		return false
+	}
+
+	elementMap := make(map[string]bool)
+
+	for _, elem := range list1 {
+		elementMap[elem] = true
+	}
+
+	for _, elem := range list2 {
+		if _, found := elementMap[*elem.Key]; !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
+	dr := NewDataReader(size, size)
+	datafile := "rand.data"
+	w, err := os.Create(datafile)
+	if err != nil {
+		return parts, err
+	}
+	defer w.Close()
+
+	_, err = io.Copy(w, dr)
+	if err != nil {
+		return parts, err
+	}
+
+	fileInfo, err := w.Stat()
+	if err != nil {
+		return parts, err
+	}
+
+	partSize := fileInfo.Size() / int64(partCount)
+	var offset int64
+
+	for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
+		partStart := (partNumber - 1) * partSize
+		partEnd := partStart + partSize - 1
+		if partEnd > fileInfo.Size()-1 {
+			partEnd = fileInfo.Size() - 1
+		}
+		partBuffer := make([]byte, partEnd-partStart+1)
+		_, err := w.ReadAt(partBuffer, partStart)
+		if err != nil {
+			return parts, err
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		pn := int32(partNumber)
+		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     &bucket,
+			Key:        &key,
+			UploadId:   &uploadId,
+			Body:       bytes.NewReader(partBuffer),
+			PartNumber: &pn,
+		})
+		cancel()
+		if err != nil {
+			return parts, err
+		}
+		parts = append(parts, types.Part{
+			ETag:       out.ETag,
+			PartNumber: &pn,
+		})
+		offset += partSize
+	}
+
+	return parts, err
+}
+
+type user struct {
+	access string
+	secret string
+	role   string
+}
+
+func createUsers(s *S3Conf, users []user) error {
+	for _, usr := range users {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
+			return fmt.Errorf("failed to create a user account")
+		}
+	}
+	return nil
+}
+
+func deleteUser(s *S3Conf, access string) error {
+	out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "delete-user", "-a", access)
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(string(out), succDeleteUserMsg) {
+		return fmt.Errorf("failed to delete the user account")
+	}
+
+	return nil
+}
+
+func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
+	for _, bucket := range buckets {
+		out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
+			return fmt.Errorf(string(out))
+		}
+	}
+
+	return nil
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func genRandString(length int) string {
+	source := rnd.NewSource(time.Now().UnixNano())
+	random := rnd.New(source)
+	result := make([]byte, length)
+	for i := range result {
+		result[i] = charset[random.Intn(len(charset))]
+	}
+	return string(result)
+}
+
+const (
+	credAccess int = iota
+	credDate
+	credRegion
+	credService
+	credTerminator
+)
+
+func changeAuthCred(uri, newVal string, index int) (string, error) {
+	urlParsed, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+
+	queries := urlParsed.Query()
+	creds := strings.Split(queries.Get("X-Amz-Credential"), "/")
+	creds[index] = newVal
+	queries.Set("X-Amz-Credential", strings.Join(creds, "/"))
+	urlParsed.RawQuery = queries.Encode()
+
+	return urlParsed.String(), nil
+}

metrics/actions.go

@@ -1,284 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package metrics
-
-type Action struct {
-	Name    string
-	Service string
-}
-
-var (
-	ActionMap map[string]Action
-)
-
-var (
-	ActionUndetected                    = "ActionUnDetected"
-	ActionAbortMultipartUpload          = "s3_AbortMultipartUpload"
-	ActionCompleteMultipartUpload       = "s3_CompleteMultipartUpload"
-	ActionCopyObject                    = "s3_CopyObject"
-	ActionCreateBucket                  = "s3_CreateBucket"
-	ActionCreateMultipartUpload         = "s3_CreateMultipartUpload"
-	ActionDeleteBucket                  = "s3_DeleteBucket"
-	ActionDeleteBucketPolicy            = "s3_DeleteBucketPolicy"
-	ActionDeleteBucketTagging           = "s3_DeleteBucketTagging"
-	ActionDeleteObject                  = "s3_DeleteObject"
-	ActionDeleteObjectTagging           = "s3_DeleteObjectTagging"
-	ActionDeleteObjects                 = "s3_DeleteObjects"
-	ActionGetBucketAcl                  = "s3_GetBucketAcl"
-	ActionGetBucketPolicy               = "s3_GetBucketPolicy"
-	ActionGetBucketTagging              = "s3_GetBucketTagging"
-	ActionGetBucketVersioning           = "s3_GetBucketVersioning"
-	ActionGetObject                     = "s3_GetObject"
-	ActionGetObjectAcl                  = "s3_GetObjectAcl"
-	ActionGetObjectAttributes           = "s3_GetObjectAttributes"
-	ActionGetObjectLegalHold            = "s3_GetObjectLegalHold"
-	ActionGetObjectLockConfiguration    = "s3_GetObjectLockConfiguration"
-	ActionGetObjectRetention            = "s3_GetObjectRetention"
-	ActionGetObjectTagging              = "s3_GetObjectTagging"
-	ActionHeadBucket                    = "s3_HeadBucket"
-	ActionHeadObject                    = "s3_HeadObject"
-	ActionListAllMyBuckets              = "s3_ListAllMyBuckets"
-	ActionListMultipartUploads          = "s3_ListMultipartUploads"
-	ActionListObjectVersions            = "s3_ListObjectVersions"
-	ActionListObjects                   = "s3_ListObjects"
-	ActionListObjectsV2                 = "s3_ListObjectsV2"
-	ActionListParts                     = "s3_ListParts"
-	ActionPutBucketAcl                  = "s3_PutBucketAcl"
-	ActionPutBucketPolicy               = "s3_PutBucketPolicy"
-	ActionPutBucketTagging              = "s3_PutBucketTagging"
-	ActionPutBucketVersioning           = "s3_PutBucketVersioning"
-	ActionPutObject                     = "s3_PutObject"
-	ActionPutObjectAcl                  = "s3_PutObjectAcl"
-	ActionPutObjectLegalHold            = "s3_PutObjectLegalHold"
-	ActionPutObjectLockConfiguration    = "s3_PutObjectLockConfiguration"
-	ActionPutObjectRetention            = "s3_PutObjectRetention"
-	ActionPutObjectTagging              = "s3_PutObjectTagging"
-	ActionRestoreObject                 = "s3_RestoreObject"
-	ActionSelectObjectContent           = "s3_SelectObjectContent"
-	ActionUploadPart                    = "s3_UploadPart"
-	ActionUploadPartCopy                = "s3_UploadPartCopy"
-	ActionPutBucketOwnershipControls    = "s3_PutBucketOwnershipControls"
-	ActionGetBucketOwnershipControls    = "s3_GetBucketOwnershipControls"
-	ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
-	ActionPutBucketCors                 = "s3_PutBucketCors"
-	ActionGetBucketCors                 = "s3_GetBucketCors"
-	ActionDeleteBucketCors              = "s3_DeleteBucketCors"
-
-	// Admin actions
-	ActionAdminCreateUser        = "admin_CreateUser"
-	ActionAdminUpdateUser        = "admin_UpdateUser"
-	ActionAdminDeleteUser        = "admin_DeleteUser"
-	ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
-	ActionAdminListUsers         = "admin_ListUsers"
-	ActionAdminListBuckets       = "admin_ListBuckets"
-)
-
-func init() {
-	ActionMap = make(map[string]Action)
-
-	ActionMap[ActionUndetected] = Action{
-		Name:    "ActionUnDetected",
-		Service: "unknown",
-	}
-
-	ActionMap[ActionAbortMultipartUpload] = Action{
-		Name:    "AbortMultipartUpload",
-		Service: "s3",
-	}
-	ActionMap[ActionCompleteMultipartUpload] = Action{
-		Name:    "CompleteMultipartUpload",
-		Service: "s3",
-	}
-	ActionMap[ActionCopyObject] = Action{
-		Name:    "CopyObject",
-		Service: "s3",
-	}
-	ActionMap[ActionCreateBucket] = Action{
-		Name:    "CreateBucket",
-		Service: "s3",
-	}
-	ActionMap[ActionCreateMultipartUpload] = Action{
-		Name:    "CreateMultipartUpload",
-		Service: "s3",
-	}
-	ActionMap[ActionDeleteBucket] = Action{
-		Name:    "DeleteBucket",
-		Service: "s3",
-	}
-	ActionMap[ActionDeleteBucketPolicy] = Action{
-		Name:    "DeleteBucketPolicy",
-		Service: "s3",
-	}
-	ActionMap[ActionDeleteBucketTagging] = Action{
-		Name:    "DeleteBucketTagging",
-		Service: "s3",
-	}
-	ActionMap[ActionDeleteObject] = Action{
-		Name:    "DeleteObject",
-		Service: "s3",
-	}
-	ActionMap[ActionDeleteObjectTagging] = Action{
-		Name:    "DeleteObjectTagging",
-		Service: "s3",
-	}
-	ActionMap[ActionDeleteObjects] = Action{
-		Name:    "DeleteObjects",
-		Service: "s3",
-	}
-	ActionMap[ActionGetBucketAcl] = Action{
-		Name:    "GetBucketAcl",
-		Service: "s3",
-	}
-	ActionMap[ActionGetBucketPolicy] = Action{
-		Name:    "GetBucketPolicy",
-		Service: "s3",
-	}
-	ActionMap[ActionGetBucketTagging] = Action{
-		Name:    "GetBucketTagging",
-		Service: "s3",
-	}
-	ActionMap[ActionGetBucketVersioning] = Action{
-		Name:    "GetBucketVersioning",
-		Service: "s3",
-	}
-	ActionMap[ActionGetObject] = Action{
-		Name:    "GetObject",
-		Service: "s3",
-	}
-	ActionMap[ActionGetObjectAcl] = Action{
-		Name:    "GetObjectAcl",
-		Service: "s3",
-	}
-	ActionMap[ActionGetObjectAttributes] = Action{
-		Name:    "GetObjectAttributes",
-		Service: "s3",
-	}
-	ActionMap[ActionGetObjectLegalHold] = Action{
-		Name:    "GetObjectLegalHold",
-		Service: "s3",
-	}
-	ActionMap[ActionGetObjectLockConfiguration] = Action{
-		Name:    "GetObjectLockConfiguration",
-		Service: "s3",
-	}
-	ActionMap[ActionGetObjectRetention] = Action{
-		Name:    "GetObjectRetention",
-		Service: "s3",
-	}
-	ActionMap[ActionGetObjectTagging] = Action{
-		Name:    "GetObjectTagging",
-		Service: "s3",
-	}
-	ActionMap[ActionHeadBucket] = Action{
-		Name:    "HeadBucket",
-		Service: "s3",
-	}
-	ActionMap[ActionHeadObject] = Action{
-		Name:    "HeadObject",
-		Service: "s3",
-	}
-	ActionMap[ActionListAllMyBuckets] = Action{
-		Name:    "ListAllMyBuckets",
-		Service: "s3",
-	}
-	ActionMap[ActionListMultipartUploads] = Action{
-		Name:    "ListMultipartUploads",
-		Service: "s3",
-	}
-	ActionMap[ActionListObjectVersions] = Action{
-		Name:    "ListObjectVersions",
-		Service: "s3",
-	}
-	ActionMap[ActionListObjects] = Action{
-		Name:    "ListObjects",
-		Service: "s3",
-	}
-	ActionMap[ActionListObjectsV2] = Action{
-		Name:    "ListObjectsV2",
-		Service: "s3",
-	}
-	ActionMap[ActionListParts] = Action{
-		Name:    "ListParts",
-		Service: "s3",
-	}
-	ActionMap[ActionPutBucketAcl] = Action{
-		Name:    "PutBucketAcl",
-		Service: "s3",
-	}
-	ActionMap[ActionPutBucketPolicy] = Action{
-		Name:    "PutBucketPolicy",
-		Service: "s3",
-	}
-	ActionMap[ActionPutBucketTagging] = Action{
-		Name:    "PutBucketTagging",
-		Service: "s3",
-	}
-	ActionMap[ActionPutBucketVersioning] = Action{
-		Name:    "PutBucketVersioning",
-		Service: "s3",
-	}
-	ActionMap[ActionPutObject] = Action{
-		Name:    "PutObject",
-		Service: "s3",
-	}
-	ActionMap[ActionPutObjectAcl] = Action{
-		Name:    "PutObjectAcl",
-		Service: "s3",
-	}
-	ActionMap[ActionPutObjectLegalHold] = Action{
-		Name:    "PutObjectLegalHold",
-		Service: "s3",
-	}
-	ActionMap[ActionPutObjectLockConfiguration] = Action{
-		Name:    "PutObjectLockConfiguration",
-		Service: "s3",
-	}
-	ActionMap[ActionPutObjectRetention] = Action{
-		Name:    "PutObjectRetention",
-		Service: "s3",
-	}
-	ActionMap[ActionPutObjectTagging] = Action{
-		Name:    "PutObjectTagging",
-		Service: "s3",
-	}
-	ActionMap[ActionRestoreObject] = Action{
-		Name:    "RestoreObject",
-		Service: "s3",
-	}
-	ActionMap[ActionSelectObjectContent] = Action{
-		Name:    "SelectObjectContent",
-		Service: "s3",
-	}
-	ActionMap[ActionUploadPart] = Action{
-		Name:    "UploadPart",
-		Service: "s3",
-	}
-	ActionMap[ActionUploadPartCopy] = Action{
-		Name:    "UploadPartCopy",
-		Service: "s3",
-	}
-	ActionMap[ActionPutBucketCors] = Action{
-		Name:    "PutBucketCors",
-		Service: "s3",
-	}
-	ActionMap[ActionGetBucketCors] = Action{
-		Name:    "GetBucketCors",
-		Service: "s3",
-	}
-	ActionMap[ActionDeleteBucketCors] = Action{
-		Name:    "DeleteBucketCors",
-		Service: "s3",
-	}
-}

metrics/dogstats.go

@@ -1,65 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package metrics
-
-import (
-	"fmt"
-
-	dogstats "github.com/DataDog/datadog-go/v5/statsd"
-)
-
-// vgwDogStatsd metrics type
-type vgwDogStatsd struct {
-	c *dogstats.Client
-}
-
-var (
-	rateSampleAlways = 1.0
-)
-
-// newDogStatsd takes a server address and returns a statsd merics
-func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
-	c, err := dogstats.New(server,
-		dogstats.WithMaxMessagesPerPayload(1000),
-		dogstats.WithNamespace("versitygw"),
-		dogstats.WithTags([]string{
-			"service:" + service,
-		}))
-	if err != nil {
-		return nil, err
-	}
-	return &vgwDogStatsd{c: c}, nil
-}
-
-// Close closes statsd connections
-func (s *vgwDogStatsd) Close() {
-	s.c.Close()
-}
-
-func (t Tag) ddString() string {
-	if t.Value == "" {
-		return t.Key
-	}
-	return fmt.Sprintf("%v:%v", t.Key, t.Value)
-}
-
-// Add adds value to key
-func (s *vgwDogStatsd) Add(key string, value int64, tags ...Tag) {
-	stags := make([]string, len(tags))
-	for i, t := range tags {
-		stags[i] = t.ddString()
-	}
-	s.c.Count(key, value, stags, rateSampleAlways)
-}

metrics/metrics.go

@@ -1,225 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package metrics
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/versity/versitygw/s3err"
-)
-
-var (
-	// max size of data items to buffer before dropping
-	// new incoming data items
-	dataItemCount = 100000
-)
-
-// Tag is added metadata for metrics
-type Tag struct {
-	// Key is tag name
-	Key string
-	// Value is tag data
-	Value string
-}
-
-// Manager is a manager of metrics plugins
-type Manager struct {
-	wg  sync.WaitGroup
-	ctx context.Context
-
-	config Config
-
-	publishers  []publisher
-	addDataChan chan datapoint
-}
-
-type Config struct {
-	ServiceName      string
-	StatsdServers    string
-	DogStatsdServers string
-}
-
-// NewManager initializes metrics plugins and returns a new metrics manager
-func NewManager(ctx context.Context, conf Config) (*Manager, error) {
-	if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
-		return nil, nil
-	}
-
-	if conf.ServiceName == "" {
-		hostname, err := os.Hostname()
-		if err != nil {
-			return nil, fmt.Errorf("failed to get hostname: %w", err)
-		}
-		conf.ServiceName = hostname
-	}
-
-	addDataChan := make(chan datapoint, dataItemCount)
-
-	mgr := &Manager{
-		addDataChan: addDataChan,
-		ctx:         ctx,
-		config:      conf,
-	}
-
-	// setup statsd endpoints
-	if len(conf.StatsdServers) > 0 {
-		statsdServers := strings.Split(conf.StatsdServers, ",")
-
-		for _, server := range statsdServers {
-			statsd, err := newStatsd(server, conf.ServiceName)
-			if err != nil {
-				return nil, err
-			}
-			mgr.publishers = append(mgr.publishers, statsd)
-		}
-	}
-
-	// setup dogstatsd endpoints
-	if len(conf.DogStatsdServers) > 0 {
-		dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
-
-		for _, server := range dogStatsdServers {
-			dogStatsd, err := newDogStatsd(server, conf.ServiceName)
-			if err != nil {
-				return nil, err
-			}
-			mgr.publishers = append(mgr.publishers, dogStatsd)
-		}
-	}
-
-	mgr.wg.Add(1)
-	go mgr.addForwarder(addDataChan)
-
-	return mgr, nil
-}
-
-func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) {
-	// In case of Authentication failures, url parsing ...
-	if action == "" {
-		action = ActionUndetected
-	}
-
-	a := ActionMap[action]
-	reqTags := []Tag{
-		{Key: "method", Value: ctx.Method()},
-		{Key: "api", Value: a.Service},
-		{Key: "action", Value: a.Name},
-	}
-
-	reqStatus := status
-
-	if err != nil {
-		var apierr s3err.APIError
-		if errors.As(err, &apierr) {
-			reqStatus = apierr.HTTPStatusCode
-		} else {
-			reqStatus = http.StatusInternalServerError
-		}
-	}
-	if reqStatus == 0 {
-		reqStatus = http.StatusOK
-	}
-
-	reqTags = append(reqTags, Tag{
-		Key:   "status",
-		Value: fmt.Sprintf("%v", reqStatus),
-	})
-
-	if err != nil {
-		m.increment("failed_count", reqTags...)
-	} else {
-		m.increment("success_count", reqTags...)
-	}
-
-	switch action {
-	case ActionPutObject:
-		m.add("bytes_written", count, reqTags...)
-		m.increment("object_created_count", reqTags...)
-	case ActionCompleteMultipartUpload:
-		m.increment("object_created_count", reqTags...)
-	case ActionUploadPart:
-		m.add("bytes_written", count, reqTags...)
-	case ActionGetObject:
-		m.add("bytes_read", count, reqTags...)
-	case ActionDeleteObject:
-		m.increment("object_removed_count", reqTags...)
-	case ActionDeleteObjects:
-		m.add("object_removed_count", count, reqTags...)
-	}
-}
-
-// increment increments the key by one
-func (m *Manager) increment(key string, tags ...Tag) {
-	m.add(key, 1, tags...)
-}
-
-// add adds value to key
-func (m *Manager) add(key string, value int64, tags ...Tag) {
-	if m.ctx.Err() != nil {
-		return
-	}
-
-	d := datapoint{
-		key:   key,
-		value: value,
-		tags:  tags,
-	}
-
-	select {
-	case m.addDataChan <- d:
-	default:
-		// channel full, drop the updates
-	}
-}
-
-// Close closes metrics channels, waits for data to complete, closes all plugins
-func (m *Manager) Close() {
-	// drain the datapoint channels
-	close(m.addDataChan)
-	m.wg.Wait()
-
-	// close all publishers
-	for _, p := range m.publishers {
-		p.Close()
-	}
-}
-
-// publisher is the interface for interacting with the metrics plugins
-type publisher interface {
-	Add(key string, value int64, tags ...Tag)
-	Close()
-}
-
-func (m *Manager) addForwarder(addChan <-chan datapoint) {
-	for data := range addChan {
-		for _, s := range m.publishers {
-			s.Add(data.key, data.value, data.tags...)
-		}
-	}
-	m.wg.Done()
-}
-
-type datapoint struct {
-	key   string
-	value int64
-	tags  []Tag
-}

metrics/statsd.go

@@ -1,51 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package metrics
-
-import (
-	"github.com/smira/go-statsd"
-)
-
-// vgwStatsd metrics type
-type vgwStatsd struct {
-	c *statsd.Client
-}
-
-// newStatsd takes a server address and returns a statsd merics
-// Supply service name to be used as a tag to identify the spcific
-// gateway instance, this may typically be the gateway hostname
-func newStatsd(server string, service string) (*vgwStatsd, error) {
-	c := statsd.NewClient(
-		server,
-		statsd.MetricPrefix("versitygw."),
-		statsd.TagStyle(statsd.TagFormatInfluxDB),
-		statsd.DefaultTags(statsd.StringTag("service", service)),
-	)
-	return &vgwStatsd{c: c}, nil
-}
-
-// Close closes statsd connections
-func (s *vgwStatsd) Close() {
-	s.c.Close()
-}
-
-// Add adds value to key
-func (s *vgwStatsd) Add(key string, value int64, tags ...Tag) {
-	stags := make([]statsd.Tag, len(tags))
-	for i, t := range tags {
-		stags[i] = statsd.StringTag(t.Key, t.Value)
-	}
-	s.c.Incr(key, value, stags...)
-}

runtests.sh

@@ -5,27 +5,15 @@ rm -rf /tmp/gw
 mkdir /tmp/gw
 rm -rf /tmp/covdata
 mkdir /tmp/covdata
-rm -rf /tmp/versioing.covdata
-mkdir /tmp/versioning.covdata
-rm -rf /tmp/versioningdir
-mkdir /tmp/versioningdir
 
-# setup tls certificate and key
-ECHO "Generating TLS certificate and key in the cert.pem and key.pem files"
-
-openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
-openssl req -new -x509 -key key.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-
-
-ECHO "Running the sdk test over http"
-# run server in background not versioning-enabled
-# port: 7070(default)
+# run server in background
 GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
 GW_PID=$!
 
+# wait a second for server to start up
 sleep 1
 
-# check if gateway process is still running
+# check if server is still running
 if ! kill -0 $GW_PID; then
 	echo "server no longer running"
 	exit 1
@@ -51,110 +39,8 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
 	exit 1
 fi
 
+# kill off server
 kill $GW_PID
-
-ECHO "Running the sdk test over https"
-
-# run server in background with TLS certificate
-# port: 7071(default)
-GOCOVERDIR=/tmp/https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7071 -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
-GW_HTTPS_PID=$!
-
-sleep 1
-
-# check if https gateway process is still running
-if ! kill -0 $GW_HTTPS_PID; then
-	echo "server no longer running"
-	exit 1
-fi
-
-# run tests
-# full flow tests
-if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow; then
-	echo "full flow tests failed"
-	kill $GW_HTTPS_PID
-	exit 1
-fi
-# posix tests
-if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 posix; then
-	echo "posix tests failed"
-	kill $GW_HTTPS_PID
-	exit 1
-fi
-# iam tests
-if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 iam; then
-	echo "iam tests failed"
-	kill $GW_HTTPS_PID
-	exit 1
-fi
-
-kill $GW_HTTPS_PID
-
-
-ECHO "Running the sdk test over http against the versioning-enabled gateway"
-# run server in background versioning-enabled
-# port: 7072
-GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7072 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
-GW_VS_PID=$!
-
-# wait a second for server to start up
-sleep 1
-
-# check if versioning-enabled gateway process is still running
-if ! kill -0 $GW_VS_PID; then
-	echo "versioning-enabled server no longer running"
-	exit 1
-fi
-
-# run tests
-# full flow tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs; then
-	echo "versioning-enabled full-flow tests failed"
-	kill $GW_VS_PID
-	exit 1
-fi
-# posix tests
-if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 posix -vs; then
-	echo "versiongin-enabled posix tests failed"
-	kill $GW_VS_PID
-	exit 1
-fi
-
-# kill off server
-kill $GW_VS_PID
-
-ECHO "Running the sdk test over https against the versioning-enabled gateway"
-# run server in background versioning-enabled
-# port: 7073
-GOCOVERDIR=/tmp/versioning.https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7073 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
-GW_VS_HTTPS_PID=$!
-
-# wait a second for server to start up
-sleep 1
-
-# check if versioning-enabled gateway process is still running
-if ! kill -0 $GW_VS_HTTPS_PID; then
-	echo "versioning-enabled server no longer running"
-	exit 1
-fi
-
-# run tests
-# full flow tests
-if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs; then
-	echo "versioning-enabled full-flow tests failed"
-	kill $GW_VS_HTTPS_PID
-	exit 1
-fi
-# posix tests
-if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 posix -vs; then
-	echo "versiongin-enabled posix tests failed"
-	kill $GW_VS_HTTPS_PID
-	exit 1
-fi
-
-# kill off server
-kill $GW_VS_HTTPS_PID
-
 exit 0
 
 # if the above binary was built with -cover enabled (make testbin),

s3api/admin-router.go

@@ -19,13 +19,12 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/controllers"
-	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminRouter struct{}
 
-func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
-	controller := controllers.NewAdminController(iam, be, logger)
+func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
+	controller := controllers.NewAdminController(iam, be)
 
 	// CreateUser admin api
 	app.Patch("/create-user", controller.CreateUser)
@@ -33,9 +32,6 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
 	// DeleteUsers admin api
 	app.Patch("/delete-user", controller.DeleteUser)
 
-	// UpdateUser admin api
-	app.Patch("/update-user", controller.UpdateUser)
-
 	// ListUsers admin api
 	app.Patch("/list-users", controller.ListUsers)
 

s3api/admin-server.go

@@ -22,7 +22,6 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/middlewares"
-	"github.com/versity/versitygw/s3log"
 )
 
 type S3AdminServer struct {
@@ -33,7 +32,7 @@ type S3AdminServer struct {
 	cert    *tls.Certificate
 }
 
-func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
+func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
 	server := &S3AdminServer{
 		app:     app,
 		backend: be,
@@ -47,16 +46,14 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
 
 	// Logging middlewares
 	app.Use(logger.New())
-	app.Use(middlewares.DecodeURL(l, nil))
+	app.Use(middlewares.DecodeURL(nil))
 
 	// Authentication middlewares
-	app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
-	app.Use(middlewares.VerifyMD5Body(l))
+	app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
+	app.Use(middlewares.VerifyMD5Body(nil))
+	app.Use(middlewares.AclParser(be, nil))
 
-	// Admin role checker
-	app.Use(middlewares.IsAdmin(l))
-
-	server.router.Init(app, be, iam, l)
+	server.router.Init(app, be, iam)
 
 	return server
 }

s3api/controllers/admin.go

@@ -16,188 +16,107 @@ package controllers
 
 import (
 	"encoding/json"
-	"encoding/xml"
 	"fmt"
-	"net/http"
-	"strings"
 
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/metrics"
-	"github.com/versity/versitygw/s3err"
-	"github.com/versity/versitygw/s3log"
-	"github.com/versity/versitygw/s3response"
 )
 
 type AdminController struct {
 	iam auth.IAMService
 	be  backend.Backend
-	l   s3log.AuditLogger
 }
 
-func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
-	return AdminController{iam: iam, be: be, l: l}
+func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
+	return AdminController{iam: iam, be: be}
 }
 
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
 	var usr auth.Account
-	err := xml.Unmarshal(ctx.Body(), &usr)
+	err := json.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminCreateUser,
-			})
+		return fmt.Errorf("failed to parse request body: %w", err)
 	}
 
-	if !usr.Role.IsValid() {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminCreateUser,
-			})
+	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
+		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		if strings.Contains(err.Error(), "user already exists") {
-			err = s3err.GetAPIError(s3err.ErrAdminUserExists)
-		}
-
-		return SendResponse(ctx, err,
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminCreateUser,
-			})
+		return fmt.Errorf("failed to create a user: %w", err)
 	}
 
-	return SendResponse(ctx, nil,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminCreateUser,
-			Status: http.StatusCreated,
-		})
-}
-
-func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
-	access := ctx.Query("access")
-	if access == "" {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminUpdateUser,
-			})
-	}
-
-	var props auth.MutableProps
-	if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminUpdateUser,
-			})
-	}
-
-	err := c.iam.UpdateUserAccount(access, props)
-	if err != nil {
-		if strings.Contains(err.Error(), "user not found") {
-			err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
-		}
-
-		return SendResponse(ctx, err,
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminUpdateUser,
-			})
-	}
-
-	return SendResponse(ctx, nil,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminUpdateUser,
-		})
+	return ctx.SendString("The user has been created successfully")
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
 
 	err := c.iam.DeleteUserAccount(access)
-	return SendResponse(ctx, err,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminDeleteUser,
-		})
+	if err != nil {
+		return err
+	}
+
+	return ctx.SendString("The user has been deleted successfully")
 }
 
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
 	accs, err := c.iam.ListUserAccounts()
-	return SendXMLResponse(ctx,
-		auth.ListUserAccountsResult{
-			Accounts: accs,
-		}, err,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminListUsers,
-		})
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(accs)
 }
 
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
 
 	accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
 	if err != nil {
-		return SendResponse(ctx, err,
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminChangeBucketOwner,
-			})
+		return err
 	}
 	if len(accs) > 0 {
-		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminChangeBucketOwner,
-			})
+		return fmt.Errorf("user specified as the new bucket owner does not exist")
 	}
 
-	acl := auth.ACL{
-		Owner: owner,
-		Grantees: []auth.Grantee{
-			{
-				Permission: auth.PermissionFullControl,
-				Access:     owner,
-				Type:       types.TypeCanonicalUser,
-			},
-		},
-	}
-
-	aclParsed, err := json.Marshal(acl)
+	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
 	if err != nil {
-		return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
-			&MetaOpts{
-				Logger: c.l,
-				Action: metrics.ActionAdminChangeBucketOwner,
-			})
+		return err
 	}
 
-	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
-	return SendResponse(ctx, err,
-		&MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminChangeBucketOwner,
-		})
+	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return fmt.Errorf("access denied: only admin users have access to this resource")
+	}
+
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
-	return SendXMLResponse(ctx,
-		s3response.ListBucketsResult{
-			Buckets: buckets,
-		}, err, &MetaOpts{
-			Logger: c.l,
-			Action: metrics.ActionAdminListBuckets,
-		})
+	if err != nil {
+		return err
+	}
+
+	return ctx.JSON(buckets)
 }