mirror of
https://github.com/versity/versitygw.git
synced 2026-01-25 04:22:02 +00:00
Compare commits
1 Commits
ben/plugin
...
ben/read_o
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9b41d53b6 |
18
.github/ISSUE_TEMPLATE/bug_report.md
vendored
18
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -1,23 +1,27 @@
|
||||
---
|
||||
name: Bug Report
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: '[Bug] - <Short Description>'
|
||||
title: ''
|
||||
labels: bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
<!-- A clear and concise description of what the bug is. -->
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
<!-- Steps to reproduce the behavior. -->
|
||||
Steps to reproduce the behavior.
|
||||
|
||||
**Expected behavior**
|
||||
<!-- A clear and concise description of what you expected to happen. -->
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Server Version**
|
||||
<!-- output of: './versitygw -version && uname -a' -->
|
||||
output of
|
||||
```
|
||||
./versitygw -version
|
||||
uname -a
|
||||
```
|
||||
|
||||
**Additional context**
|
||||
<!-- Describe s3 client and version if applicable.
|
||||
Describe s3 client and version if applicable.
|
||||
|
||||
8
.github/ISSUE_TEMPLATE/feature_request.md
vendored
8
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -1,14 +1,14 @@
|
||||
---
|
||||
name: Feature Request
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: '[Feature] - <Short Description>'
|
||||
title: ''
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the solution you'd like**
|
||||
<!-- A clear and concise description of what you want to happen. -->
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Additional context**
|
||||
<!-- Add any other context or screenshots about the feature request here. -->
|
||||
Add any other context or screenshots about the feature request here.
|
||||
|
||||
33
.github/ISSUE_TEMPLATE/test_case.md
vendored
33
.github/ISSUE_TEMPLATE/test_case.md
vendored
@@ -1,33 +0,0 @@
|
||||
---
|
||||
name: Test Case Request
|
||||
about: Request new test cases or additional test coverage
|
||||
title: '[Test Case] - <Short Description>'
|
||||
labels: 'testcase'
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
<!-- Please provide a detailed description of the test case or test coverage request. -->
|
||||
|
||||
## Purpose
|
||||
<!-- Explain why this test case is important and what it aims to achieve. -->
|
||||
|
||||
## Scope
|
||||
<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
|
||||
|
||||
## Acceptance Criteria
|
||||
<!-- List the criteria that must be met for the test case to be considered complete. -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
## Additional Context
|
||||
<!-- Add any other context or screenshots about the feature request here. -->
|
||||
|
||||
## Resources
|
||||
<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
|
||||
|
||||
|
||||
**Thank you for contributing to our project!**
|
||||
37
.github/workflows/azurite.yml
vendored
37
.github/workflows/azurite.yml
vendored
@@ -1,37 +0,0 @@
|
||||
name: azurite functional tests
|
||||
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Compose
|
||||
run: |
|
||||
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
|
||||
|
||||
- name: Wait for Azurite to be ready
|
||||
run: sleep 40
|
||||
|
||||
- name: Get Dependencies
|
||||
run: |
|
||||
go mod download
|
||||
|
||||
- name: Build and Run
|
||||
run: |
|
||||
make
|
||||
./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
|
||||
|
||||
- name: Shut down services
|
||||
run: |
|
||||
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw
|
||||
29
.github/workflows/docker-bats.yml
vendored
29
.github/workflows/docker-bats.yml
vendored
@@ -1,29 +0,0 @@
|
||||
name: docker bats tests
|
||||
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
cp tests/.env.docker.default tests/.env.docker
|
||||
cp tests/.secrets.default tests/.secrets
|
||||
# see https://github.com/versity/versitygw/issues/1034
|
||||
docker build \
|
||||
--build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
|
||||
--build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
|
||||
--build-arg="MC_FOLDER=linux-amd64" \
|
||||
--progress=plain \
|
||||
-f tests/Dockerfile_test_bats \
|
||||
-t bats_test .
|
||||
|
||||
- name: Run Docker Container
|
||||
run: |
|
||||
docker compose -f tests/docker-compose-bats.yml --project-directory . \
|
||||
up --exit-code-from s3api_np_only s3api_np_only
|
||||
@@ -15,13 +15,6 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -50,7 +43,6 @@ jobs:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build-args: |
|
||||
VERSION=${{ github.event.release.tag_name }}
|
||||
TIME=${{ github.event.release.published_at }}
|
||||
5
.github/workflows/functional.yml
vendored
5
.github/workflows/functional.yml
vendored
@@ -1,8 +1,7 @@
|
||||
name: functional tests
|
||||
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
@@ -19,7 +18,7 @@ jobs:
|
||||
|
||||
- name: Get Dependencies
|
||||
run: |
|
||||
go mod download
|
||||
go get -v -t -d ./...
|
||||
|
||||
- name: Build and Run
|
||||
run: |
|
||||
|
||||
16
.github/workflows/shellcheck.yml
vendored
16
.github/workflows/shellcheck.yml
vendored
@@ -1,16 +0,0 @@
|
||||
name: shellcheck
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: Run shellcheck
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run checks
|
||||
run: |
|
||||
shellcheck --version
|
||||
shellcheck -e SC1091 tests/*.sh tests/*/*.sh
|
||||
179
.github/workflows/system.yml
vendored
179
.github/workflows/system.yml
vendored
@@ -4,105 +4,16 @@ jobs:
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- set: "mc, posix, non-file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "mc-non-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "mc, posix, file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "mc-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "REST, posix, non-static, all, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "rest"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3, posix, non-file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3-non-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3, posix, file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, policy, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-policy"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, user, non-static, s3 IAM"
|
||||
IAM_TYPE: s3
|
||||
RUN_SET: "s3api-user"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, bucket, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-bucket"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, multipart, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-multipart"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, object, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-object"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, policy, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-policy"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, user, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-user"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
# TODO fix/debug s3 gateway
|
||||
#- set: "s3api, s3, multipart|object, non-static, folder IAM"
|
||||
# IAM_TYPE: folder
|
||||
# RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
|
||||
# RECREATE_BUCKETS: "true"
|
||||
# BACKEND: "s3"
|
||||
#- set: "s3api, s3, policy|user, non-static, folder IAM"
|
||||
# IAM_TYPE: folder
|
||||
# RUN_SET: "s3api-policy,s3api-user"
|
||||
# RECREATE_BUCKETS: "true"
|
||||
# BACKEND: "s3"
|
||||
- set: "s3cmd, posix, file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3cmd-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3cmd, posix, non-user, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3cmd-non-user"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3cmd, posix, user, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3cmd-user"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install ShellCheck
|
||||
run: sudo apt-get install shellcheck
|
||||
|
||||
- name: Run ShellCheck
|
||||
run: shellcheck -S warning ./tests/*.sh
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
@@ -117,8 +28,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/bats-core/bats-core.git
|
||||
cd bats-core && ./install.sh $HOME
|
||||
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
|
||||
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
|
||||
|
||||
- name: Install s3cmd
|
||||
run: |
|
||||
@@ -129,73 +38,35 @@ jobs:
|
||||
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
|
||||
chmod 755 /usr/local/bin/mc
|
||||
|
||||
- name: Install xmllint (for rest)
|
||||
run: |
|
||||
sudo apt-get install libxml2-utils
|
||||
|
||||
# see https://github.com/versity/versitygw/issues/1034
|
||||
- name: Install AWS cli
|
||||
run: |
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip"
|
||||
unzip -o awscliv2.zip
|
||||
./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
|
||||
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Build and run
|
||||
env:
|
||||
IAM_TYPE: ${{ matrix.IAM_TYPE }}
|
||||
RUN_SET: ${{ matrix.RUN_SET }}
|
||||
AWS_PROFILE: versity
|
||||
VERSITY_EXE: ${{ github.workspace }}/versitygw
|
||||
RUN_VERSITYGW: true
|
||||
BACKEND: ${{ matrix.BACKEND }}
|
||||
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
|
||||
CERT: ${{ github.workspace }}/cert.pem
|
||||
KEY: ${{ github.workspace }}/versitygw.pem
|
||||
LOCAL_FOLDER: /tmp/gw
|
||||
BUCKET_ONE_NAME: versity-gwtest-bucket-one
|
||||
BUCKET_TWO_NAME: versity-gwtest-bucket-two
|
||||
USERS_FOLDER: /tmp/iam
|
||||
USERS_BUCKET: versity-gwtest-iam
|
||||
AWS_ENDPOINT_URL: https://127.0.0.1:7070
|
||||
PORT: 7070
|
||||
S3CMD_CONFIG: tests/s3cfg.local.default
|
||||
MC_ALIAS: versity
|
||||
LOG_LEVEL: 4
|
||||
GOCOVERDIR: ${{ github.workspace }}/cover
|
||||
USERNAME_ONE: ABCDEFG
|
||||
PASSWORD_ONE: 1234567
|
||||
USERNAME_TWO: HIJKLMN
|
||||
PASSWORD_TWO: 8901234
|
||||
TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
|
||||
REMOVE_TEST_FILE_FOLDER: true
|
||||
VERSIONING_DIR: ${{ github.workspace }}/versioning
|
||||
COMMAND_LOG: command.log
|
||||
TIME_LOG: time.log
|
||||
PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
|
||||
- name: Build and run, posix backend
|
||||
run: |
|
||||
make testbin
|
||||
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
|
||||
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
export AWS_REGION=us-east-1
|
||||
export AWS_ACCESS_KEY_ID_TWO=user
|
||||
export AWS_SECRET_ACCESS_KEY_TWO=pass
|
||||
export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
|
||||
aws configure set aws_region $AWS_REGION --profile versity
|
||||
mkdir $LOCAL_FOLDER
|
||||
mkdir /tmp/gw
|
||||
export WORKSPACE=$GITHUB_WORKSPACE
|
||||
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
|
||||
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
mkdir $GOCOVERDIR $USERS_FOLDER
|
||||
if [[ $RECREATE_BUCKETS == "false" ]]; then
|
||||
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
|
||||
fi
|
||||
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
|
||||
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
|
||||
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
mkdir cover
|
||||
VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
|
||||
|
||||
- name: Time report
|
||||
run: cat ${{ github.workspace }}/time.log
|
||||
#- name: Build and run, s3 backend
|
||||
# run: |
|
||||
# make testbin
|
||||
# export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
|
||||
# export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
# export AWS_REGION=us-east-1
|
||||
# aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
|
||||
# aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
|
||||
# aws configure set aws_region $AWS_REGION --profile versity_s3
|
||||
# export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
|
||||
# export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
# export WORKSPACE=$GITHUB_WORKSPACE
|
||||
# VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
|
||||
|
||||
- name: Coverage report
|
||||
run: |
|
||||
|
||||
22
.gitignore
vendored
22
.gitignore
vendored
@@ -45,25 +45,7 @@ tests/.secrets*
|
||||
|
||||
# IAM users files often created in testing
|
||||
users.json
|
||||
users.json.backup
|
||||
|
||||
# env files for testing
|
||||
**/.env*
|
||||
**/!.env.default
|
||||
|
||||
# s3cmd config files (testing)
|
||||
tests/s3cfg.local*
|
||||
tests/!s3cfg.local.default
|
||||
|
||||
# keys
|
||||
*.pem
|
||||
|
||||
# patches
|
||||
*.patch
|
||||
|
||||
# grafana's local database (kept on filesystem for survival between instantiations)
|
||||
metrics-exploration/grafana_data/**
|
||||
|
||||
# bats tools
|
||||
/tests/bats-assert
|
||||
/tests/bats-support
|
||||
.env*
|
||||
!.env.default
|
||||
|
||||
@@ -32,4 +32,4 @@ RUN mkdir -p $SETUP_DIR
|
||||
|
||||
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
|
||||
|
||||
ENTRYPOINT [ "/app/versitygw" ]
|
||||
ENTRYPOINT [ "/app/versitygw" ]
|
||||
@@ -6,7 +6,7 @@ COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY ./ ./
|
||||
COPY ./tests/certs/* /etc/pki/tls/certs/
|
||||
COPY certs/* /etc/pki/tls/certs/
|
||||
|
||||
ARG IAM_DIR=/tmp/vgw
|
||||
ARG SETUP_DIR=/tmp/vgw
|
||||
@@ -1,12 +1,8 @@
|
||||
FROM ubuntu:latest
|
||||
FROM --platform=linux/arm64 ubuntu:latest
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG SECRETS_FILE=tests/.secrets
|
||||
ARG CONFIG_FILE=tests/.env.docker
|
||||
ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz
|
||||
# see https://github.com/versity/versitygw/issues/1034
|
||||
ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip
|
||||
ARG MC_FOLDER=linux-arm64
|
||||
|
||||
ENV TZ=Etc/UTC
|
||||
RUN apt-get update && \
|
||||
@@ -20,7 +16,6 @@ RUN apt-get update && \
|
||||
s3cmd \
|
||||
jq \
|
||||
bc \
|
||||
libxml2-utils \
|
||||
ca-certificates && \
|
||||
update-ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -29,20 +24,20 @@ RUN apt-get update && \
|
||||
WORKDIR /tmp
|
||||
|
||||
# Install AWS cli
|
||||
RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
|
||||
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
|
||||
|
||||
# Install mc
|
||||
RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
|
||||
RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
|
||||
--create-dirs \
|
||||
-o /usr/local/minio-binaries/mc && \
|
||||
chmod -R 755 /usr/local/minio-binaries
|
||||
ENV PATH=/usr/local/minio-binaries:${PATH}
|
||||
ENV PATH="/usr/local/minio-binaries":${PATH}
|
||||
|
||||
# Download Go 1.21 (adjust the version and platform as needed)
|
||||
RUN wget https://golang.org/dl/${GO_LIBRARY}
|
||||
RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
|
||||
|
||||
# Extract the downloaded archive
|
||||
RUN tar -xvf $GO_LIBRARY -C /usr/local
|
||||
RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
|
||||
|
||||
# Set Go environment variables
|
||||
ENV PATH="/usr/local/go/bin:${PATH}"
|
||||
@@ -65,11 +60,9 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
|
||||
USER tester
|
||||
COPY --chown=tester:tester . /home/tester
|
||||
|
||||
# add bats support libraries
|
||||
RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
|
||||
RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
|
||||
|
||||
WORKDIR /home/tester
|
||||
#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
|
||||
RUN cp tests/s3cfg.local.default tests/s3cfg.local
|
||||
RUN make
|
||||
|
||||
RUN . $SECRETS_FILE && \
|
||||
@@ -86,7 +79,5 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
|
||||
|
||||
ENV WORKSPACE=.
|
||||
ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
|
||||
#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
|
||||
|
||||
ENTRYPOINT ["tests/run.sh"]
|
||||
CMD ["s3api,s3,s3cmd,mc,rest"]
|
||||
CMD ["tests/run_all.sh"]
|
||||
12
Makefile
12
Makefile
@@ -18,10 +18,6 @@ GOBUILD=$(GOCMD) build
|
||||
GOCLEAN=$(GOCMD) clean
|
||||
GOTEST=$(GOCMD) test
|
||||
|
||||
# docker-compose
|
||||
DCCMD=docker-compose
|
||||
DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
|
||||
|
||||
BIN=versitygw
|
||||
|
||||
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
|
||||
@@ -75,19 +71,19 @@ dist:
|
||||
# Creates and runs S3 gateway instance in a docker container
|
||||
.PHONY: up-posix
|
||||
up-posix:
|
||||
$(DOCKERCOMPOSE) up posix
|
||||
docker compose --env-file .env.dev up posix
|
||||
|
||||
# Creates and runs S3 gateway proxy instance in a docker container
|
||||
.PHONY: up-proxy
|
||||
up-proxy:
|
||||
$(DOCKERCOMPOSE) up proxy
|
||||
docker compose --env-file .env.dev up proxy
|
||||
|
||||
# Creates and runs S3 gateway to azurite instance in a docker container
|
||||
.PHONY: up-azurite
|
||||
up-azurite:
|
||||
$(DOCKERCOMPOSE) up azurite azuritegw
|
||||
docker compose --env-file .env.dev up azurite azuritegw
|
||||
|
||||
# Creates and runs both S3 gateway and proxy server instances in docker containers
|
||||
.PHONY: up-app
|
||||
up-app:
|
||||
$(DOCKERCOMPOSE) up
|
||||
docker compose --env-file .env.dev up
|
||||
|
||||
21
README.md
21
README.md
@@ -6,7 +6,7 @@
|
||||
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
|
||||
</picture>
|
||||
|
||||
[](https://github.com/versity/versitygw/blob/main/LICENSE) [](https://goreportcard.com/report/github.com/versity/versitygw) [](https://pkg.go.dev/github.com/versity/versitygw)
|
||||
[](https://github.com/versity/versitygw/blob/main/LICENSE)
|
||||
|
||||
### Binary release builds
|
||||
Download [latest release](https://github.com/versity/versitygw/releases)
|
||||
@@ -14,15 +14,8 @@ Download [latest release](https://github.com/versity/versitygw/releases)
|
||||
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
|
||||
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
|
||||
|
||||
### Use Cases
|
||||
* Turn your local filesystem into an S3 server with a single command!
|
||||
* Proxy S3 requests to S3 storage
|
||||
* Simple to deploy S3 server with a single command
|
||||
* Protocol compatibility in `posix` allows common access to files via posix or S3
|
||||
* Simplified interface for adding new storage system support
|
||||
|
||||
### News
|
||||
Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
|
||||
* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
|
||||
|
||||
### Mailing List
|
||||
Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
|
||||
@@ -35,12 +28,18 @@ Ask questions in the [community discussions](https://github.com/versity/versityg
|
||||
<br>
|
||||
Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
|
||||
|
||||
### Use Cases
|
||||
* Share filesystem directory via S3 protocol
|
||||
* Proxy S3 requests to S3 storage
|
||||
* Simple to deploy S3 server with a single command
|
||||
* Protocol compatibility in `posix` allows common access to files via posix or S3
|
||||
|
||||
### Overview
|
||||
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
|
||||
|
||||
The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
|
||||
|
||||
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versity’s open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.
|
||||
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.
|
||||
|
||||
The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance.
|
||||
|
||||
@@ -68,7 +67,7 @@ The command format is
|
||||
```
|
||||
versitygw [global options] command [command options] [arguments...]
|
||||
```
|
||||
The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
|
||||
The global options are specified before the backend type and the backend options are specified after.
|
||||
|
||||
***
|
||||
|
||||
|
||||
343
auth/acl.go
343
auth/acl.go
@@ -17,8 +17,6 @@ package auth
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
@@ -29,156 +27,27 @@ import (
|
||||
)
|
||||
|
||||
type ACL struct {
|
||||
ACL types.BucketCannedACL
|
||||
Owner string
|
||||
Grantees []Grantee
|
||||
}
|
||||
|
||||
type Grantee struct {
|
||||
Permission Permission
|
||||
Permission types.Permission
|
||||
Access string
|
||||
Type types.Type
|
||||
}
|
||||
|
||||
type GetBucketAclOutput struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
|
||||
Owner *types.Owner
|
||||
AccessControlList AccessControlList
|
||||
}
|
||||
|
||||
type PutBucketAclInput struct {
|
||||
Bucket *string
|
||||
ACL types.BucketCannedACL
|
||||
AccessControlPolicy *AccessControlPolicy
|
||||
GrantFullControl *string
|
||||
GrantRead *string
|
||||
GrantReadACP *string
|
||||
GrantWrite *string
|
||||
GrantWriteACP *string
|
||||
type AccessControlList struct {
|
||||
Grants []types.Grant `xml:"Grant"`
|
||||
}
|
||||
|
||||
type AccessControlPolicy struct {
|
||||
AccessControlList AccessControlList `xml:"AccessControlList"`
|
||||
Owner *types.Owner
|
||||
}
|
||||
|
||||
func (acp *AccessControlPolicy) Validate() error {
|
||||
if !acp.AccessControlList.isValid() {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedACL)
|
||||
}
|
||||
|
||||
// The Owner can't be nil
|
||||
if acp.Owner == nil {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedACL)
|
||||
}
|
||||
|
||||
// The Owner ID can't be empty
|
||||
if acp.Owner.ID == nil || *acp.Owner.ID == "" {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedACL)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type AccessControlList struct {
|
||||
Grants []Grant `xml:"Grant"`
|
||||
}
|
||||
|
||||
// Validates the AccessControlList
|
||||
func (acl *AccessControlList) isValid() bool {
|
||||
for _, el := range acl.Grants {
|
||||
if !el.isValid() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
type Permission string
|
||||
|
||||
const (
|
||||
PermissionFullControl Permission = "FULL_CONTROL"
|
||||
PermissionWrite Permission = "WRITE"
|
||||
PermissionWriteAcp Permission = "WRITE_ACP"
|
||||
PermissionRead Permission = "READ"
|
||||
PermissionReadAcp Permission = "READ_ACP"
|
||||
)
|
||||
|
||||
// Check if the permission is valid
|
||||
func (p Permission) isValid() bool {
|
||||
return p == PermissionFullControl ||
|
||||
p == PermissionRead ||
|
||||
p == PermissionReadAcp ||
|
||||
p == PermissionWrite ||
|
||||
p == PermissionWriteAcp
|
||||
}
|
||||
|
||||
type Grant struct {
|
||||
Grantee *Grt `xml:"Grantee"`
|
||||
Permission Permission `xml:"Permission"`
|
||||
}
|
||||
|
||||
// Checks if Grant is valid
|
||||
func (g *Grant) isValid() bool {
|
||||
return g.Permission.isValid() && g.Grantee.isValid()
|
||||
}
|
||||
|
||||
type Grt struct {
|
||||
XMLNS string `xml:"xmlns:xsi,attr"`
|
||||
Type types.Type `xml:"xsi:type,attr"`
|
||||
ID string `xml:"ID"`
|
||||
}
|
||||
|
||||
// Custom Unmarshalling for Grt to parse xsi:type properly
|
||||
func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
// Iterate through the XML tokens to process the attributes
|
||||
for _, attr := range start.Attr {
|
||||
// Check if the attribute is xsi:type and belongs to the xsi namespace
|
||||
if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
|
||||
g.Type = types.Type(attr.Value)
|
||||
}
|
||||
// Handle xmlns:xsi
|
||||
if attr.Name.Local == "xmlns:xsi" {
|
||||
g.XMLNS = attr.Value
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the inner XML elements like ID
|
||||
for {
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch se := t.(type) {
|
||||
case xml.StartElement:
|
||||
if se.Name.Local == "ID" {
|
||||
if err := d.DecodeElement(&g.ID, &se); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case xml.EndElement:
|
||||
if se.Name.Local == start.Name.Local {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validates Grt
|
||||
func (g *Grt) isValid() bool {
|
||||
// Validate the Type
|
||||
// Only these 2 types are supported in the gateway
|
||||
if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
|
||||
return false
|
||||
}
|
||||
|
||||
// The ID prop shouldn't be empty
|
||||
if g.ID == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
Owner types.Owner
|
||||
}
|
||||
|
||||
func ParseACL(data []byte) (ACL, error) {
|
||||
@@ -193,35 +62,17 @@ func ParseACL(data []byte) (ACL, error) {
|
||||
return acl, nil
|
||||
}
|
||||
|
||||
func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
|
||||
grants := []Grant{}
|
||||
|
||||
if len(data) == 0 {
|
||||
return GetBucketAclOutput{
|
||||
Owner: &types.Owner{
|
||||
ID: &owner,
|
||||
},
|
||||
AccessControlList: AccessControlList{
|
||||
Grants: grants,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
|
||||
var acl ACL
|
||||
if err := json.Unmarshal(data, &acl); err != nil {
|
||||
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
|
||||
}
|
||||
|
||||
grants := []types.Grant{}
|
||||
|
||||
for _, elem := range acl.Grantees {
|
||||
acs := elem.Access
|
||||
grants = append(grants, Grant{
|
||||
Grantee: &Grt{
|
||||
XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
|
||||
ID: acs,
|
||||
Type: elem.Type,
|
||||
},
|
||||
Permission: elem.Permission,
|
||||
})
|
||||
grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
|
||||
}
|
||||
|
||||
return GetBucketAclOutput{
|
||||
@@ -234,43 +85,20 @@ func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
|
||||
func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
|
||||
if input == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
defaultGrantees := []Grantee{
|
||||
{
|
||||
Permission: PermissionFullControl,
|
||||
Access: acl.Owner,
|
||||
Type: types.TypeCanonicalUser,
|
||||
},
|
||||
if acl.Owner != *input.AccessControlPolicy.Owner.ID {
|
||||
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
// if the ACL is specified, set the ACL, else replace the grantees
|
||||
if input.ACL != "" {
|
||||
switch input.ACL {
|
||||
case types.BucketCannedACLPublicRead:
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Permission: PermissionRead,
|
||||
Access: "all-users",
|
||||
Type: types.TypeGroup,
|
||||
})
|
||||
case types.BucketCannedACLPublicReadWrite:
|
||||
defaultGrantees = append(defaultGrantees, []Grantee{
|
||||
{
|
||||
Permission: PermissionRead,
|
||||
Access: "all-users",
|
||||
Type: types.TypeGroup,
|
||||
},
|
||||
{
|
||||
Permission: PermissionWrite,
|
||||
Access: "all-users",
|
||||
Type: types.TypeGroup,
|
||||
},
|
||||
}...)
|
||||
}
|
||||
acl.ACL = input.ACL
|
||||
acl.Grantees = []Grantee{}
|
||||
} else {
|
||||
grantees := []Grantee{}
|
||||
accs := []string{}
|
||||
|
||||
if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
|
||||
@@ -279,71 +107,45 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
|
||||
if input.GrantFullControl != nil && *input.GrantFullControl != "" {
|
||||
fullControlList = splitUnique(*input.GrantFullControl, ",")
|
||||
for _, str := range fullControlList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionFullControl,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
|
||||
}
|
||||
}
|
||||
if input.GrantRead != nil && *input.GrantRead != "" {
|
||||
readList = splitUnique(*input.GrantRead, ",")
|
||||
for _, str := range readList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionRead,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
|
||||
}
|
||||
}
|
||||
if input.GrantReadACP != nil && *input.GrantReadACP != "" {
|
||||
readACPList = splitUnique(*input.GrantReadACP, ",")
|
||||
for _, str := range readACPList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionReadAcp,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
|
||||
}
|
||||
}
|
||||
if input.GrantWrite != nil && *input.GrantWrite != "" {
|
||||
writeList = splitUnique(*input.GrantWrite, ",")
|
||||
for _, str := range writeList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionWrite,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
|
||||
}
|
||||
}
|
||||
if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
|
||||
writeACPList = splitUnique(*input.GrantWriteACP, ",")
|
||||
for _, str := range writeACPList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionWriteAcp,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
|
||||
}
|
||||
}
|
||||
|
||||
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
|
||||
} else {
|
||||
cache := make(map[string]bool)
|
||||
for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
|
||||
if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
|
||||
for _, grt := range input.AccessControlPolicy.Grants {
|
||||
if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
access := grt.Grantee.ID
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: access,
|
||||
Permission: grt.Permission,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
if _, ok := cache[access]; !ok {
|
||||
cache[access] = true
|
||||
accs = append(accs, access)
|
||||
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
|
||||
if _, ok := cache[*grt.Grantee.ID]; !ok {
|
||||
cache[*grt.Grantee.ID] = true
|
||||
accs = append(accs, *grt.Grantee.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -356,9 +158,10 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
|
||||
if len(accList) > 0 {
|
||||
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
acl.Grantees = defaultGrantees
|
||||
acl.Grantees = grantees
|
||||
acl.ACL = ""
|
||||
}
|
||||
|
||||
result, err := json.Marshal(acl)
|
||||
if err != nil {
|
||||
@@ -378,8 +181,8 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
|
||||
result = append(result, acc)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
|
||||
return nil, err
|
||||
if err == ErrNotSupported {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
return nil, fmt.Errorf("check user account: %w", err)
|
||||
}
|
||||
@@ -402,34 +205,35 @@ func splitUnique(s, divider string) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
func verifyACL(acl ACL, access string, permission Permission) error {
|
||||
grantee := Grantee{
|
||||
Access: access,
|
||||
Permission: permission,
|
||||
Type: types.TypeCanonicalUser,
|
||||
}
|
||||
granteeFullCtrl := Grantee{
|
||||
Access: access,
|
||||
Permission: PermissionFullControl,
|
||||
Type: types.TypeCanonicalUser,
|
||||
}
|
||||
granteeAllUsers := Grantee{
|
||||
Access: "all-users",
|
||||
Permission: permission,
|
||||
Type: types.TypeGroup,
|
||||
}
|
||||
|
||||
isFound := false
|
||||
|
||||
for _, grt := range acl.Grantees {
|
||||
if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
|
||||
isFound = true
|
||||
break
|
||||
func verifyACL(acl ACL, access string, permission types.Permission) error {
|
||||
if acl.ACL != "" {
|
||||
if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
}
|
||||
|
||||
if isFound {
|
||||
return nil
|
||||
} else {
|
||||
if len(acl.Grantees) == 0 {
|
||||
return nil
|
||||
}
|
||||
grantee := Grantee{Access: access, Permission: permission}
|
||||
granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
|
||||
|
||||
isFound := false
|
||||
|
||||
for _, grt := range acl.Grantees {
|
||||
if grt == grantee || grt == granteeFullCtrl {
|
||||
isFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if isFound {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
@@ -469,37 +273,38 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
|
||||
|
||||
type AccessOptions struct {
|
||||
Acl ACL
|
||||
AclPermission Permission
|
||||
AclPermission types.Permission
|
||||
IsRoot bool
|
||||
Acc Account
|
||||
Bucket string
|
||||
Object string
|
||||
Action Action
|
||||
Readonly bool
|
||||
}
|
||||
|
||||
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
|
||||
if opts.Readonly {
|
||||
if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
}
|
||||
if opts.IsRoot {
|
||||
return nil
|
||||
}
|
||||
if opts.Acc.Role == RoleAdmin {
|
||||
return nil
|
||||
}
|
||||
|
||||
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
|
||||
if policyErr != nil {
|
||||
if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return policyErr
|
||||
}
|
||||
} else {
|
||||
return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
|
||||
if opts.Acc.Access == opts.Acl.Owner {
|
||||
return nil
|
||||
}
|
||||
|
||||
policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If bucket policy is not set and the ACL is default, only the owner has access
|
||||
if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -538,7 +343,7 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource
|
||||
|
||||
if err := VerifyAccess(ctx, be, AccessOptions{
|
||||
Acl: srcBucketAcl,
|
||||
AclPermission: PermissionRead,
|
||||
AclPermission: types.PermissionRead,
|
||||
IsRoot: opts.IsRoot,
|
||||
Acc: opts.Acc,
|
||||
Bucket: srcBucket,
|
||||
|
||||
@@ -16,52 +16,16 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type policyErr string
|
||||
|
||||
func (p policyErr) Error() string {
|
||||
return string(p)
|
||||
}
|
||||
|
||||
const (
|
||||
policyErrResourceMismatch = policyErr("Action does not apply to any resource(s) in statement")
|
||||
policyErrInvalidResource = policyErr("Policy has invalid resource")
|
||||
policyErrInvalidPrincipal = policyErr("Invalid principal in policy")
|
||||
policyErrInvalidAction = policyErr("Policy has invalid action")
|
||||
policyErrInvalidPolicy = policyErr("This policy contains invalid Json")
|
||||
policyErrInvalidFirstChar = policyErr("Policies must be valid JSON and the first byte must be '{'")
|
||||
policyErrEmptyStatement = policyErr("Could not parse the policy: Statement is empty!")
|
||||
policyErrMissingStatmentField = policyErr("Missing required field Statement")
|
||||
)
|
||||
|
||||
type BucketPolicy struct {
|
||||
Statement []BucketPolicyItem `json:"Statement"`
|
||||
}
|
||||
|
||||
func (bp *BucketPolicy) UnmarshalJSON(data []byte) error {
|
||||
var tmp struct {
|
||||
Statement *[]BucketPolicyItem `json:"Statement"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If Statement is nil (not present in JSON), return an error
|
||||
if tmp.Statement == nil {
|
||||
return policyErrMissingStatmentField
|
||||
}
|
||||
|
||||
// Assign the parsed value to the actual struct
|
||||
bp.Statement = *tmp.Statement
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
|
||||
for _, statement := range bp.Statement {
|
||||
err := statement.Validate(bucket, iam)
|
||||
@@ -74,19 +38,18 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
|
||||
}
|
||||
|
||||
func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
|
||||
var isAllowed bool
|
||||
for _, statement := range bp.Statement {
|
||||
if statement.findMatch(principal, action, resource) {
|
||||
switch statement.Effect {
|
||||
case BucketPolicyAccessTypeAllow:
|
||||
isAllowed = true
|
||||
return true
|
||||
case BucketPolicyAccessTypeDeny:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return isAllowed
|
||||
return false
|
||||
}
|
||||
|
||||
type BucketPolicyItem struct {
|
||||
@@ -112,14 +75,11 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
|
||||
|
||||
for action := range bpi.Actions {
|
||||
isObjectAction := action.IsObjectAction()
|
||||
if isObjectAction == nil {
|
||||
break
|
||||
if isObjectAction && !containsObjectAction {
|
||||
return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
|
||||
}
|
||||
if *isObjectAction && !containsObjectAction {
|
||||
return policyErrResourceMismatch
|
||||
}
|
||||
if !*isObjectAction && !containsBucketAction {
|
||||
return policyErrResourceMismatch
|
||||
if !isObjectAction && !containsBucketAction {
|
||||
return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,20 +103,9 @@ func getMalformedPolicyError(err error) error {
|
||||
}
|
||||
|
||||
func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
|
||||
if len(policyBin) == 0 || policyBin[0] != '{' {
|
||||
return getMalformedPolicyError(policyErrInvalidFirstChar)
|
||||
}
|
||||
var policy BucketPolicy
|
||||
if err := json.Unmarshal(policyBin, &policy); err != nil {
|
||||
var pe policyErr
|
||||
if errors.As(err, &pe) {
|
||||
return getMalformedPolicyError(err)
|
||||
}
|
||||
return getMalformedPolicyError(policyErrInvalidPolicy)
|
||||
}
|
||||
|
||||
if len(policy.Statement) == 0 {
|
||||
return getMalformedPolicyError(policyErrEmptyStatement)
|
||||
return getMalformedPolicyError(err)
|
||||
}
|
||||
|
||||
if err := policy.Validate(bucket, iam); err != nil {
|
||||
@@ -166,7 +115,12 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
|
||||
func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
|
||||
// If bucket policy is not set
|
||||
if len(policy) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var bucketPolicy BucketPolicy
|
||||
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
|
||||
return err
|
||||
@@ -177,6 +131,7 @@ func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Act
|
||||
resource += "/" + object
|
||||
}
|
||||
|
||||
fmt.Println(access, action, resource)
|
||||
if !bucketPolicy.isAllowed(access, action, resource) {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
@@ -16,120 +16,92 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
GetBucketAclAction Action = "s3:GetBucketAcl"
|
||||
CreateBucketAction Action = "s3:CreateBucket"
|
||||
PutBucketAclAction Action = "s3:PutBucketAcl"
|
||||
DeleteBucketAction Action = "s3:DeleteBucket"
|
||||
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
|
||||
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
|
||||
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
|
||||
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
|
||||
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
|
||||
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
|
||||
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
|
||||
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
|
||||
PutObjectAction Action = "s3:PutObject"
|
||||
GetObjectAction Action = "s3:GetObject"
|
||||
GetObjectVersionAction Action = "s3:GetObjectVersion"
|
||||
DeleteObjectAction Action = "s3:DeleteObject"
|
||||
GetObjectAclAction Action = "s3:GetObjectAcl"
|
||||
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
|
||||
PutObjectAclAction Action = "s3:PutObjectAcl"
|
||||
RestoreObjectAction Action = "s3:RestoreObject"
|
||||
GetBucketTaggingAction Action = "s3:GetBucketTagging"
|
||||
PutBucketTaggingAction Action = "s3:PutBucketTagging"
|
||||
GetObjectTaggingAction Action = "s3:GetObjectTagging"
|
||||
PutObjectTaggingAction Action = "s3:PutObjectTagging"
|
||||
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
|
||||
ListBucketVersionsAction Action = "s3:ListBucketVersions"
|
||||
ListBucketAction Action = "s3:ListBucket"
|
||||
GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
|
||||
PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
|
||||
GetObjectLegalHoldAction Action = "s3:GetObjectLegalHold"
|
||||
PutObjectLegalHoldAction Action = "s3:PutObjectLegalHold"
|
||||
GetObjectRetentionAction Action = "s3:GetObjectRetention"
|
||||
PutObjectRetentionAction Action = "s3:PutObjectRetention"
|
||||
BypassGovernanceRetentionAction Action = "s3:BypassGovernanceRetention"
|
||||
PutBucketOwnershipControlsAction Action = "s3:PutBucketOwnershipControls"
|
||||
GetBucketOwnershipControlsAction Action = "s3:GetBucketOwnershipControls"
|
||||
PutBucketCorsAction Action = "s3:PutBucketCORS"
|
||||
GetBucketCorsAction Action = "s3:GetBucketCORS"
|
||||
AllActions Action = "s3:*"
|
||||
GetBucketAclAction Action = "s3:GetBucketAcl"
|
||||
CreateBucketAction Action = "s3:CreateBucket"
|
||||
PutBucketAclAction Action = "s3:PutBucketAcl"
|
||||
DeleteBucketAction Action = "s3:DeleteBucket"
|
||||
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
|
||||
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
|
||||
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
|
||||
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
|
||||
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
|
||||
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
|
||||
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
|
||||
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
|
||||
PutObjectAction Action = "s3:PutObject"
|
||||
GetObjectAction Action = "s3:GetObject"
|
||||
DeleteObjectAction Action = "s3:DeleteObject"
|
||||
GetObjectAclAction Action = "s3:GetObjectAcl"
|
||||
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
|
||||
PutObjectAclAction Action = "s3:PutObjectAcl"
|
||||
RestoreObjectAction Action = "s3:RestoreObject"
|
||||
GetBucketTaggingAction Action = "s3:GetBucketTagging"
|
||||
PutBucketTaggingAction Action = "s3:PutBucketTagging"
|
||||
GetObjectTaggingAction Action = "s3:GetObjectTagging"
|
||||
PutObjectTaggingAction Action = "s3:PutObjectTagging"
|
||||
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
|
||||
ListBucketVersionsAction Action = "s3:ListBucketVersions"
|
||||
ListBucketAction Action = "s3:ListBucket"
|
||||
AllActions Action = "s3:*"
|
||||
)
|
||||
|
||||
var supportedActionList = map[Action]struct{}{
|
||||
GetBucketAclAction: {},
|
||||
CreateBucketAction: {},
|
||||
PutBucketAclAction: {},
|
||||
DeleteBucketAction: {},
|
||||
PutBucketVersioningAction: {},
|
||||
GetBucketVersioningAction: {},
|
||||
PutBucketPolicyAction: {},
|
||||
GetBucketPolicyAction: {},
|
||||
DeleteBucketPolicyAction: {},
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
ListBucketMultipartUploadsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
GetObjectVersionAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetBucketTaggingAction: {},
|
||||
PutBucketTaggingAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
ListBucketVersionsAction: {},
|
||||
ListBucketAction: {},
|
||||
PutBucketObjectLockConfigurationAction: {},
|
||||
GetObjectLegalHoldAction: {},
|
||||
PutObjectLegalHoldAction: {},
|
||||
GetObjectRetentionAction: {},
|
||||
PutObjectRetentionAction: {},
|
||||
BypassGovernanceRetentionAction: {},
|
||||
PutBucketOwnershipControlsAction: {},
|
||||
GetBucketOwnershipControlsAction: {},
|
||||
PutBucketCorsAction: {},
|
||||
GetBucketCorsAction: {},
|
||||
AllActions: {},
|
||||
GetBucketAclAction: {},
|
||||
CreateBucketAction: {},
|
||||
PutBucketAclAction: {},
|
||||
DeleteBucketAction: {},
|
||||
PutBucketVersioningAction: {},
|
||||
GetBucketVersioningAction: {},
|
||||
PutBucketPolicyAction: {},
|
||||
GetBucketPolicyAction: {},
|
||||
DeleteBucketPolicyAction: {},
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
ListBucketMultipartUploadsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetBucketTaggingAction: {},
|
||||
PutBucketTaggingAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
ListBucketVersionsAction: {},
|
||||
ListBucketAction: {},
|
||||
AllActions: {},
|
||||
}
|
||||
|
||||
var supportedObjectActionList = map[Action]struct{}{
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
GetObjectVersionAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
GetObjectLegalHoldAction: {},
|
||||
PutObjectLegalHoldAction: {},
|
||||
GetObjectRetentionAction: {},
|
||||
PutObjectRetentionAction: {},
|
||||
BypassGovernanceRetentionAction: {},
|
||||
AllActions: {},
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
AllActions: {},
|
||||
}
|
||||
|
||||
// Validates Action: it should either wildcard match with supported actions list or be in it
|
||||
func (a Action) IsValid() error {
|
||||
if !strings.HasPrefix(string(a), "s3:") {
|
||||
return policyErrInvalidAction
|
||||
return fmt.Errorf("invalid action: %v", a)
|
||||
}
|
||||
|
||||
if a == AllActions {
|
||||
@@ -144,39 +116,31 @@ func (a Action) IsValid() error {
|
||||
}
|
||||
}
|
||||
|
||||
return policyErrInvalidAction
|
||||
return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
|
||||
}
|
||||
|
||||
_, found := supportedActionList[a]
|
||||
if !found {
|
||||
return policyErrInvalidAction
|
||||
return fmt.Errorf("unsupported action: %v", a)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getBoolPtr(bl bool) *bool {
|
||||
return &bl
|
||||
}
|
||||
|
||||
// Checks if the action is object action
|
||||
// nil points to 's3:*'
|
||||
func (a Action) IsObjectAction() *bool {
|
||||
if a == AllActions {
|
||||
return nil
|
||||
}
|
||||
func (a Action) IsObjectAction() bool {
|
||||
if a[len(a)-1] == '*' {
|
||||
pattern := strings.TrimSuffix(string(a), "*")
|
||||
for act := range supportedObjectActionList {
|
||||
if strings.HasPrefix(string(act), pattern) {
|
||||
return getBoolPtr(true)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return getBoolPtr(false)
|
||||
return false
|
||||
}
|
||||
|
||||
_, found := supportedObjectActionList[a]
|
||||
return &found
|
||||
return found
|
||||
}
|
||||
|
||||
func (a Action) WildCardMatch(act Action) bool {
|
||||
@@ -195,7 +159,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
|
||||
var err error
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return policyErrInvalidAction
|
||||
return fmt.Errorf("actions can't be empty")
|
||||
}
|
||||
*a = make(Actions)
|
||||
for _, s := range ss {
|
||||
@@ -208,7 +172,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return policyErrInvalidAction
|
||||
return fmt.Errorf("actions can't be empty")
|
||||
}
|
||||
*a = make(Actions)
|
||||
err = a.Add(s)
|
||||
|
||||
@@ -30,6 +30,5 @@ func (bpat BucketPolicyAccessType) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
|
||||
return fmt.Errorf("Invalid effect: %v", bpat)
|
||||
return fmt.Errorf("invalid effect: %v", bpat)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type Principals map[string]struct{}
|
||||
@@ -27,50 +28,23 @@ func (p Principals) Add(key string) {
|
||||
// Override UnmarshalJSON method to decode both []string and string properties
|
||||
func (p *Principals) UnmarshalJSON(data []byte) error {
|
||||
ss := []string{}
|
||||
var s string
|
||||
var k struct {
|
||||
AWS string
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return policyErrInvalidPrincipal
|
||||
return fmt.Errorf("principals can't be empty")
|
||||
}
|
||||
*p = make(Principals)
|
||||
for _, s := range ss {
|
||||
p.Add(s)
|
||||
}
|
||||
return nil
|
||||
} else if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return policyErrInvalidPrincipal
|
||||
}
|
||||
*p = make(Principals)
|
||||
p.Add(s)
|
||||
|
||||
return nil
|
||||
} else if err = json.Unmarshal(data, &k); err == nil {
|
||||
if k.AWS == "" {
|
||||
return policyErrInvalidPrincipal
|
||||
}
|
||||
*p = make(Principals)
|
||||
p.Add(k.AWS)
|
||||
|
||||
return nil
|
||||
} else {
|
||||
var sk struct {
|
||||
AWS []string
|
||||
}
|
||||
if err = json.Unmarshal(data, &sk); err == nil {
|
||||
if len(sk.AWS) == 0 {
|
||||
return policyErrInvalidPrincipal
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return fmt.Errorf("principals can't be empty")
|
||||
}
|
||||
*p = make(Principals)
|
||||
for _, s := range sk.AWS {
|
||||
p.Add(s)
|
||||
}
|
||||
p.Add(s)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,7 +71,7 @@ func (p Principals) Validate(iam IAMService) error {
|
||||
if len(p) == 1 {
|
||||
return nil
|
||||
}
|
||||
return policyErrInvalidPrincipal
|
||||
return fmt.Errorf("principals should either contain * or user access keys")
|
||||
}
|
||||
|
||||
accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
|
||||
@@ -105,7 +79,7 @@ func (p Principals) Validate(iam IAMService) error {
|
||||
return err
|
||||
}
|
||||
if len(accs) > 0 {
|
||||
return policyErrInvalidPrincipal
|
||||
return fmt.Errorf("user accounts don't exist: %v", accs)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -16,6 +16,7 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -29,7 +30,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
|
||||
var err error
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return policyErrInvalidResource
|
||||
return fmt.Errorf("resources can't be empty")
|
||||
}
|
||||
*r = make(Resources)
|
||||
for _, s := range ss {
|
||||
@@ -42,7 +43,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return policyErrInvalidResource
|
||||
return fmt.Errorf("resources can't be empty")
|
||||
}
|
||||
*r = make(Resources)
|
||||
err = r.Add(s)
|
||||
@@ -59,7 +60,12 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
|
||||
func (r Resources) Add(rc string) error {
|
||||
ok, pattern := isValidResource(rc)
|
||||
if !ok {
|
||||
return policyErrInvalidResource
|
||||
return fmt.Errorf("invalid resource: %v", rc)
|
||||
}
|
||||
|
||||
_, found := r[pattern]
|
||||
if found {
|
||||
return fmt.Errorf("duplicate resource: %v", rc)
|
||||
}
|
||||
|
||||
r[pattern] = struct{}{}
|
||||
@@ -93,7 +99,7 @@ func (r Resources) ContainsBucketPattern() bool {
|
||||
func (r Resources) Validate(bucket string) error {
|
||||
for resource := range r {
|
||||
if !strings.HasPrefix(resource, bucket) {
|
||||
return policyErrInvalidResource
|
||||
return fmt.Errorf("incorrect bucket name in %v", resource)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,45 +108,21 @@ func (r Resources) Validate(bucket string) error {
|
||||
|
||||
func (r Resources) FindMatch(resource string) bool {
|
||||
for res := range r {
|
||||
if r.Match(res, resource) {
|
||||
return true
|
||||
if strings.HasSuffix(res, "*") {
|
||||
pattern := strings.TrimSuffix(res, "*")
|
||||
if strings.HasPrefix(resource, pattern) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
if res == resource {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Match checks if the input string matches the given pattern with wildcards (`*`, `?`).
|
||||
// - `?` matches exactly one occurrence of any character.
|
||||
// - `*` matches arbitrary many (including zero) occurrences of any character.
|
||||
func (r Resources) Match(pattern, input string) bool {
|
||||
pIdx, sIdx := 0, 0
|
||||
starIdx, matchIdx := -1, 0
|
||||
|
||||
for sIdx < len(input) {
|
||||
if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
|
||||
sIdx++
|
||||
pIdx++
|
||||
} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
|
||||
starIdx = pIdx
|
||||
matchIdx = sIdx
|
||||
pIdx++
|
||||
} else if starIdx != -1 {
|
||||
pIdx = starIdx + 1
|
||||
matchIdx++
|
||||
sIdx = matchIdx
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
for pIdx < len(pattern) && pattern[pIdx] == '*' {
|
||||
pIdx++
|
||||
}
|
||||
|
||||
return pIdx == len(pattern)
|
||||
}
|
||||
|
||||
// Checks the resource to have arn prefix and not starting with /
|
||||
func isValidResource(rc string) (isValid bool, pattern string) {
|
||||
if !strings.HasPrefix(rc, ResourceArnPrefix) {
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnmarshalJSON(t *testing.T) {
|
||||
var r Resources
|
||||
|
||||
cases := []struct {
|
||||
input string
|
||||
expected int
|
||||
wantErr bool
|
||||
}{
|
||||
{`"arn:aws:s3:::my-bucket/*"`, 1, false},
|
||||
{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
|
||||
{`""`, 0, true},
|
||||
{`[]`, 0, true},
|
||||
{`["invalid-bucket"]`, 0, true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
r = Resources{}
|
||||
err := json.Unmarshal([]byte(tc.input), &r)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
|
||||
}
|
||||
if len(r) != tc.expected {
|
||||
t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdd(t *testing.T) {
|
||||
r := Resources{}
|
||||
|
||||
cases := []struct {
|
||||
input string
|
||||
wantErr bool
|
||||
}{
|
||||
{"arn:aws:s3:::valid-bucket/*", false},
|
||||
{"arn:aws:s3:::valid-bucket/object", false},
|
||||
{"invalid-bucket/*", true},
|
||||
{"/invalid-start", true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
err := r.Add(tc.input)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsObjectPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
resources []string
|
||||
expected bool
|
||||
}{
|
||||
{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
|
||||
{[]string{"arn:aws:s3:::my-bucket/*"}, true},
|
||||
{[]string{"arn:aws:s3:::my-bucket"}, false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
r := Resources{}
|
||||
for _, res := range tc.resources {
|
||||
r.Add(res)
|
||||
}
|
||||
if r.ContainsObjectPattern() != tc.expected {
|
||||
t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsBucketPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
resources []string
|
||||
expected bool
|
||||
}{
|
||||
{[]string{"arn:aws:s3:::my-bucket"}, true},
|
||||
{[]string{"arn:aws:s3:::my-bucket/*"}, false},
|
||||
{[]string{"arn:aws:s3:::my-bucket/object"}, false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
r := Resources{}
|
||||
for _, res := range tc.resources {
|
||||
r.Add(res)
|
||||
}
|
||||
if r.ContainsBucketPattern() != tc.expected {
|
||||
t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
cases := []struct {
|
||||
resources []string
|
||||
bucket string
|
||||
expected bool
|
||||
}{
|
||||
{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
|
||||
{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
|
||||
{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
r := Resources{}
|
||||
for _, res := range tc.resources {
|
||||
r.Add(res)
|
||||
}
|
||||
if (r.Validate(tc.bucket) == nil) != tc.expected {
|
||||
t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindMatch(t *testing.T) {
|
||||
cases := []struct {
|
||||
resources []string
|
||||
input string
|
||||
expected bool
|
||||
}{
|
||||
{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
|
||||
{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
|
||||
{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
|
||||
{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
r := Resources{}
|
||||
for _, res := range tc.resources {
|
||||
r.Add(res)
|
||||
}
|
||||
if r.FindMatch(tc.input) != tc.expected {
|
||||
t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatch(t *testing.T) {
|
||||
r := Resources{}
|
||||
cases := []struct {
|
||||
pattern string
|
||||
input string
|
||||
expected bool
|
||||
}{
|
||||
{"my-bucket/*", "my-bucket/object", true},
|
||||
{"my-bucket/?bject", "my-bucket/object", true},
|
||||
{"my-bucket/*", "other-bucket/object", false},
|
||||
{"*", "any-bucket/object", true},
|
||||
{"my-bucket/*", "my-bucket/subdir/object", true},
|
||||
{"my-bucket/*", "other-bucket", false},
|
||||
{"my-bucket/*/*", "my-bucket/hello", false},
|
||||
{"my-bucket/*/*", "my-bucket/hello/world", true},
|
||||
{"foo/???/bar", "foo/qux/bar", true},
|
||||
{"foo/???/bar", "foo/quxx/bar", false},
|
||||
{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
|
||||
{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if r.Match(tc.pattern, tc.input) != tc.expected {
|
||||
t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
131
auth/iam.go
131
auth/iam.go
@@ -28,49 +28,14 @@ const (
|
||||
RoleUserPlus Role = "userplus"
|
||||
)
|
||||
|
||||
func (r Role) IsValid() bool {
|
||||
switch r {
|
||||
case RoleAdmin:
|
||||
return true
|
||||
case RoleUser:
|
||||
return true
|
||||
case RoleUserPlus:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Account is a gateway IAM account
|
||||
type Account struct {
|
||||
Access string `json:"access"`
|
||||
Secret string `json:"secret"`
|
||||
Role Role `json:"role"`
|
||||
UserID int `json:"userID"`
|
||||
GroupID int `json:"groupID"`
|
||||
}
|
||||
|
||||
type ListUserAccountsResult struct {
|
||||
Accounts []Account
|
||||
}
|
||||
|
||||
// Mutable props, which could be changed when updating an IAM account
|
||||
type MutableProps struct {
|
||||
Secret *string `json:"secret"`
|
||||
UserID *int `json:"userID"`
|
||||
GroupID *int `json:"groupID"`
|
||||
}
|
||||
|
||||
func updateAcc(acc *Account, props MutableProps) {
|
||||
if props.Secret != nil {
|
||||
acc.Secret = *props.Secret
|
||||
}
|
||||
if props.GroupID != nil {
|
||||
acc.GroupID = *props.GroupID
|
||||
}
|
||||
if props.UserID != nil {
|
||||
acc.UserID = *props.UserID
|
||||
}
|
||||
Access string `json:"access"`
|
||||
Secret string `json:"secret"`
|
||||
Role Role `json:"role"`
|
||||
UserID int `json:"userID"`
|
||||
GroupID int `json:"groupID"`
|
||||
ProjectID int `json:"projectID"`
|
||||
}
|
||||
|
||||
// IAMService is the interface for all IAM service implementations
|
||||
@@ -79,57 +44,33 @@ func updateAcc(acc *Account, props MutableProps) {
|
||||
type IAMService interface {
|
||||
CreateAccount(account Account) error
|
||||
GetUserAccount(access string) (Account, error)
|
||||
UpdateUserAccount(access string, props MutableProps) error
|
||||
DeleteUserAccount(access string) error
|
||||
ListUserAccounts() ([]Account, error)
|
||||
Shutdown() error
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrUserExists is returned when the user already exists
|
||||
ErrUserExists = errors.New("user already exists")
|
||||
// ErrNoSuchUser is returned when the user does not exist
|
||||
ErrNoSuchUser = errors.New("user not found")
|
||||
)
|
||||
var ErrNoSuchUser = errors.New("user not found")
|
||||
|
||||
type Opts struct {
|
||||
RootAccount Account
|
||||
Dir string
|
||||
LDAPServerURL string
|
||||
LDAPBindDN string
|
||||
LDAPPassword string
|
||||
LDAPQueryBase string
|
||||
LDAPObjClasses string
|
||||
LDAPAccessAtr string
|
||||
LDAPSecretAtr string
|
||||
LDAPRoleAtr string
|
||||
LDAPUserIdAtr string
|
||||
LDAPGroupIdAtr string
|
||||
VaultEndpointURL string
|
||||
VaultSecretStoragePath string
|
||||
VaultMountPath string
|
||||
VaultRootToken string
|
||||
VaultRoleId string
|
||||
VaultRoleSecret string
|
||||
VaultServerCert string
|
||||
VaultClientCert string
|
||||
VaultClientCertKey string
|
||||
S3Access string
|
||||
S3Secret string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3Endpoint string
|
||||
S3DisableSSlVerfiy bool
|
||||
S3Debug bool
|
||||
CacheDisable bool
|
||||
CacheTTL int
|
||||
CachePrune int
|
||||
IpaHost string
|
||||
IpaVaultName string
|
||||
IpaUser string
|
||||
IpaPassword string
|
||||
IpaInsecure bool
|
||||
IpaDebug bool
|
||||
Dir string
|
||||
LDAPServerURL string
|
||||
LDAPBindDN string
|
||||
LDAPPassword string
|
||||
LDAPQueryBase string
|
||||
LDAPObjClasses string
|
||||
LDAPAccessAtr string
|
||||
LDAPSecretAtr string
|
||||
LDAPRoleAtr string
|
||||
S3Access string
|
||||
S3Secret string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3Endpoint string
|
||||
S3DisableSSlVerfiy bool
|
||||
S3Debug bool
|
||||
CacheDisable bool
|
||||
CacheTTL int
|
||||
CachePrune int
|
||||
}
|
||||
|
||||
func New(o *Opts) (IAMService, error) {
|
||||
@@ -138,30 +79,22 @@ func New(o *Opts) (IAMService, error) {
|
||||
|
||||
switch {
|
||||
case o.Dir != "":
|
||||
svc, err = NewInternal(o.RootAccount, o.Dir)
|
||||
svc, err = NewInternal(o.Dir)
|
||||
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
|
||||
case o.LDAPServerURL != "":
|
||||
svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
|
||||
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
|
||||
o.LDAPGroupIdAtr, o.LDAPObjClasses)
|
||||
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
|
||||
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
|
||||
o.LDAPObjClasses)
|
||||
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
|
||||
case o.S3Endpoint != "":
|
||||
svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
|
||||
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
|
||||
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
|
||||
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
|
||||
o.S3Endpoint, o.S3Bucket)
|
||||
case o.VaultEndpointURL != "":
|
||||
svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
|
||||
o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
|
||||
o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
|
||||
fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
|
||||
case o.IpaHost != "":
|
||||
svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure, o.IpaDebug)
|
||||
fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
|
||||
default:
|
||||
// if no iam options selected, default to the single user mode
|
||||
fmt.Println("No IAM service configured, enabling single account mode")
|
||||
return NewIAMServiceSingle(o.RootAccount), nil
|
||||
return IAMServiceSingle{}, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -66,21 +66,6 @@ func (i *icache) get(k string) (Account, bool) {
|
||||
return v.value, true
|
||||
}
|
||||
|
||||
func (i *icache) update(k string, props MutableProps) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
item, found := i.items[k]
|
||||
if found {
|
||||
updateAcc(&item.value, props)
|
||||
|
||||
// refresh the expiration date
|
||||
item.exp = time.Now().Add(i.expire)
|
||||
|
||||
i.items[k] = item
|
||||
}
|
||||
}
|
||||
|
||||
func (i *icache) Delete(k string) {
|
||||
i.Lock()
|
||||
delete(i.items, k)
|
||||
@@ -181,16 +166,6 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
|
||||
err := c.service.UpdateUserAccount(access, props)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.iamcache.update(access, props)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListUserAccounts is a passthrough to the underlying service and
|
||||
// does not make use of the cache
|
||||
func (c *IAMCache) ListUserAccounts() ([]Account, error) {
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -33,15 +32,7 @@ const (
|
||||
|
||||
// IAMServiceInternal manages the internal IAM service
|
||||
type IAMServiceInternal struct {
|
||||
// This mutex will help with racing updates to the IAM data
|
||||
// from multiple requests to this gateway instance, but
|
||||
// will not help with racing updates to multiple load balanced
|
||||
// gateway instances. This is a limitation of the internal
|
||||
// IAM service. All account updates should be sent to a single
|
||||
// gateway instance if possible.
|
||||
sync.RWMutex
|
||||
dir string
|
||||
rootAcc Account
|
||||
dir string
|
||||
}
|
||||
|
||||
// UpdateAcctFunc accepts the current data and returns the new data to be stored
|
||||
@@ -55,10 +46,9 @@ type iAMConfig struct {
|
||||
var _ IAMService = &IAMServiceInternal{}
|
||||
|
||||
// NewInternal creates a new instance for the Internal IAM service
|
||||
func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
|
||||
func NewInternal(dir string) (*IAMServiceInternal, error) {
|
||||
i := &IAMServiceInternal{
|
||||
dir: dir,
|
||||
rootAcc: rootAcc,
|
||||
dir: dir,
|
||||
}
|
||||
|
||||
err := i.initIAM()
|
||||
@@ -72,13 +62,6 @@ func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
|
||||
// CreateAccount creates a new IAM account. Returns an error if the account
|
||||
// already exists.
|
||||
func (s *IAMServiceInternal) CreateAccount(account Account) error {
|
||||
if account.Access == s.rootAcc.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
@@ -87,7 +70,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
|
||||
|
||||
_, ok := conf.AccessAccounts[account.Access]
|
||||
if ok {
|
||||
return nil, ErrUserExists
|
||||
return nil, fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[account.Access] = account
|
||||
|
||||
@@ -103,13 +86,6 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
|
||||
// GetUserAccount retrieves account info for the requested user. Returns
|
||||
// ErrNoSuchUser if the account does not exist.
|
||||
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
|
||||
if access == s.rootAcc.Access {
|
||||
return s.rootAcc, nil
|
||||
}
|
||||
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getIAM()
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("get iam data: %w", err)
|
||||
@@ -123,41 +99,9 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
// UpdateUserAccount updates the specified user account fields. Returns
|
||||
// ErrNoSuchUser if the account does not exist.
|
||||
func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
acc, found := conf.AccessAccounts[access]
|
||||
if !found {
|
||||
return nil, ErrNoSuchUser
|
||||
}
|
||||
|
||||
updateAcc(&acc, props)
|
||||
conf.AccessAccounts[access] = acc
|
||||
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteUserAccount deletes the specified user account. Does not check if
|
||||
// account exists.
|
||||
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
@@ -177,9 +121,6 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
|
||||
|
||||
// ListUserAccounts lists all the user accounts stored.
|
||||
func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getIAM()
|
||||
if err != nil {
|
||||
return []Account{}, fmt.Errorf("get iam data: %w", err)
|
||||
@@ -194,11 +135,12 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
|
||||
var accs []Account
|
||||
for _, k := range keys {
|
||||
accs = append(accs, Account{
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
ProjectID: conf.AccessAccounts[k].ProjectID,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -247,10 +189,6 @@ func parseIAM(b []byte) (iAMConfig, error) {
|
||||
return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
|
||||
}
|
||||
|
||||
if conf.AccessAccounts == nil {
|
||||
conf.AccessAccounts = make(map[string]Account)
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
|
||||
446
auth/iam_ipa.go
446
auth/iam_ipa.go
@@ -1,446 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const IpaVersion = "2.254"
|
||||
|
||||
type IpaIAMService struct {
|
||||
client http.Client
|
||||
id int
|
||||
version string
|
||||
host string
|
||||
vaultName string
|
||||
username string
|
||||
password string
|
||||
kraTransportKey *rsa.PublicKey
|
||||
debug bool
|
||||
rootAcc Account
|
||||
}
|
||||
|
||||
var _ IAMService = &IpaIAMService{}
|
||||
|
||||
func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
|
||||
|
||||
ipa := IpaIAMService{
|
||||
id: 0,
|
||||
version: IpaVersion,
|
||||
host: host,
|
||||
vaultName: vaultName,
|
||||
username: username,
|
||||
password: password,
|
||||
debug: debug,
|
||||
rootAcc: rootAcc,
|
||||
}
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
// this should never happen
|
||||
return nil, fmt.Errorf("cookie jar creation: %w", err)
|
||||
}
|
||||
|
||||
mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: mTLSConfig,
|
||||
}
|
||||
ipa.client = http.Client{Jar: jar, Transport: tr}
|
||||
|
||||
err = ipa.login()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa login failed: %w", err)
|
||||
}
|
||||
|
||||
req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
|
||||
}
|
||||
vaultConfig := struct {
|
||||
Kra_Server_Server []string
|
||||
Transport_Cert Base64EncodedWrapped
|
||||
Wrapping_default_algorithm string
|
||||
Wrapping_supported_algorithms []string
|
||||
}{}
|
||||
err = ipa.rpc(req, &vaultConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa vault config: %w", err)
|
||||
}
|
||||
|
||||
cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
|
||||
}
|
||||
|
||||
ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
|
||||
|
||||
isSupported := false
|
||||
for _, algo := range vaultConfig.Wrapping_supported_algorithms {
|
||||
if algo == "aes-128-cbc" {
|
||||
isSupported = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !isSupported {
|
||||
return nil,
|
||||
fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
|
||||
vaultConfig.Wrapping_supported_algorithms)
|
||||
}
|
||||
return &ipa, nil
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) CreateAccount(account Account) error {
|
||||
return fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
|
||||
if access == ipa.rootAcc.Access {
|
||||
return ipa.rootAcc, nil
|
||||
}
|
||||
|
||||
req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa user_show: %w", err)
|
||||
}
|
||||
|
||||
userResult := struct {
|
||||
Gidnumber []string
|
||||
Uidnumber []string
|
||||
}{}
|
||||
|
||||
err = ipa.rpc(req, &userResult)
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
uid, err := strconv.Atoi(userResult.Uidnumber[0])
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
|
||||
}
|
||||
gid, err := strconv.Atoi(userResult.Gidnumber[0])
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
|
||||
}
|
||||
|
||||
account := Account{
|
||||
Access: access,
|
||||
Role: RoleUser,
|
||||
UserID: uid,
|
||||
GroupID: gid,
|
||||
}
|
||||
|
||||
session_key := make([]byte, 16)
|
||||
|
||||
_, err = rand.Read(session_key)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa cannot generate session key: %w", err)
|
||||
}
|
||||
|
||||
encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
|
||||
}
|
||||
|
||||
req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
|
||||
map[string]any{"username": access,
|
||||
"session_key": Base64EncodedWrapped(encryptedKey),
|
||||
"wrapping_algo": "aes-128-cbc"})
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
|
||||
}
|
||||
|
||||
data := struct {
|
||||
Vault_data Base64EncodedWrapped
|
||||
Nonce Base64EncodedWrapped
|
||||
}{}
|
||||
|
||||
err = ipa.rpc(req, &data)
|
||||
if err != nil {
|
||||
return account, err
|
||||
}
|
||||
|
||||
aes, err := aes.NewCipher(session_key)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
|
||||
}
|
||||
cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
|
||||
cbc.CryptBlocks(data.Vault_data, data.Vault_data)
|
||||
secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
|
||||
}
|
||||
|
||||
secret := struct {
|
||||
Data Base64Encoded
|
||||
}{}
|
||||
json.Unmarshal(secretUnpaddedJson, &secret)
|
||||
account.Secret = string(secret.Data)
|
||||
|
||||
return account, nil
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
|
||||
return fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
|
||||
return fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
|
||||
return []Account{}, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Implementation
|
||||
|
||||
func (ipa *IpaIAMService) login() error {
|
||||
form := url.Values{}
|
||||
form.Set("user", ipa.username)
|
||||
form.Set("password", ipa.password)
|
||||
|
||||
req, err := http.NewRequest(
|
||||
"POST",
|
||||
fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
|
||||
strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
resp, err := ipa.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode == 401 {
|
||||
return errors.New("cannot login to FreeIPA: invalid credentials")
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type rpcRequest = string
|
||||
|
||||
type rpcResponse struct {
|
||||
Result json.RawMessage
|
||||
Principal string
|
||||
Id int
|
||||
Version string
|
||||
}
|
||||
|
||||
func (p rpcResponse) String() string {
|
||||
return string(p.Result)
|
||||
}
|
||||
|
||||
var errRpc = errors.New("IPA RPC error")
|
||||
|
||||
func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
|
||||
err := ipa.login()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res, err := ipa.rpcInternal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return json.Unmarshal(res.Result, value)
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
|
||||
httpReq, err := http.NewRequest("POST",
|
||||
fmt.Sprintf("%s/ipa/session/json", ipa.host),
|
||||
strings.NewReader(req))
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
|
||||
ipa.log(fmt.Sprintf("%v", req))
|
||||
httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
httpResp, err := ipa.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
|
||||
bytes, err := io.ReadAll(httpResp.Body)
|
||||
ipa.log(string(bytes))
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
|
||||
result := struct {
|
||||
Result struct {
|
||||
Json json.RawMessage `json:"result"`
|
||||
Value string `json:"value"`
|
||||
Summary any `json:"summary"`
|
||||
} `json:"result"`
|
||||
Error json.RawMessage `json:"error"`
|
||||
Id int `json:"id"`
|
||||
Principal string `json:"principal"`
|
||||
Version string `json:"version"`
|
||||
}{}
|
||||
|
||||
err = json.Unmarshal(bytes, &result)
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
if string(result.Error) != "null" {
|
||||
return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
|
||||
}
|
||||
|
||||
return rpcResponse{
|
||||
Result: result.Result.Json,
|
||||
Principal: result.Principal,
|
||||
Id: result.Id,
|
||||
Version: result.Version,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
|
||||
|
||||
id := ipa.id
|
||||
ipa.id++
|
||||
|
||||
dict["version"] = ipa.version
|
||||
|
||||
jmethod, errMethod := json.Marshal(method)
|
||||
jargs, errArgs := json.Marshal(args)
|
||||
jdict, errDict := json.Marshal(dict)
|
||||
|
||||
err := errors.Join(errMethod, errArgs, errDict)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ipa request invalid: %w", err)
|
||||
}
|
||||
|
||||
request := map[string]interface{}{
|
||||
"id": id,
|
||||
"method": json.RawMessage(jmethod),
|
||||
"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
|
||||
}
|
||||
|
||||
requestJSON, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
return string(requestJSON), nil
|
||||
}
|
||||
|
||||
// pkcs7Unpad validates and unpads data from the given bytes slice.
|
||||
// The returned value will be 1 to n bytes smaller depending on the
|
||||
// amount of padding, where n is the block size.
|
||||
func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
|
||||
if blocksize <= 0 {
|
||||
return nil, errors.New("invalid blocksize")
|
||||
}
|
||||
if len(b) == 0 {
|
||||
return nil, errors.New("invalid PKCS7 data (empty or not padded)")
|
||||
}
|
||||
if len(b)%blocksize != 0 {
|
||||
return nil, errors.New("invalid padding on input")
|
||||
}
|
||||
c := b[len(b)-1]
|
||||
n := int(c)
|
||||
if n == 0 || n > len(b) {
|
||||
return nil, errors.New("invalid padding on input")
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if b[len(b)-n+i] != c {
|
||||
return nil, errors.New("invalid padding on input")
|
||||
}
|
||||
}
|
||||
return b[:len(b)-n], nil
|
||||
}
|
||||
|
||||
/*
|
||||
e.g.
|
||||
|
||||
"value" {
|
||||
"__base64__": "aGVsbG93b3JsZAo="
|
||||
}
|
||||
*/
|
||||
type Base64EncodedWrapped []byte
|
||||
|
||||
func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
|
||||
intermediate := struct {
|
||||
Base64 string `json:"__base64__"`
|
||||
}{}
|
||||
err := json.Unmarshal(data, &intermediate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
|
||||
intermediate := struct {
|
||||
Base64 string `json:"__base64__"`
|
||||
}{Base64: base64.StdEncoding.EncodeToString(*b)}
|
||||
return json.Marshal(intermediate)
|
||||
}
|
||||
|
||||
/*
|
||||
e.g.
|
||||
|
||||
"value": "aGVsbG93b3JsZAo="
|
||||
*/
|
||||
type Base64Encoded []byte
|
||||
|
||||
func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
|
||||
var intermediate string
|
||||
err := json.Unmarshal(data, &intermediate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*b, err = base64.StdEncoding.DecodeString(intermediate)
|
||||
return err
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) log(msg string) {
|
||||
if ipa.debug {
|
||||
log.Println(msg)
|
||||
}
|
||||
}
|
||||
102
auth/iam_ldap.go
102
auth/iam_ldap.go
@@ -1,22 +1,7 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-ldap/ldap/v3"
|
||||
@@ -29,19 +14,15 @@ type LdapIAMService struct {
|
||||
accessAtr string
|
||||
secretAtr string
|
||||
roleAtr string
|
||||
groupIdAtr string
|
||||
userIdAtr string
|
||||
rootAcc Account
|
||||
}
|
||||
|
||||
var _ IAMService = &LdapIAMService{}
|
||||
|
||||
func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
|
||||
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
|
||||
secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
|
||||
func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
|
||||
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
|
||||
return nil, fmt.Errorf("required parameters list not fully provided")
|
||||
}
|
||||
conn, err := ldap.DialURL(url)
|
||||
conn, err := ldap.Dial("tcp", url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
|
||||
}
|
||||
@@ -57,23 +38,15 @@ func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAt
|
||||
accessAtr: accAtr,
|
||||
secretAtr: secAtr,
|
||||
roleAtr: roleAtr,
|
||||
userIdAtr: userIdAtr,
|
||||
groupIdAtr: groupIdAtr,
|
||||
rootAcc: rootAcc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) CreateAccount(account Account) error {
|
||||
if ld.rootAcc.Access == account.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
|
||||
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
|
||||
userEntry.Attribute("objectClass", ld.objClasses)
|
||||
userEntry.Attribute(ld.accessAtr, []string{account.Access})
|
||||
userEntry.Attribute(ld.secretAtr, []string{account.Secret})
|
||||
userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
|
||||
userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
|
||||
userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
|
||||
|
||||
err := ld.conn.Add(userEntry)
|
||||
if err != nil {
|
||||
@@ -84,9 +57,6 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
|
||||
if access == ld.rootAcc.Access {
|
||||
return ld.rootAcc, nil
|
||||
}
|
||||
searchRequest := ldap.NewSearchRequest(
|
||||
ld.queryBase,
|
||||
ldap.ScopeWholeSubtree,
|
||||
@@ -95,7 +65,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
|
||||
0,
|
||||
false,
|
||||
fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
|
||||
nil,
|
||||
)
|
||||
|
||||
@@ -104,50 +74,14 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
if len(result.Entries) == 0 {
|
||||
return Account{}, ErrNoSuchUser
|
||||
}
|
||||
|
||||
entry := result.Entries[0]
|
||||
groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("invalid entry value for group-id %q: %w",
|
||||
entry.GetAttributeValue(ld.groupIdAtr), err)
|
||||
}
|
||||
userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("invalid entry value for user-id %q: %w",
|
||||
entry.GetAttributeValue(ld.userIdAtr), err)
|
||||
}
|
||||
return Account{
|
||||
Access: entry.GetAttributeValue(ld.accessAtr),
|
||||
Secret: entry.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
|
||||
GroupID: groupId,
|
||||
UserID: userId,
|
||||
Access: entry.GetAttributeValue(ld.accessAtr),
|
||||
Secret: entry.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
|
||||
req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
|
||||
if props.Secret != nil {
|
||||
req.Replace(ld.secretAtr, []string{*props.Secret})
|
||||
}
|
||||
if props.GroupID != nil {
|
||||
req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
|
||||
}
|
||||
if props.UserID != nil {
|
||||
req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
|
||||
}
|
||||
|
||||
err := ld.conn.Modify(req)
|
||||
//TODO: Handle non existing user case
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) DeleteUserAccount(access string) error {
|
||||
delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
|
||||
|
||||
@@ -172,7 +106,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
|
||||
0,
|
||||
false,
|
||||
fmt.Sprintf("(&%v)", searchFilter),
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
|
||||
nil,
|
||||
)
|
||||
|
||||
@@ -183,22 +117,10 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
|
||||
|
||||
result := []Account{}
|
||||
for _, el := range resp.Entries {
|
||||
groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid entry value for group-id %q: %w",
|
||||
el.GetAttributeValue(ld.groupIdAtr), err)
|
||||
}
|
||||
userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid entry value for user-id %q: %w",
|
||||
el.GetAttributeValue(ld.userIdAtr), err)
|
||||
}
|
||||
result = append(result, Account{
|
||||
Access: el.GetAttributeValue(ld.accessAtr),
|
||||
Secret: el.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(el.GetAttributeValue(ld.roleAtr)),
|
||||
GroupID: groupId,
|
||||
UserID: userId,
|
||||
Access: el.GetAttributeValue(ld.accessAtr),
|
||||
Secret: el.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(el.GetAttributeValue(ld.roleAtr)),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
@@ -42,14 +41,6 @@ import (
|
||||
// coming from iAMConfig and iamFile in iam_internal.
|
||||
|
||||
type IAMServiceS3 struct {
|
||||
// This mutex will help with racing updates to the IAM data
|
||||
// from multiple requests to this gateway instance, but
|
||||
// will not help with racing updates to multiple load balanced
|
||||
// gateway instances. This is a limitation of the internal
|
||||
// IAM service. All account updates should be sent to a single
|
||||
// gateway instance if possible.
|
||||
sync.RWMutex
|
||||
|
||||
access string
|
||||
secret string
|
||||
region string
|
||||
@@ -57,13 +48,12 @@ type IAMServiceS3 struct {
|
||||
endpoint string
|
||||
sslSkipVerify bool
|
||||
debug bool
|
||||
rootAcc Account
|
||||
client *s3.Client
|
||||
}
|
||||
|
||||
var _ IAMService = &IAMServiceS3{}
|
||||
|
||||
func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
|
||||
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
|
||||
if access == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service access key")
|
||||
}
|
||||
@@ -88,7 +78,6 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
|
||||
endpoint: endpoint,
|
||||
sslSkipVerify: sslSkipVerify,
|
||||
debug: debug,
|
||||
rootAcc: rootAcc,
|
||||
}
|
||||
|
||||
cfg, err := i.getConfig()
|
||||
@@ -96,25 +85,11 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
|
||||
return nil, fmt.Errorf("init s3 IAM: %v", err)
|
||||
}
|
||||
|
||||
if endpoint != "" {
|
||||
i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = &endpoint
|
||||
})
|
||||
return i, nil
|
||||
}
|
||||
|
||||
i.client = s3.NewFromConfig(cfg)
|
||||
return i, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
if s.rootAcc.Access == account.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -122,7 +97,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
|
||||
_, ok := conf.AccessAccounts[account.Access]
|
||||
if ok {
|
||||
return ErrUserExists
|
||||
return fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[account.Access] = account
|
||||
|
||||
@@ -130,13 +105,6 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
|
||||
if access == s.rootAcc.Access {
|
||||
return s.rootAcc, nil
|
||||
}
|
||||
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
@@ -150,30 +118,7 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
acc, ok := conf.AccessAccounts[access]
|
||||
if !ok {
|
||||
return ErrNoSuchUser
|
||||
}
|
||||
|
||||
updateAcc(&acc, props)
|
||||
conf.AccessAccounts[access] = acc
|
||||
|
||||
return s.storeAccts(conf)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -189,9 +134,6 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -206,17 +148,28 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
|
||||
var accs []Account
|
||||
for _, k := range keys {
|
||||
accs = append(accs, Account{
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
ProjectID: conf.AccessAccounts[k].ProjectID,
|
||||
})
|
||||
}
|
||||
|
||||
return accs, nil
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
SigningRegion: s.region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
@@ -235,6 +188,11 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(s))
|
||||
}
|
||||
|
||||
if s.debug {
|
||||
opts = append(opts,
|
||||
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
|
||||
@@ -252,15 +210,15 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
|
||||
})
|
||||
if err != nil {
|
||||
// if the error is object not exists,
|
||||
// init empty accounts struct and return that
|
||||
// init empty accounts stuct and return that
|
||||
var nsk *types.NoSuchKey
|
||||
if errors.As(err, &nsk) {
|
||||
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
if apiErr.ErrorCode() == "NotFound" {
|
||||
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,49 +15,34 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// IAMServiceSingle manages the single tenant (root-only) IAM service
|
||||
type IAMServiceSingle struct {
|
||||
root Account
|
||||
}
|
||||
type IAMServiceSingle struct{}
|
||||
|
||||
var _ IAMService = &IAMServiceSingle{}
|
||||
|
||||
func NewIAMServiceSingle(r Account) IAMService {
|
||||
return &IAMServiceSingle{
|
||||
root: r,
|
||||
}
|
||||
}
|
||||
var ErrNotSupported = errors.New("method is not supported")
|
||||
|
||||
// CreateAccount not valid in single tenant mode
|
||||
func (IAMServiceSingle) CreateAccount(account Account) error {
|
||||
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
// GetUserAccount returns root account, if the root access key
|
||||
// is provided and "ErrAdminUserNotFound" otherwise
|
||||
func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) {
|
||||
if access == s.root.Access {
|
||||
return s.root, nil
|
||||
}
|
||||
return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound)
|
||||
}
|
||||
|
||||
// UpdateUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
|
||||
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
// GetUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
|
||||
return Account{}, ErrNotSupported
|
||||
}
|
||||
|
||||
// DeleteUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) DeleteUserAccount(access string) error {
|
||||
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
// ListUserAccounts no accounts in single tenant mode
|
||||
func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
|
||||
return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
return []Account{}, nil
|
||||
}
|
||||
|
||||
// Shutdown graceful termination of service
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
vault "github.com/hashicorp/vault-client-go"
|
||||
"github.com/hashicorp/vault-client-go/schema"
|
||||
)
|
||||
|
||||
type VaultIAMService struct {
|
||||
client *vault.Client
|
||||
reqOpts []vault.RequestOption
|
||||
secretStoragePath string
|
||||
rootAcc Account
|
||||
}
|
||||
|
||||
var _ IAMService = &VaultIAMService{}
|
||||
|
||||
func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
|
||||
opts := []vault.ClientOption{
|
||||
vault.WithAddress(endpoint),
|
||||
// set request timeout to 10 secs
|
||||
vault.WithRequestTimeout(10 * time.Second),
|
||||
}
|
||||
if serverCert != "" {
|
||||
tls := vault.TLSConfiguration{}
|
||||
|
||||
tls.ServerCertificate.FromBytes = []byte(serverCert)
|
||||
if clientCert != "" {
|
||||
if clientCertKey == "" {
|
||||
return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
|
||||
}
|
||||
|
||||
tls.ClientCertificate.FromBytes = []byte(clientCert)
|
||||
tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
|
||||
}
|
||||
|
||||
opts = append(opts, vault.WithTLS(tls))
|
||||
}
|
||||
|
||||
client, err := vault.New(opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init vault client: %w", err)
|
||||
}
|
||||
|
||||
reqOpts := []vault.RequestOption{}
|
||||
// if mount path is not specified, it defaults to "approle"
|
||||
if mountPath != "" {
|
||||
reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
|
||||
}
|
||||
|
||||
// Authentication
|
||||
switch {
|
||||
case rootToken != "":
|
||||
err := client.SetToken(rootToken)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("root token authentication failure: %w", err)
|
||||
}
|
||||
case roleID != "":
|
||||
if roleSecret == "" {
|
||||
return nil, fmt.Errorf("role id and role secret must both be specified")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
|
||||
RoleId: roleID,
|
||||
SecretId: roleSecret,
|
||||
}, reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("approle authentication failure: %w", err)
|
||||
}
|
||||
|
||||
if err := client.SetToken(resp.Auth.ClientToken); err != nil {
|
||||
return nil, fmt.Errorf("approle authentication set token failure: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
|
||||
}
|
||||
|
||||
return &VaultIAMService{
|
||||
client: client,
|
||||
reqOpts: reqOpts,
|
||||
secretStoragePath: secretStoragePath,
|
||||
rootAcc: rootAcc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) CreateAccount(account Account) error {
|
||||
if vt.rootAcc.Access == account.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
|
||||
Data: map[string]any{
|
||||
account.Access: account,
|
||||
},
|
||||
Options: map[string]interface{}{
|
||||
"cas": 0,
|
||||
},
|
||||
}, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "check-and-set") {
|
||||
return ErrUserExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
|
||||
if vt.rootAcc.Access == access {
|
||||
return vt.rootAcc, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
acc, err := parseVaultUserAccount(resp.Data.Data, access)
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
return acc, nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
|
||||
//TODO: We need something like a transaction here ?
|
||||
acc, err := vt.GetUserAccount(access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updateAcc(&acc, props)
|
||||
|
||||
err = vt.DeleteUserAccount(access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = vt.CreateAccount(acc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) DeleteUserAccount(access string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if vault.IsErrorStatus(err, 404) {
|
||||
return []Account{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accs := []Account{}
|
||||
|
||||
for _, acss := range resp.Data.Keys {
|
||||
acc, err := vt.GetUserAccount(acss)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accs = append(accs, acc)
|
||||
}
|
||||
|
||||
return accs, nil
|
||||
}
|
||||
|
||||
// the client doesn't have explicit shutdown, as it uses http.Client
|
||||
func (vt *VaultIAMService) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
|
||||
|
||||
func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
|
||||
usrAcc, ok := data[access].(map[string]interface{})
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
|
||||
acss, ok := usrAcc["access"].(string)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
secret, ok := usrAcc["secret"].(string)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
role, ok := usrAcc["role"].(string)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
userIdJson, ok := usrAcc["userID"].(json.Number)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
userId, err := userIdJson.Int64()
|
||||
if err != nil {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
groupIdJson, ok := usrAcc["groupID"].(json.Number)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
groupId, err := groupIdJson.Int64()
|
||||
if err != nil {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
|
||||
return Account{
|
||||
Access: acss,
|
||||
Secret: secret,
|
||||
Role: Role(role),
|
||||
UserID: int(userId),
|
||||
GroupID: int(groupId),
|
||||
}, nil
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type BucketLockConfig struct {
|
||||
Enabled bool
|
||||
DefaultRetention *types.DefaultRetention
|
||||
CreatedAt *time.Time
|
||||
}
|
||||
|
||||
func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
|
||||
var lockConfig types.ObjectLockConfiguration
|
||||
if err := xml.Unmarshal(input, &lockConfig); err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
config := BucketLockConfig{
|
||||
Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
|
||||
}
|
||||
|
||||
if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
|
||||
retention := lockConfig.Rule.DefaultRetention
|
||||
|
||||
if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
if retention.Years != nil && retention.Days != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
if retention.Days != nil && *retention.Days <= 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
|
||||
}
|
||||
if retention.Years != nil && *retention.Years <= 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
|
||||
}
|
||||
|
||||
config.DefaultRetention = retention
|
||||
now := time.Now()
|
||||
config.CreatedAt = &now
|
||||
}
|
||||
|
||||
return json.Marshal(config)
|
||||
}
|
||||
|
||||
func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
|
||||
var config BucketLockConfig
|
||||
if err := json.Unmarshal(input, &config); err != nil {
|
||||
return nil, fmt.Errorf("parse object lock config: %w", err)
|
||||
}
|
||||
|
||||
result := &types.ObjectLockConfiguration{
|
||||
Rule: &types.ObjectLockRule{
|
||||
DefaultRetention: config.DefaultRetention,
|
||||
},
|
||||
}
|
||||
|
||||
if config.Enabled {
|
||||
result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
|
||||
var retention s3response.PutObjectRetentionInput
|
||||
if err := xml.Unmarshal(input, &retention); err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
if retention.RetainUntilDate.Before(time.Now()) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
|
||||
}
|
||||
switch retention.Mode {
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
default:
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
return json.Marshal(retention)
|
||||
}
|
||||
|
||||
func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
|
||||
var retention types.ObjectLockRetention
|
||||
if err := json.Unmarshal(input, &retention); err != nil {
|
||||
return nil, fmt.Errorf("parse object lock retention: %w", err)
|
||||
}
|
||||
|
||||
return &retention, nil
|
||||
}
|
||||
|
||||
func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResult {
|
||||
if status == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if *status {
|
||||
return &s3response.GetObjectLegalHoldResult{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
}
|
||||
}
|
||||
|
||||
return &s3response.GetObjectLegalHoldResult{
|
||||
Status: types.ObjectLockLegalHoldStatusOff,
|
||||
}
|
||||
}
|
||||
|
||||
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
|
||||
data, err := be.GetObjectLockConfiguration(ctx, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var bucketLockConfig BucketLockConfig
|
||||
if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
|
||||
return fmt.Errorf("parse object lock config: %w", err)
|
||||
}
|
||||
|
||||
if !bucketLockConfig.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
checkDefaultRetention := false
|
||||
|
||||
if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
|
||||
expirationDate := *bucketLockConfig.CreatedAt
|
||||
if bucketLockConfig.DefaultRetention.Days != nil {
|
||||
expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
|
||||
}
|
||||
if bucketLockConfig.DefaultRetention.Years != nil {
|
||||
expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
|
||||
}
|
||||
|
||||
if expirationDate.After(time.Now()) {
|
||||
checkDefaultRetention = true
|
||||
}
|
||||
}
|
||||
|
||||
for _, obj := range objects {
|
||||
var key, versionId string
|
||||
if obj.Key != nil {
|
||||
key = *obj.Key
|
||||
}
|
||||
if obj.VersionId != nil {
|
||||
versionId = *obj.VersionId
|
||||
}
|
||||
checkRetention := true
|
||||
retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
|
||||
checkRetention = false
|
||||
}
|
||||
if err != nil && checkRetention {
|
||||
return err
|
||||
}
|
||||
|
||||
if checkRetention {
|
||||
retention, err := ParseObjectLockRetentionOutput(retentionData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if retention.Mode != "" && retention.RetainUntilDate != nil {
|
||||
if retention.RetainUntilDate.After(time.Now()) {
|
||||
switch retention.Mode {
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
if !bypass {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
} else {
|
||||
policy, err := be.GetBucketPolicy(ctx, bucket)
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checkLegalHold := true
|
||||
|
||||
status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
|
||||
if err != nil {
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
|
||||
checkLegalHold = false
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if checkLegalHold && *status {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
|
||||
if checkDefaultRetention {
|
||||
switch bucketLockConfig.DefaultRetention.Mode {
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
if !bypass {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
} else {
|
||||
policy, err := be.GetBucketPolicy(ctx, bucket)
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
|
||||
|
||||
actual := req.Header.Get("Authorization")
|
||||
if e, a := c.ExpSig, actual; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.OrigURI, req.URL.Path; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
|
||||
|
||||
actual := req.Header.Get("Authorization")
|
||||
if e, a := c.ExpSig, actual; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.OrigURI, req.URL.Path; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,9 +18,9 @@ import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
"github.com/versity/versitygw/s3select"
|
||||
@@ -32,46 +32,40 @@ type Backend interface {
|
||||
Shutdown()
|
||||
|
||||
// bucket operations
|
||||
ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
|
||||
ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
|
||||
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
|
||||
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
|
||||
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
|
||||
PutBucketAcl(_ context.Context, bucket string, data []byte) error
|
||||
DeleteBucket(_ context.Context, bucket string) error
|
||||
PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
|
||||
GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
|
||||
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
|
||||
PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
|
||||
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
|
||||
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
|
||||
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
|
||||
DeleteBucketPolicy(_ context.Context, bucket string) error
|
||||
PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
|
||||
GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
|
||||
DeleteBucketOwnershipControls(_ context.Context, bucket string) error
|
||||
PutBucketCors(context.Context, []byte) error
|
||||
GetBucketCors(_ context.Context, bucket string) ([]byte, error)
|
||||
DeleteBucketCors(_ context.Context, bucket string) error
|
||||
|
||||
// multipart operations
|
||||
CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
|
||||
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
|
||||
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
|
||||
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
|
||||
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
|
||||
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
|
||||
UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
|
||||
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
|
||||
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
|
||||
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
|
||||
|
||||
// standard object operations
|
||||
PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
|
||||
PutObject(context.Context, *s3.PutObjectInput) (string, error)
|
||||
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
|
||||
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
|
||||
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
|
||||
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
|
||||
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
|
||||
CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
|
||||
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
|
||||
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
|
||||
DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
|
||||
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
|
||||
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
|
||||
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
|
||||
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
|
||||
DeleteObject(context.Context, *s3.DeleteObjectInput) error
|
||||
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
|
||||
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
|
||||
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
|
||||
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
|
||||
|
||||
// special case object operations
|
||||
RestoreObject(context.Context, *s3.RestoreObjectInput) error
|
||||
@@ -87,23 +81,11 @@ type Backend interface {
|
||||
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
|
||||
DeleteObjectTagging(_ context.Context, bucket, object string) error
|
||||
|
||||
// object lock operations
|
||||
PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
|
||||
GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
|
||||
PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
|
||||
GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
|
||||
PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
|
||||
GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
|
||||
|
||||
// non AWS actions
|
||||
ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
|
||||
ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
|
||||
ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
|
||||
}
|
||||
|
||||
// InterfaceVersion tracks changes to the Backend interface for plugins.
|
||||
// Increment this when the Backend interface changes.
|
||||
const InterfaceVersion = 1
|
||||
|
||||
type BackendUnsupported struct{}
|
||||
|
||||
var _ Backend = &BackendUnsupported{}
|
||||
@@ -115,7 +97,7 @@ func (BackendUnsupported) Shutdown() {}
|
||||
func (BackendUnsupported) String() string {
|
||||
return "Unsupported"
|
||||
}
|
||||
func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
@@ -130,14 +112,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
|
||||
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
|
||||
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
|
||||
func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
@@ -148,27 +130,9 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
|
||||
func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketCors(context.Context, []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
|
||||
return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
@@ -182,40 +146,40 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
|
||||
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
|
||||
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
|
||||
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
|
||||
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
|
||||
return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
|
||||
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
@@ -241,8 +205,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
|
||||
}
|
||||
}
|
||||
|
||||
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
|
||||
return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
|
||||
@@ -265,26 +229,7 @@ func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
|
||||
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
|
||||
|
||||
@@ -18,8 +18,7 @@ import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"io/fs"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -29,13 +28,9 @@ import (
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
const (
|
||||
// this is the media type for directories in AWS and Nextcloud
|
||||
DirContentType = "application/x-directory"
|
||||
DefaultContentType = "binary/octet-stream"
|
||||
|
||||
// this is the minimum allowed size for mp parts
|
||||
MinPartSize = 5 * 1024 * 1024
|
||||
var (
|
||||
// RFC3339TimeFormat RFC3339 time format
|
||||
RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
|
||||
)
|
||||
|
||||
func IsValidBucketName(name string) bool { return true }
|
||||
@@ -52,196 +47,65 @@ func (d ByObjectName) Len() int { return len(d) }
|
||||
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
||||
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
|
||||
|
||||
func GetPtrFromString(str string) *string {
|
||||
if str == "" {
|
||||
return nil
|
||||
}
|
||||
return &str
|
||||
}
|
||||
|
||||
func GetStringFromPtr(str *string) string {
|
||||
if str == nil {
|
||||
return ""
|
||||
}
|
||||
return *str
|
||||
func GetStringPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
func GetTimePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
|
||||
func TrimEtag(etag *string) *string {
|
||||
if etag == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return GetPtrFromString(strings.Trim(*etag, "\""))
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
|
||||
errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
|
||||
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
|
||||
)
|
||||
|
||||
// ParseGetObjectRange parses input range header and returns startoffset, length, isValid
|
||||
// and error. If no endoffset specified, then length is set to the object size
|
||||
// for invalid inputs, it returns no error, but isValid=false
|
||||
// `InvalidRange` error is returnd, only if startoffset is greater than the object size
|
||||
func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
|
||||
// ParseRange parses input range header and returns startoffset, length, and
|
||||
// error. If no endoffset specified, then length is set to -1.
|
||||
func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
|
||||
if acceptRange == "" {
|
||||
return 0, size, false, nil
|
||||
return 0, fi.Size(), nil
|
||||
}
|
||||
|
||||
rangeKv := strings.Split(acceptRange, "=")
|
||||
|
||||
if len(rangeKv) != 2 {
|
||||
return 0, size, false, nil
|
||||
}
|
||||
|
||||
if rangeKv[0] != "bytes" {
|
||||
return 0, size, false, nil
|
||||
if len(rangeKv) < 2 {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
bRange := strings.Split(rangeKv[1], "-")
|
||||
if len(bRange) != 2 {
|
||||
return 0, size, false, nil
|
||||
if len(bRange) < 1 || len(bRange) > 2 {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, size, false, nil
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
if startOffset >= size {
|
||||
return 0, 0, false, errInvalidRange
|
||||
endOffset := int64(-1)
|
||||
if len(bRange) == 1 || bRange[1] == "" {
|
||||
return startOffset, endOffset, nil
|
||||
}
|
||||
|
||||
if bRange[1] == "" {
|
||||
return startOffset, size - startOffset, true, nil
|
||||
}
|
||||
|
||||
endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
|
||||
endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, size, false, nil
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
if endOffset < startOffset {
|
||||
return 0, size, false, nil
|
||||
}
|
||||
|
||||
if endOffset >= size {
|
||||
return startOffset, size - startOffset, true, nil
|
||||
}
|
||||
|
||||
return startOffset, endOffset - startOffset + 1, true, nil
|
||||
}
|
||||
|
||||
// ParseCopySourceRange parses input range header and returns startoffset, length
|
||||
// and error. If no endoffset specified, then length is set to the object size
|
||||
func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
|
||||
if acceptRange == "" {
|
||||
return 0, size, nil
|
||||
}
|
||||
|
||||
rangeKv := strings.Split(acceptRange, "=")
|
||||
|
||||
if len(rangeKv) != 2 {
|
||||
return 0, 0, errInvalidCopySourceRange
|
||||
}
|
||||
|
||||
if rangeKv[0] != "bytes" {
|
||||
return 0, 0, errInvalidCopySourceRange
|
||||
}
|
||||
|
||||
bRange := strings.Split(rangeKv[1], "-")
|
||||
if len(bRange) != 2 {
|
||||
return 0, 0, errInvalidCopySourceRange
|
||||
}
|
||||
|
||||
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errInvalidCopySourceRange
|
||||
}
|
||||
|
||||
if startOffset >= size {
|
||||
return 0, 0, s3err.CreateExceedingRangeErr(size)
|
||||
}
|
||||
|
||||
if bRange[1] == "" {
|
||||
return startOffset, size - startOffset + 1, nil
|
||||
}
|
||||
|
||||
endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errInvalidCopySourceRange
|
||||
}
|
||||
|
||||
if endOffset < startOffset {
|
||||
return 0, 0, errInvalidCopySourceRange
|
||||
}
|
||||
|
||||
if endOffset >= size {
|
||||
return 0, 0, s3err.CreateExceedingRangeErr(size)
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
return startOffset, endOffset - startOffset + 1, nil
|
||||
}
|
||||
|
||||
// ParseCopySource parses x-amz-copy-source header and returns source bucket,
|
||||
// source object, versionId, error respectively
|
||||
func ParseCopySource(copySourceHeader string) (string, string, string, error) {
|
||||
if copySourceHeader[0] == '/' {
|
||||
copySourceHeader = copySourceHeader[1:]
|
||||
}
|
||||
|
||||
var copySource, versionId string
|
||||
i := strings.LastIndex(copySourceHeader, "?versionId=")
|
||||
if i == -1 {
|
||||
copySource = copySourceHeader
|
||||
} else {
|
||||
copySource = copySourceHeader[:i]
|
||||
versionId = copySourceHeader[i+11:]
|
||||
}
|
||||
|
||||
srcBucket, srcObject, ok := strings.Cut(copySource, "/")
|
||||
if !ok {
|
||||
return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
|
||||
}
|
||||
|
||||
return srcBucket, srcObject, versionId, nil
|
||||
}
|
||||
|
||||
// ParseObjectTags parses the url encoded input string into
|
||||
// map[string]string key-value tag set
|
||||
func ParseObjectTags(t string) (map[string]string, error) {
|
||||
if t == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
tagging := make(map[string]string)
|
||||
|
||||
tagParts := strings.Split(t, "&")
|
||||
for _, prt := range tagParts {
|
||||
p := strings.Split(prt, "=")
|
||||
if len(p) != 2 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
|
||||
}
|
||||
if len(p[0]) > 128 || len(p[1]) > 256 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
|
||||
}
|
||||
tagging[p[0]] = p[1]
|
||||
}
|
||||
|
||||
return tagging, nil
|
||||
}
|
||||
|
||||
func GetMultipartMD5(parts []types.CompletedPart) string {
|
||||
var partsEtagBytes []byte
|
||||
for _, part := range parts {
|
||||
partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
|
||||
s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
|
||||
return s3MD5
|
||||
}
|
||||
|
||||
func getEtagBytes(etag string) []byte {
|
||||
@@ -256,16 +120,3 @@ func md5String(data []byte) string {
|
||||
sum := md5.Sum(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
type FileSectionReadCloser struct {
|
||||
R io.Reader
|
||||
F *os.File
|
||||
}
|
||||
|
||||
func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
|
||||
return f.R.Read(p)
|
||||
}
|
||||
|
||||
func (f *FileSectionReadCloser) Close() error {
|
||||
return f.F.Close()
|
||||
}
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import "os"
|
||||
|
||||
// MetadataStorer defines the interface for managing metadata.
|
||||
// When object == "", the operation is on the bucket.
|
||||
type MetadataStorer interface {
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
// Returns the value of the attribute, or an error if the attribute does not exist.
|
||||
RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
// If attribute already exists, new attribute should replace existing.
|
||||
// Returns an error if the operation fails.
|
||||
StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
// Returns an error if the operation fails.
|
||||
DeleteAttribute(bucket, object, attribute string) error
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
// Returns list of attribute names, or an error if the operation fails.
|
||||
ListAttributes(bucket, object string) ([]string, error)
|
||||
|
||||
// DeleteAttributes removes all attributes for an object or a bucket.
|
||||
// Returns an error if the operation fails.
|
||||
DeleteAttributes(bucket, object string) error
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// NoMeta is a metadata storer that does not store metadata.
|
||||
// This can be useful for read only mounts where attempting to store metadata
|
||||
// would fail.
|
||||
type NoMeta struct{}
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
// always returns ErrNoSuchKey
|
||||
func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
// always returns nil without storing the attribute
|
||||
func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
// always returns nil without deleting the attribute
|
||||
func (NoMeta) DeleteAttribute(_, _, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
// always returns an empty list of attributes
|
||||
func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// DeleteAttributes removes all attributes for an object or a bucket.
|
||||
// always returns nil without deleting any attributes
|
||||
func (NoMeta) DeleteAttributes(bucket, object string) error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// SideCar is a metadata storer that uses sidecar files to store metadata.
|
||||
type SideCar struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
const (
|
||||
sidecarmeta = "meta"
|
||||
)
|
||||
|
||||
// NewSideCar creates a new SideCar metadata storer.
|
||||
func NewSideCar(dir string) (SideCar, error) {
|
||||
fi, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return SideCar{}, fmt.Errorf("not a directory")
|
||||
}
|
||||
|
||||
return SideCar{dir: dir}, nil
|
||||
}
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
attr := filepath.Join(metadir, attribute)
|
||||
|
||||
value, err := os.ReadFile(attr)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read attribute: %v", err)
|
||||
}
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
err := os.MkdirAll(metadir, 0777)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create metadata directory: %v", err)
|
||||
}
|
||||
|
||||
attr := filepath.Join(metadir, attribute)
|
||||
err = os.WriteFile(attr, value, 0666)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write attribute: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
attr := filepath.Join(metadir, attribute)
|
||||
|
||||
err := os.Remove(attr)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return ErrNoSuchKey
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove attribute: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
|
||||
ents, err := os.ReadDir(metadir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return []string{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list attributes: %v", err)
|
||||
}
|
||||
|
||||
var attrs []string
|
||||
for _, ent := range ents {
|
||||
attrs = append(attrs, ent.Name())
|
||||
}
|
||||
|
||||
return attrs, nil
|
||||
}
|
||||
|
||||
// DeleteAttributes removes all attributes for an object or a bucket.
|
||||
func (s SideCar) DeleteAttributes(bucket, object string) error {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
|
||||
err := os.RemoveAll(metadir)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("failed to remove attributes: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
const (
|
||||
xattrPrefix = "user."
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoSuchKey is returned when the key does not exist.
|
||||
ErrNoSuchKey = errors.New("no such key")
|
||||
)
|
||||
|
||||
type XattrMeta struct{}
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
|
||||
if f != nil {
|
||||
b, err := xattr.FGet(f, xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
|
||||
if f != nil {
|
||||
err := xattr.FSet(f, xattrPrefix+attribute, value)
|
||||
if errors.Is(err, syscall.EROFS) {
|
||||
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
|
||||
if errors.Is(err, syscall.EROFS) {
|
||||
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
|
||||
err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return ErrNoSuchKey
|
||||
}
|
||||
if errors.Is(err, syscall.EROFS) {
|
||||
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteAttributes is not implemented for xattr since xattrs
|
||||
// are automatically removed when the file is deleted.
|
||||
func (x XattrMeta) DeleteAttributes(bucket, object string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAttributes lists all attributes for an object in a bucket.
|
||||
func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
|
||||
attrs, err := xattr.List(filepath.Join(bucket, object))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
attributes := make([]string, 0, len(attrs))
|
||||
for _, attr := range attrs {
|
||||
if !isUserAttr(attr) {
|
||||
continue
|
||||
}
|
||||
attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
|
||||
}
|
||||
return attributes, nil
|
||||
}
|
||||
|
||||
func isUserAttr(attr string) bool {
|
||||
return strings.HasPrefix(attr, xattrPrefix)
|
||||
}
|
||||
|
||||
// Test is a helper function to test if xattrs are supported.
|
||||
func (x XattrMeta) Test(path string) error {
|
||||
// check for platform support
|
||||
if !xattr.XATTR_SUPPORTED {
|
||||
return fmt.Errorf("xattrs are not supported on this platform")
|
||||
}
|
||||
|
||||
// check if the filesystem supports xattrs
|
||||
_, err := xattr.Get(path, "user.test")
|
||||
if errors.Is(err, syscall.ENOTSUP) {
|
||||
return fmt.Errorf("xattrs are not supported on this filesystem")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
// Copyright 2024 Versity Software
|
||||
|
||||
// MkdirAll borrowed from stdlib to add ability to set ownership
|
||||
// as directories are created
|
||||
@@ -15,6 +14,11 @@ import (
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultDirPerm fs.FileMode = 0755
|
||||
)
|
||||
|
||||
// MkdirAll is similar to os.MkdirAll but it will return
|
||||
// ErrObjectParentIsFile when appropriate
|
||||
// MkdirAll creates a directory named path,
|
||||
@@ -25,9 +29,9 @@ import (
|
||||
// Any newly created directory is set to provided uid/gid ownership.
|
||||
// If path is already a directory, MkdirAll does nothing
|
||||
// and returns nil.
|
||||
// Any directory created will be set to provided uid/gid ownership
|
||||
// Any directoy created will be set to provided uid/gid ownership
|
||||
// if doChown is true.
|
||||
func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
|
||||
func MkdirAll(path string, uid, gid int, doChown bool) error {
|
||||
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
|
||||
dir, err := os.Stat(path)
|
||||
if err == nil {
|
||||
@@ -50,14 +54,14 @@ func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) erro
|
||||
|
||||
if j > 1 {
|
||||
// Create parent.
|
||||
err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
|
||||
err = MkdirAll(path[:j-1], uid, gid, doChown)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Parent now exists; invoke Mkdir and use its result.
|
||||
err = os.Mkdir(path, dirPerm)
|
||||
err = os.Mkdir(path, defaultDirPerm)
|
||||
if err != nil {
|
||||
// Handle arguments like "foo/." by
|
||||
// double-checking that directory doesn't exist.
|
||||
|
||||
@@ -1,516 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package vgwplugin
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"plugin"
|
||||
"reflect"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
// The plugin backend is used to dynamically load a Go plugin at runtime.
|
||||
// It loads the plugin and calls the InitPlugin function to initialize it.
|
||||
// A config string option is passed to init the plugin, it is expected that the
|
||||
// plugin will handle its own configuration and initialization from this.
|
||||
// If the plugin cannot be loaded or initialized, it returns an error.
|
||||
// The InitPlugin function should be defined in the plugin and should have
|
||||
// the signature func(configfile string) (version int, err error).
|
||||
// The plugin should also implement the backend.Backend interface functions.
|
||||
// However, the plugin does not need to implement all functions of the
|
||||
// backend.Backend interface. It can implement only the functions it needs.
|
||||
// Any non-implemented functions will return an error indicating that
|
||||
// the function is not implemented.
|
||||
// The plugin file should be compiled with the same Go version as the
|
||||
// application using it. The plugin file should be built with the
|
||||
// -buildmode=plugin flag.
|
||||
// Example: go build -buildmode=plugin -o myplugin.so myplugin.go
|
||||
// See the following for caveats and details:
|
||||
// https://pkg.go.dev/plugin#hdr-Warnings
|
||||
|
||||
// PluginBackend implements the backend.Backend interface using Go plugins.
|
||||
type PluginBackend struct {
|
||||
p *plugin.Plugin
|
||||
}
|
||||
|
||||
// NewPluginBackend creates a new PluginBackend. The path parameter should
|
||||
// point to the compiled plugin file (e.g., .so file).
|
||||
func NewPluginBackend(path, config string) (*PluginBackend, error) {
|
||||
p, err := plugin.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open plugin: %w", err)
|
||||
}
|
||||
|
||||
initSymbol, err := p.Lookup("InitPlugin")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to lookup InitPlugin symbol: %w", err)
|
||||
}
|
||||
|
||||
initFunc, ok := initSymbol.(func(string) (int, error))
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("InitPlugin symbol is not a func() (int, error)")
|
||||
}
|
||||
|
||||
version, err := initFunc(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("InitPlugin failed: %w", err)
|
||||
}
|
||||
|
||||
if version != backend.InterfaceVersion {
|
||||
return nil, fmt.Errorf("plugin interface version mismatch: gateway %v, plugin %v",
|
||||
backend.InterfaceVersion, version)
|
||||
}
|
||||
|
||||
return &PluginBackend{p: p}, nil
|
||||
}
|
||||
|
||||
func (p *PluginBackend) callPluginFunc(name string, args []any) ([]reflect.Value, error) {
|
||||
symbol, err := p.p.Lookup(name)
|
||||
if err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
symbolValue := reflect.ValueOf(symbol)
|
||||
if symbolValue.Kind() != reflect.Func {
|
||||
return nil, fmt.Errorf("symbol %s is not a function", name)
|
||||
}
|
||||
|
||||
numIn := symbolValue.Type().NumIn()
|
||||
if len(args) != numIn {
|
||||
return nil, fmt.Errorf("incorrect number of arguments for function %s, expected %d, got %d", name, numIn, len(args))
|
||||
}
|
||||
|
||||
in := make([]reflect.Value, len(args))
|
||||
for i := range args {
|
||||
in[i] = reflect.ValueOf(args[i])
|
||||
}
|
||||
|
||||
return symbolValue.Call(in), nil
|
||||
}
|
||||
|
||||
func (p *PluginBackend) String() string { return "Plugin Gateway" }
|
||||
func (p *PluginBackend) Shutdown() {}
|
||||
|
||||
func (p *PluginBackend) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
results, err := p.callPluginFunc("ListBuckets", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.ListAllMyBucketsResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.ListAllMyBucketsResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
results, err := p.callPluginFunc("HeadBucket", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.HeadBucketOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
|
||||
results, err := p.callPluginFunc("GetBucketAcl", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().([]byte), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, defaultACL []byte) error {
|
||||
_, err := p.callPluginFunc("CreateBucket", []any{ctx, input, defaultACL})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
|
||||
_, err := p.callPluginFunc("PutBucketAcl", []any{ctx, bucket, data})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
_, err := p.callPluginFunc("DeleteBucket", []any{ctx, bucket})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
|
||||
_, err := p.callPluginFunc("PutBucketVersioning", []any{ctx, bucket, status})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
results, err := p.callPluginFunc("GetBucketVersioning", []any{ctx, bucket})
|
||||
if err != nil {
|
||||
return s3response.GetBucketVersioningOutput{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.GetBucketVersioningOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
|
||||
_, err := p.callPluginFunc("PutBucketPolicy", []any{ctx, bucket, policy})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
|
||||
results, err := p.callPluginFunc("GetBucketPolicy", []any{ctx, bucket})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().([]byte), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
||||
_, err := p.callPluginFunc("DeleteBucketPolicy", []any{ctx, bucket})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
|
||||
_, err := p.callPluginFunc("PutBucketOwnershipControls", []any{ctx, bucket, ownership})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
results, err := p.callPluginFunc("GetBucketOwnershipControls", []any{ctx, bucket})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return results[0].Interface().(types.ObjectOwnership), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
|
||||
_, err := p.callPluginFunc("DeleteBucketOwnershipControls", []any{ctx, bucket})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutBucketCors(ctx context.Context, data []byte) error {
|
||||
_, err := p.callPluginFunc("PutBucketCors", []any{ctx, data})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetBucketCors(ctx context.Context, bucket string) ([]byte, error) {
|
||||
results, err := p.callPluginFunc("GetBucketCors", []any{ctx, bucket})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().([]byte), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteBucketCors(ctx context.Context, bucket string) error {
|
||||
_, err := p.callPluginFunc("DeleteBucketCors", []any{ctx, bucket})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
|
||||
results, err := p.callPluginFunc("CreateMultipartUpload", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.InitiateMultipartUploadResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.InitiateMultipartUploadResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
results, err := p.callPluginFunc("CompleteMultipartUpload", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.CompleteMultipartUploadOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
|
||||
_, err := p.callPluginFunc("AbortMultipartUpload", []any{ctx, input})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
results, err := p.callPluginFunc("ListMultipartUploads", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.ListMultipartUploadsResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.ListMultipartUploadsResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
results, err := p.callPluginFunc("ListParts", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.ListPartsResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.ListPartsResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
|
||||
results, err := p.callPluginFunc("UploadPart", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.UploadPartOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
|
||||
results, err := p.callPluginFunc("UploadPartCopy", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.CopyPartResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.CopyPartResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutObject(ctx context.Context, input s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
results, err := p.callPluginFunc("PutObject", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.PutObjectOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
results, err := p.callPluginFunc("HeadObject", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.HeadObjectOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
results, err := p.callPluginFunc("GetObject", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.GetObjectOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetObjectAcl(ctx context.Context, input *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
|
||||
results, err := p.callPluginFunc("GetObjectAcl", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.GetObjectAclOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
|
||||
results, err := p.callPluginFunc("GetObjectAttributes", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.GetObjectAttributesResponse{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.GetObjectAttributesResponse), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
results, err := p.callPluginFunc("CopyObject", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.CopyObjectOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
results, err := p.callPluginFunc("ListObjects", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.ListObjectsResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.ListObjectsResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
|
||||
results, err := p.callPluginFunc("ListObjectsV2", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.ListObjectsV2Result{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.ListObjectsV2Result), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
results, err := p.callPluginFunc("DeleteObject", []any{ctx, input})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(*s3.DeleteObjectOutput), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
results, err := p.callPluginFunc("DeleteObjects", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.DeleteResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.DeleteResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutObjectAcl(ctx context.Context, input *s3.PutObjectAclInput) error {
|
||||
_, err := p.callPluginFunc("PutObjectAcl", []any{ctx, input})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
|
||||
results, err := p.callPluginFunc("ListObjectVersions", []any{ctx, input})
|
||||
if err != nil {
|
||||
return s3response.ListVersionsResult{}, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(s3response.ListVersionsResult), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) RestoreObject(ctx context.Context, input *s3.RestoreObjectInput) error {
|
||||
_, err := p.callPluginFunc("RestoreObject", []any{ctx, input})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
results, err := p.callPluginFunc("SelectObjectContent", []any{ctx, input})
|
||||
if err != nil {
|
||||
return func(w *bufio.Writer) {}
|
||||
}
|
||||
|
||||
return results[0].Interface().(func(w *bufio.Writer))
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) {
|
||||
results, err := p.callPluginFunc("GetBucketTagging", []any{ctx, bucket})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(map[string]string), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
|
||||
_, err := p.callPluginFunc("PutBucketTagging", []any{ctx, bucket, tags})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteBucketTagging(ctx context.Context, bucket string) error {
|
||||
_, err := p.callPluginFunc("DeleteBucketTagging", []any{ctx, bucket})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
|
||||
results, err := p.callPluginFunc("GetObjectTagging", []any{ctx, bucket, object})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().(map[string]string), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
|
||||
_, err := p.callPluginFunc("PutObjectTagging", []any{ctx, bucket, object, tags})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
|
||||
_, err := p.callPluginFunc("DeleteObjectTagging", []any{ctx, bucket, object})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
|
||||
_, err := p.callPluginFunc("PutObjectLockConfiguration", []any{ctx, bucket, config})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
|
||||
results, err := p.callPluginFunc("GetObjectLockConfiguration", []any{ctx, bucket})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().([]byte), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
|
||||
_, err := p.callPluginFunc("PutObjectRetention", []any{ctx, bucket, object, versionId, bypass, retention})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
results, err := p.callPluginFunc("GetObjectRetention", []any{ctx, bucket, object, versionId})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().([]byte), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
|
||||
_, err := p.callPluginFunc("PutObjectLegalHold", []any{ctx, bucket, object, versionId, status})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
results, err := p.callPluginFunc("GetObjectLegalHold", []any{ctx, bucket, object, versionId})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
val := results[0].Interface()
|
||||
|
||||
if val == nil {
|
||||
return nil, convertError(results[1])
|
||||
}
|
||||
|
||||
return val.(*bool), convertError(results[1])
|
||||
}
|
||||
|
||||
func (p *PluginBackend) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
|
||||
_, err := p.callPluginFunc("ChangeBucketOwner", []any{ctx, bucket, acl})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PluginBackend) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
|
||||
results, err := p.callPluginFunc("ListBucketsAndOwners", []any{ctx})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results[0].Interface().([]s3response.Bucket), convertError(results[1])
|
||||
}
|
||||
|
||||
func convertError(result reflect.Value) error {
|
||||
if result.IsNil() {
|
||||
return nil
|
||||
}
|
||||
|
||||
err, ok := result.Interface().(error)
|
||||
if !ok {
|
||||
return fmt.Errorf("expected error, got %T", result.Interface())
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var _ backend.Backend = &PluginBackend{}
|
||||
File diff suppressed because it is too large
Load Diff
24
backend/posix/with_enodata.go
Normal file
24
backend/posix/with_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !freebsd && !openbsd && !netbsd
|
||||
// +build !freebsd,!openbsd,!netbsd
|
||||
|
||||
package posix
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENODATA
|
||||
)
|
||||
@@ -29,7 +29,6 @@ import (
|
||||
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
@@ -44,7 +43,6 @@ type tmpfile struct {
|
||||
needsChown bool
|
||||
uid int
|
||||
gid int
|
||||
newDirPerm fs.FileMode
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -52,7 +50,7 @@ var (
|
||||
defaultFilePerm uint32 = 0644
|
||||
)
|
||||
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
|
||||
uid, gid, doChown := p.getChownIDs(acct)
|
||||
|
||||
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
|
||||
@@ -63,12 +61,8 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
|
||||
// this is not supported.
|
||||
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EROFS) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
}
|
||||
|
||||
// O_TMPFILE not supported, try fallback
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
@@ -87,7 +81,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
|
||||
gid: gid,
|
||||
}
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 && dofalloc {
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
@@ -114,11 +108,10 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
|
||||
needsChown: doChown,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
newDirPerm: p.newDirPerm,
|
||||
}
|
||||
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 && dofalloc {
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
@@ -141,9 +134,6 @@ func (tmp *tmpfile) falloc() error {
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// make sure this is cleaned up in all error cases
|
||||
defer tmp.f.Close()
|
||||
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
// with any other simultaneous uploads. The final operation is to move the
|
||||
@@ -158,7 +148,7 @@ func (tmp *tmpfile) link() error {
|
||||
|
||||
dir := filepath.Dir(objPath)
|
||||
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make parent dir: %w", err)
|
||||
}
|
||||
@@ -180,21 +170,11 @@ func (tmp *tmpfile) link() error {
|
||||
}
|
||||
defer dirf.Close()
|
||||
|
||||
for {
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
if errors.Is(err, syscall.EEXIST) {
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile (fd %q as %q): %w",
|
||||
filepath.Base(tmp.f.Name()), objPath, err)
|
||||
}
|
||||
break
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile (%q in %q): %w",
|
||||
filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
@@ -241,7 +221,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return tmp.f
|
||||
}
|
||||
|
||||
24
backend/posix/without_enodata.go
Normal file
24
backend/posix/without_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build freebsd || openbsd || netbsd
|
||||
// +build freebsd openbsd netbsd
|
||||
|
||||
package posix
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENOATTR
|
||||
)
|
||||
@@ -24,11 +24,9 @@ import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type tmpfile struct {
|
||||
@@ -38,24 +36,18 @@ type tmpfile struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
|
||||
uid, gid, doChown := p.getChownIDs(acct)
|
||||
|
||||
// Create a temp file for upload while in progress (see link comments below).
|
||||
var err error
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EROFS) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
}
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
f, err := os.CreateTemp(dir,
|
||||
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EROFS) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
}
|
||||
return nil, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
|
||||
@@ -120,7 +112,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return tmp.f
|
||||
}
|
||||
|
||||
@@ -33,12 +33,6 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = &s.endpoint
|
||||
}), nil
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(cfg), nil
|
||||
}
|
||||
|
||||
@@ -56,6 +50,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(s))
|
||||
}
|
||||
|
||||
if s.disableChecksum {
|
||||
opts = append(opts,
|
||||
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
|
||||
@@ -68,3 +67,13 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
|
||||
|
||||
return config.LoadDefaultConfig(ctx, opts...)
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
SigningRegion: s.awsRegion,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -23,24 +23,21 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/versity/scoutfs-go"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
metastore := meta.XattrMeta{}
|
||||
|
||||
p, err := posix.New(rootdir, metastore, posix.PosixOpts{
|
||||
ChownUID: opts.ChownUID,
|
||||
ChownGID: opts.ChownGID,
|
||||
BucketLinks: opts.BucketLinks,
|
||||
NewDirPerm: opts.NewDirPerm,
|
||||
p, err := posix.New(rootdir, posix.PosixOpts{
|
||||
ChownUID: opts.ChownUID,
|
||||
ChownGID: opts.ChownGID,
|
||||
ReadOnly: opts.ReadOnly,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -52,15 +49,12 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
}
|
||||
|
||||
return &ScoutFS{
|
||||
Posix: p,
|
||||
rootfd: f,
|
||||
rootdir: rootdir,
|
||||
meta: metastore,
|
||||
chownuid: opts.ChownUID,
|
||||
chowngid: opts.ChownGID,
|
||||
glaciermode: opts.GlacierMode,
|
||||
newDirPerm: opts.NewDirPerm,
|
||||
disableNoArchive: opts.DisableNoArchive,
|
||||
Posix: p,
|
||||
rootfd: f,
|
||||
rootdir: rootdir,
|
||||
chownuid: opts.ChownUID,
|
||||
chowngid: opts.ChownGID,
|
||||
readonly: opts.ReadOnly,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -74,10 +68,10 @@ type tmpfile struct {
|
||||
needsChown bool
|
||||
uid int
|
||||
gid int
|
||||
newDirPerm fs.FileMode
|
||||
}
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultFilePerm uint32 = 0644
|
||||
)
|
||||
|
||||
@@ -105,7 +99,11 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
|
||||
needsChown: doChown,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
newDirPerm: s.newDirPerm,
|
||||
}
|
||||
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
if doChown {
|
||||
@@ -118,6 +116,14 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) falloc() error {
|
||||
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fallocate: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
@@ -133,7 +139,7 @@ func (tmp *tmpfile) link() error {
|
||||
|
||||
dir := filepath.Dir(objPath)
|
||||
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make parent dir: %w", err)
|
||||
}
|
||||
@@ -178,10 +184,6 @@ func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return tmp.f
|
||||
}
|
||||
|
||||
func moveData(from *os.File, to *os.File) error {
|
||||
return scoutfs.MoveData(from, to)
|
||||
}
|
||||
|
||||
@@ -28,7 +28,9 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
return nil, fmt.Errorf("scoutfs only available on linux")
|
||||
}
|
||||
|
||||
type tmpfile struct{}
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
var (
|
||||
errNotSupported = errors.New("not supported")
|
||||
@@ -40,6 +42,7 @@ func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile
|
||||
_ = s.chowngid
|
||||
_ = s.euid
|
||||
_ = s.egid
|
||||
_ = s.readonly
|
||||
return nil, errNotSupported
|
||||
}
|
||||
|
||||
@@ -54,10 +57,6 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return nil
|
||||
}
|
||||
|
||||
func moveData(_, _ *os.File) error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
24
backend/scoutfs/with_enodata.go
Normal file
24
backend/scoutfs/with_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !freebsd && !openbsd && !netbsd
|
||||
// +build !freebsd,!openbsd,!netbsd
|
||||
|
||||
package scoutfs
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENODATA
|
||||
)
|
||||
24
backend/scoutfs/without_enodata.go
Normal file
24
backend/scoutfs/without_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build freebsd || openbsd || netbsd
|
||||
// +build freebsd openbsd netbsd
|
||||
|
||||
package scoutfs
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENOATTR
|
||||
)
|
||||
373
backend/walk.go
373
backend/walk.go
@@ -15,66 +15,46 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type WalkResults struct {
|
||||
CommonPrefixes []types.CommonPrefix
|
||||
Objects []s3response.Object
|
||||
Objects []types.Object
|
||||
Truncated bool
|
||||
NextMarker string
|
||||
}
|
||||
|
||||
type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
|
||||
type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
|
||||
|
||||
var ErrSkipObj = errors.New("skip this object")
|
||||
|
||||
// Walk walks the supplied fs.FS and returns results compatible with list
|
||||
// objects responses
|
||||
func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
|
||||
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
|
||||
cpmap := make(map[string]struct{})
|
||||
var objects []s3response.Object
|
||||
|
||||
// if max is 0, it should return empty non-truncated result
|
||||
if max == 0 {
|
||||
return WalkResults{
|
||||
Truncated: false,
|
||||
}, nil
|
||||
}
|
||||
var objects []types.Object
|
||||
|
||||
var pastMarker bool
|
||||
if marker == "" {
|
||||
pastMarker = true
|
||||
}
|
||||
|
||||
var pastMax bool
|
||||
pastMax := max == 0
|
||||
var newMarker string
|
||||
var truncated bool
|
||||
|
||||
root := "."
|
||||
if strings.Contains(prefix, "/") {
|
||||
idx := strings.LastIndex(prefix, "/")
|
||||
if idx > 0 {
|
||||
root = prefix[:idx]
|
||||
}
|
||||
}
|
||||
|
||||
err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
|
||||
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
// Ignore the root directory
|
||||
if path == "." {
|
||||
return nil
|
||||
@@ -83,9 +63,14 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// After this point, return skipflag instead of nil
|
||||
// so we can skip a directory without an early return
|
||||
var skipflag error
|
||||
if pastMax {
|
||||
if len(objects) != 0 {
|
||||
newMarker = *objects[len(objects)-1].Key
|
||||
truncated = true
|
||||
}
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
// If prefix is defined and the directory does not match prefix,
|
||||
// do not descend into the directory because nothing will
|
||||
@@ -95,66 +80,44 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
// building to match. So only skip if path isn't a prefix of prefix
|
||||
// and prefix isn't a prefix of path.
|
||||
if prefix != "" &&
|
||||
!strings.HasPrefix(path+"/", prefix) &&
|
||||
!strings.HasPrefix(prefix, path+"/") {
|
||||
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
|
||||
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// Don't recurse into subdirectories which contain the delimiter
|
||||
// after reaching the prefix
|
||||
if delimiter != "" &&
|
||||
strings.HasPrefix(path+"/", prefix) &&
|
||||
strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
|
||||
skipflag = fs.SkipDir
|
||||
} else {
|
||||
if delimiter == "" {
|
||||
dirobj, err := getObj(path+"/", d)
|
||||
if err == ErrSkipObj {
|
||||
return skipflag
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
if pastMax {
|
||||
truncated = true
|
||||
return fs.SkipAll
|
||||
}
|
||||
objects = append(objects, dirobj)
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
newMarker = path
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return skipflag
|
||||
}
|
||||
|
||||
// TODO: can we do better here rather than a second readdir
|
||||
// per directory?
|
||||
ents, err := fs.ReadDir(fileSystem, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("readdir %q: %w", path, err)
|
||||
}
|
||||
|
||||
if len(ents) != 0 {
|
||||
return skipflag
|
||||
}
|
||||
// TODO: can we do better here rather than a second readdir
|
||||
// per directory?
|
||||
ents, err := fs.ReadDir(fileSystem, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("readdir %q: %w", path, err)
|
||||
}
|
||||
path += "/"
|
||||
if len(ents) == 0 {
|
||||
dirobj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, dirobj)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if !pastMarker {
|
||||
if path == marker {
|
||||
pastMarker = true
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
if path < marker {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// If object doesn't have prefix, don't include in results.
|
||||
if prefix != "" && !strings.HasPrefix(path, prefix) {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
|
||||
if delimiter == "" {
|
||||
@@ -162,24 +125,18 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
// prefix are included in results
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
if pastMax {
|
||||
truncated = true
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
objects = append(objects, obj)
|
||||
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
newMarker = path
|
||||
if max > 0 && (len(objects)+len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
|
||||
// Since delimiter is specified, we only want results that
|
||||
@@ -208,55 +165,29 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
if !found {
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
if pastMax {
|
||||
truncated = true
|
||||
return fs.SkipAll
|
||||
}
|
||||
objects = append(objects, obj)
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
newMarker = path
|
||||
pastMax = true
|
||||
}
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
|
||||
// Common prefixes are a set, so should not have duplicates.
|
||||
// These are abstractly a "directory", so need to include the
|
||||
// delimiter at the end when we add to the map.
|
||||
cprefNoDelim := prefix + before
|
||||
cpref := prefix + before + delimiter
|
||||
if cpref == marker {
|
||||
pastMarker = true
|
||||
return skipflag
|
||||
}
|
||||
|
||||
if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
|
||||
// skip common prefixes that are before the marker
|
||||
return skipflag
|
||||
}
|
||||
|
||||
if pastMax {
|
||||
truncated = true
|
||||
return fs.SkipAll
|
||||
}
|
||||
cpmap[cpref] = struct{}{}
|
||||
// delimiter at the end.
|
||||
cpmap[prefix+before+delimiter] = struct{}{}
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
newMarker = cpref
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return skipflag
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
// suppress file not found caused by user's prefix
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
return WalkResults{}, nil
|
||||
}
|
||||
return WalkResults{}, err
|
||||
}
|
||||
|
||||
@@ -273,10 +204,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
})
|
||||
}
|
||||
|
||||
if !truncated {
|
||||
newMarker = ""
|
||||
}
|
||||
|
||||
return WalkResults{
|
||||
CommonPrefixes: commonPrefixes,
|
||||
Objects: objects,
|
||||
@@ -293,213 +220,3 @@ func contains(a string, strs []string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type WalkVersioningResults struct {
|
||||
CommonPrefixes []types.CommonPrefix
|
||||
ObjectVersions []types.ObjectVersion
|
||||
DelMarkers []types.DeleteMarkerEntry
|
||||
Truncated bool
|
||||
NextMarker string
|
||||
NextVersionIdMarker string
|
||||
}
|
||||
|
||||
type ObjVersionFuncResult struct {
|
||||
ObjectVersions []types.ObjectVersion
|
||||
DelMarkers []types.DeleteMarkerEntry
|
||||
NextVersionIdMarker string
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
|
||||
|
||||
// WalkVersions walks the supplied fs.FS and returns results compatible with
|
||||
// ListObjectVersions action response
|
||||
func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
|
||||
cpmap := make(map[string]struct{})
|
||||
var objects []types.ObjectVersion
|
||||
var delMarkers []types.DeleteMarkerEntry
|
||||
|
||||
var pastMarker bool
|
||||
if keyMarker == "" {
|
||||
pastMarker = true
|
||||
}
|
||||
var nextMarker string
|
||||
var nextVersionIdMarker string
|
||||
var truncated bool
|
||||
|
||||
pastVersionIdMarker := versionIdMarker == ""
|
||||
|
||||
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
// Ignore the root directory
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
if contains(d.Name(), skipdirs) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
if !pastMarker {
|
||||
if path == keyMarker {
|
||||
pastMarker = true
|
||||
}
|
||||
if path < keyMarker {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
// If prefix is defined and the directory does not match prefix,
|
||||
// do not descend into the directory because nothing will
|
||||
// match this prefix. Make sure to append the / at the end of
|
||||
// directories since this is implied as a directory path name.
|
||||
// If path is a prefix of prefix, then path could still be
|
||||
// building to match. So only skip if path isn't a prefix of prefix
|
||||
// and prefix isn't a prefix of path.
|
||||
if prefix != "" &&
|
||||
!strings.HasPrefix(path+"/", prefix) &&
|
||||
!strings.HasPrefix(prefix, path+"/") {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// Don't recurse into subdirectories when listing with delimiter.
|
||||
if delimiter == "/" &&
|
||||
prefix != path+"/" &&
|
||||
strings.HasPrefix(path+"/", prefix) {
|
||||
cpmap[path+"/"] = struct{}{}
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, res.ObjectVersions...)
|
||||
delMarkers = append(delMarkers, res.DelMarkers...)
|
||||
if res.Truncated {
|
||||
truncated = true
|
||||
nextMarker = path
|
||||
nextVersionIdMarker = res.NextVersionIdMarker
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// If object doesn't have prefix, don't include in results.
|
||||
if prefix != "" && !strings.HasPrefix(path, prefix) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if delimiter == "" {
|
||||
// If no delimiter specified, then all files with matching
|
||||
// prefix are included in results
|
||||
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, res.ObjectVersions...)
|
||||
delMarkers = append(delMarkers, res.DelMarkers...)
|
||||
if res.Truncated {
|
||||
truncated = true
|
||||
nextMarker = path
|
||||
nextVersionIdMarker = res.NextVersionIdMarker
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Since delimiter is specified, we only want results that
|
||||
// do not contain the delimiter beyond the prefix. If the
|
||||
// delimiter exists past the prefix, then the substring
|
||||
// between the prefix and delimiter is part of common prefixes.
|
||||
//
|
||||
// For example:
|
||||
// prefix = A/
|
||||
// delimiter = /
|
||||
// and objects:
|
||||
// A/file
|
||||
// A/B/file
|
||||
// B/C
|
||||
// would return:
|
||||
// objects: A/file
|
||||
// common prefix: A/B/
|
||||
//
|
||||
// Note: No objects are included past the common prefix since
|
||||
// these are all rolled up into the common prefix.
|
||||
// Note: The delimiter can be anything, so we have to operate on
|
||||
// the full path without any assumptions on posix directory hierarchy
|
||||
// here. Usually the delimiter will be "/", but thats not required.
|
||||
suffix := strings.TrimPrefix(path, prefix)
|
||||
before, _, found := strings.Cut(suffix, delimiter)
|
||||
if !found {
|
||||
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, res.ObjectVersions...)
|
||||
delMarkers = append(delMarkers, res.DelMarkers...)
|
||||
|
||||
if res.Truncated {
|
||||
truncated = true
|
||||
nextMarker = path
|
||||
nextVersionIdMarker = res.NextVersionIdMarker
|
||||
return fs.SkipAll
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Common prefixes are a set, so should not have duplicates.
|
||||
// These are abstractly a "directory", so need to include the
|
||||
// delimiter at the end.
|
||||
cpmap[prefix+before+delimiter] = struct{}{}
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
nextMarker = path
|
||||
truncated = true
|
||||
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return WalkVersioningResults{}, err
|
||||
}
|
||||
|
||||
var commonPrefixStrings []string
|
||||
for k := range cpmap {
|
||||
commonPrefixStrings = append(commonPrefixStrings, k)
|
||||
}
|
||||
sort.Strings(commonPrefixStrings)
|
||||
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
|
||||
for _, cp := range commonPrefixStrings {
|
||||
pfx := cp
|
||||
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
|
||||
Prefix: &pfx,
|
||||
})
|
||||
}
|
||||
|
||||
return WalkVersioningResults{
|
||||
CommonPrefixes: commonPrefixes,
|
||||
ObjectVersions: objects,
|
||||
DelMarkers: delMarkers,
|
||||
Truncated: truncated,
|
||||
NextMarker: nextMarker,
|
||||
NextVersionIdMarker: nextVersionIdMarker,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -15,50 +15,36 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type walkTest struct {
|
||||
fsys fs.FS
|
||||
getobj backend.GetObjFunc
|
||||
cases []testcase
|
||||
fsys fs.FS
|
||||
expected backend.WalkResults
|
||||
getobj backend.GetObjFunc
|
||||
}
|
||||
|
||||
type testcase struct {
|
||||
name string
|
||||
prefix string
|
||||
delimiter string
|
||||
marker string
|
||||
maxObjs int32
|
||||
expected backend.WalkResults
|
||||
}
|
||||
|
||||
func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
func getObj(path string, d fs.DirEntry) (types.Object, error) {
|
||||
if d.IsDir() {
|
||||
etag := getMD5(path)
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
mtime := fi.ModTime()
|
||||
|
||||
return s3response.Object{
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: &mtime,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -66,16 +52,15 @@ func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
mtime := fi.ModTime()
|
||||
|
||||
return s3response.Object{
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: &mtime,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
}, nil
|
||||
}
|
||||
@@ -97,154 +82,50 @@ func TestWalk(t *testing.T) {
|
||||
"photos/2006/February/sample3.jpg": {},
|
||||
"photos/2006/February/sample4.jpg": {},
|
||||
},
|
||||
getobj: getObj,
|
||||
cases: []testcase{
|
||||
{
|
||||
name: "aws example",
|
||||
delimiter: "/",
|
||||
maxObjs: 1000,
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photos/"),
|
||||
}},
|
||||
Objects: []s3response.Object{{
|
||||
Key: backend.GetPtrFromString("sample.jpg"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("photos/"),
|
||||
}},
|
||||
Objects: []types.Object{{
|
||||
Key: backend.GetStringPtr("sample.jpg"),
|
||||
}},
|
||||
},
|
||||
getobj: getObj,
|
||||
},
|
||||
{
|
||||
// test case single dir/single file
|
||||
fsys: fstest.MapFS{
|
||||
"test/file": {},
|
||||
},
|
||||
getobj: getObj,
|
||||
cases: []testcase{
|
||||
{
|
||||
name: "single dir single file",
|
||||
delimiter: "/",
|
||||
maxObjs: 1000,
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("test/"),
|
||||
}},
|
||||
Objects: []s3response.Object{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// non-standard delimiter
|
||||
fsys: fstest.MapFS{
|
||||
"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
|
||||
"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
|
||||
"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("test/"),
|
||||
}},
|
||||
Objects: []types.Object{},
|
||||
},
|
||||
getobj: getObj,
|
||||
cases: []testcase{
|
||||
{
|
||||
name: "different delimiter 1",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 2",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 3",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 4",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 5",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|6/Januar|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 6",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|6/Januar|y/sampl|",
|
||||
expected: backend.WalkResults{
|
||||
Objects: []s3response.Object{
|
||||
{
|
||||
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
|
||||
},
|
||||
{
|
||||
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
|
||||
},
|
||||
{
|
||||
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
for _, tc := range tt.cases {
|
||||
res, err := backend.Walk(context.Background(),
|
||||
tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
|
||||
tt.getobj, []string{})
|
||||
if err != nil {
|
||||
t.Errorf("tc.name: walk: %v", err)
|
||||
}
|
||||
|
||||
compareResults(tc.name, res, tc.expected, t)
|
||||
res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
|
||||
if err != nil {
|
||||
t.Fatalf("walk: %v", err)
|
||||
}
|
||||
|
||||
compareResults(res, tt.expected, t)
|
||||
}
|
||||
}
|
||||
|
||||
func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
|
||||
func compareResults(got, wanted backend.WalkResults, t *testing.T) {
|
||||
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
|
||||
t.Errorf("%v: unexpected common prefix, got %v wanted %v",
|
||||
name,
|
||||
t.Errorf("unexpected common prefix, got %v wanted %v",
|
||||
printCommonPrefixes(got.CommonPrefixes),
|
||||
printCommonPrefixes(wanted.CommonPrefixes))
|
||||
}
|
||||
|
||||
if !compareObjects(got.Objects, wanted.Objects) {
|
||||
t.Errorf("%v: unexpected object, got %v wanted %v",
|
||||
name,
|
||||
t.Errorf("unexpected object, got %v wanted %v",
|
||||
printObjects(got.Objects),
|
||||
printObjects(wanted.Objects))
|
||||
}
|
||||
@@ -287,7 +168,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
|
||||
return res + "]"
|
||||
}
|
||||
|
||||
func compareObjects(a, b []s3response.Object) bool {
|
||||
func compareObjects(a, b []types.Object) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
@@ -303,7 +184,7 @@ func compareObjects(a, b []s3response.Object) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func containsObject(c s3response.Object, list []s3response.Object) bool {
|
||||
func containsObject(c types.Object, list []types.Object) bool {
|
||||
for _, cp := range list {
|
||||
if *c.Key == *cp.Key {
|
||||
return true
|
||||
@@ -312,67 +193,14 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func printObjects(list []s3response.Object) string {
|
||||
func printObjects(list []types.Object) string {
|
||||
res := "["
|
||||
for _, cp := range list {
|
||||
var key string
|
||||
if cp.Key == nil {
|
||||
key = "<nil>"
|
||||
} else {
|
||||
key = *cp.Key
|
||||
}
|
||||
if res == "[" {
|
||||
res = res + key
|
||||
res = res + *cp.Key
|
||||
} else {
|
||||
res = res + ", " + key
|
||||
res = res + ", " + *cp.Key
|
||||
}
|
||||
}
|
||||
return res + "]"
|
||||
}
|
||||
|
||||
type slowFS struct {
|
||||
fstest.MapFS
|
||||
}
|
||||
|
||||
const (
|
||||
readDirPause = 100 * time.Millisecond
|
||||
|
||||
// walkTimeOut should be less than the tree traversal time
|
||||
// which is the readdirPause time * the number of directories
|
||||
walkTimeOut = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||
time.Sleep(readDirPause)
|
||||
return s.MapFS.ReadDir(name)
|
||||
}
|
||||
|
||||
func TestWalkStop(t *testing.T) {
|
||||
s := &slowFS{MapFS: fstest.MapFS{
|
||||
"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
|
||||
}}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
|
||||
defer cancel()
|
||||
|
||||
var err error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, err = backend.Walk(ctx, s, "", "/", "", 1000,
|
||||
func(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
return s3response.Object{}, nil
|
||||
}, []string{})
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-time.After(1 * time.Second):
|
||||
t.Fatalf("walk is not terminated in time")
|
||||
case <-ctx.Done():
|
||||
}
|
||||
wg.Wait()
|
||||
if err != ctx.Err() {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,9 +17,8 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -29,7 +28,6 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
@@ -38,9 +36,7 @@ import (
|
||||
var (
|
||||
adminAccess string
|
||||
adminSecret string
|
||||
adminRegion string
|
||||
adminEndpoint string
|
||||
allowInsecure bool
|
||||
)
|
||||
|
||||
func adminCommand() *cli.Command {
|
||||
@@ -82,33 +78,10 @@ func adminCommand() *cli.Command {
|
||||
Usage: "groupID for the new user",
|
||||
Aliases: []string{"gi"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "update-user",
|
||||
Usage: "Updates a user account",
|
||||
Action: updateUser,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "user access key id to be updated",
|
||||
Required: true,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "secret access key for the new user",
|
||||
Aliases: []string{"s"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "user-id",
|
||||
Usage: "userID for the new user",
|
||||
Aliases: []string{"ui"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "group-id",
|
||||
Usage: "groupID for the new user",
|
||||
Aliases: []string{"gi"},
|
||||
Name: "project-id",
|
||||
Usage: "projectID for the new user",
|
||||
Aliases: []string{"pi"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -173,14 +146,6 @@ func adminCommand() *cli.Command {
|
||||
Required: true,
|
||||
Destination: &adminSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "region",
|
||||
Usage: "admin s3 region string",
|
||||
EnvVars: []string{"ADMIN_REGION"},
|
||||
Value: "us-east-1",
|
||||
Destination: &adminRegion,
|
||||
Aliases: []string{"r"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "endpoint-url",
|
||||
Usage: "admin apis endpoint url",
|
||||
@@ -189,80 +154,65 @@ func adminCommand() *cli.Command {
|
||||
Required: true,
|
||||
Destination: &adminEndpoint,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "allow-insecure",
|
||||
Usage: "disable tls certificate verification for the admin endpoint",
|
||||
EnvVars: []string{"ADMIN_ALLOW_INSECURE"},
|
||||
Aliases: []string{"ai"},
|
||||
Destination: &allowInsecure,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func initHTTPClient() *http.Client {
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
|
||||
}
|
||||
return &http.Client{Transport: tr}
|
||||
}
|
||||
|
||||
func createUser(ctx *cli.Context) error {
|
||||
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
|
||||
userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
|
||||
userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
|
||||
if access == "" || secret == "" {
|
||||
return fmt.Errorf("invalid input parameters for the new user access/secret keys")
|
||||
return fmt.Errorf("invalid input parameters for the new user")
|
||||
}
|
||||
if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
|
||||
return fmt.Errorf("invalid input parameter for role: %v", role)
|
||||
}
|
||||
|
||||
acc := auth.Account{
|
||||
Access: access,
|
||||
Secret: secret,
|
||||
Role: auth.Role(role),
|
||||
UserID: userID,
|
||||
GroupID: groupID,
|
||||
Access: access,
|
||||
Secret: secret,
|
||||
Role: auth.Role(role),
|
||||
UserID: userID,
|
||||
GroupID: groupID,
|
||||
ProjectID: projectID,
|
||||
}
|
||||
|
||||
accxml, err := xml.Marshal(acc)
|
||||
accJson, err := json.Marshal(acc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse user data: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256(accxml)
|
||||
hashedPayload := sha256.Sum256(accJson)
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -270,7 +220,7 @@ func createUser(ctx *cli.Context) error {
|
||||
func deleteUser(ctx *cli.Context) error {
|
||||
access := ctx.String("access")
|
||||
if access == "" {
|
||||
return fmt.Errorf("invalid input parameter for the user access key")
|
||||
return fmt.Errorf("invalid input parameter for the new user")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
|
||||
@@ -285,82 +235,25 @@ func deleteUser(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateUser(ctx *cli.Context) error {
|
||||
access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
|
||||
props := auth.MutableProps{}
|
||||
if ctx.IsSet("secret") {
|
||||
props.Secret = &secret
|
||||
}
|
||||
if ctx.IsSet("user-id") {
|
||||
props.UserID = &userId
|
||||
}
|
||||
if ctx.IsSet("group-id") {
|
||||
props.GroupID = &groupId
|
||||
}
|
||||
|
||||
propsxml, err := xml.Marshal(props)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse user attributes: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256(propsxml)
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -378,34 +271,34 @@ func listUsers(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
var accs auth.ListUserAccountsResult
|
||||
if err := xml.Unmarshal(body, &accs); err != nil {
|
||||
var accs []auth.Account
|
||||
if err := json.Unmarshal(body, &accs); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printAcctTable(accs.Accounts)
|
||||
printAcctTable(accs)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -422,10 +315,10 @@ const (
|
||||
func printAcctTable(accs []auth.Account) {
|
||||
w := new(tabwriter.Writer)
|
||||
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
|
||||
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
|
||||
fmt.Fprintln(w, "-------\t----\t------\t-------")
|
||||
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
|
||||
fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
|
||||
for _, acc := range accs {
|
||||
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
|
||||
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
w.Flush()
|
||||
@@ -445,27 +338,25 @@ func changeBucketOwner(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -495,45 +386,34 @@ func listBuckets(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
var result s3response.ListBucketsResult
|
||||
if err := xml.Unmarshal(body, &result); err != nil {
|
||||
var buckets []s3response.Bucket
|
||||
if err := json.Unmarshal(body, &buckets); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printBuckets(result.Buckets)
|
||||
printBuckets(buckets)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseApiError(body []byte) error {
|
||||
var apiErr smithy.GenericAPIError
|
||||
err := xml.Unmarshal(body, &apiErr)
|
||||
if err != nil {
|
||||
apiErr.Code = "InternalServerError"
|
||||
apiErr.Message = err.Error()
|
||||
}
|
||||
|
||||
return &apiErr
|
||||
}
|
||||
|
||||
@@ -7,9 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
"github.com/versity/versitygw/tests/integration"
|
||||
)
|
||||
@@ -58,9 +56,7 @@ func initPosix(ctx context.Context) {
|
||||
log.Fatalf("make temp directory: %v", err)
|
||||
}
|
||||
|
||||
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
|
||||
NewDirPerm: 0755,
|
||||
})
|
||||
be, err := posix.New(tempdir, posix.PosixOpts{})
|
||||
if err != nil {
|
||||
log.Fatalf("init posix: %v", err)
|
||||
}
|
||||
@@ -78,9 +74,6 @@ func initPosix(ctx context.Context) {
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// wait for server to start
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
|
||||
@@ -19,17 +19,14 @@ import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
@@ -37,46 +34,33 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
port, admPort string
|
||||
rootUserAccess string
|
||||
rootUserSecret string
|
||||
region string
|
||||
admCertFile, admKeyFile string
|
||||
certFile, keyFile string
|
||||
kafkaURL, kafkaTopic, kafkaKey string
|
||||
natsURL, natsTopic string
|
||||
eventWebhookURL string
|
||||
eventConfigFilePath string
|
||||
logWebhookURL, accessLog string
|
||||
adminLogFile string
|
||||
healthPath string
|
||||
debug bool
|
||||
pprof string
|
||||
quiet bool
|
||||
readonly bool
|
||||
iamDir string
|
||||
ldapURL, ldapBindDN, ldapPassword string
|
||||
ldapQueryBase, ldapObjClasses string
|
||||
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
|
||||
ldapUserIdAtr, ldapGroupIdAtr string
|
||||
vaultEndpointURL, vaultSecretStoragePath string
|
||||
vaultMountPath, vaultRootToken string
|
||||
vaultRoleId, vaultRoleSecret string
|
||||
vaultServerCert, vaultClientCert string
|
||||
vaultClientCertKey string
|
||||
s3IamAccess, s3IamSecret string
|
||||
s3IamRegion, s3IamBucket string
|
||||
s3IamEndpoint string
|
||||
s3IamSslNoVerify, s3IamDebug bool
|
||||
iamCacheDisable bool
|
||||
iamCacheTTL int
|
||||
iamCachePrune int
|
||||
metricsService string
|
||||
statsdServers string
|
||||
dogstatsServers string
|
||||
ipaHost, ipaVaultName string
|
||||
ipaUser, ipaPassword string
|
||||
ipaInsecure, ipaDebug bool
|
||||
port, admPort string
|
||||
rootUserAccess string
|
||||
rootUserSecret string
|
||||
region string
|
||||
admCertFile, admKeyFile string
|
||||
certFile, keyFile string
|
||||
kafkaURL, kafkaTopic, kafkaKey string
|
||||
natsURL, natsTopic string
|
||||
eventWebhookURL string
|
||||
eventConfigFilePath string
|
||||
logWebhookURL string
|
||||
accessLog string
|
||||
healthPath string
|
||||
debug bool
|
||||
pprof string
|
||||
quiet bool
|
||||
iamDir string
|
||||
ldapURL, ldapBindDN, ldapPassword string
|
||||
ldapQueryBase, ldapObjClasses string
|
||||
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
|
||||
s3IamAccess, s3IamSecret string
|
||||
s3IamRegion, s3IamBucket string
|
||||
s3IamEndpoint string
|
||||
s3IamSslNoVerify, s3IamDebug bool
|
||||
iamCacheDisable bool
|
||||
iamCacheTTL int
|
||||
iamCachePrune int
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -98,7 +82,6 @@ func main() {
|
||||
scoutfsCommand(),
|
||||
s3Command(),
|
||||
azureCommand(),
|
||||
pluginCommand(),
|
||||
adminCommand(),
|
||||
testCommand(),
|
||||
utilsCommand(),
|
||||
@@ -118,13 +101,10 @@ func main() {
|
||||
|
||||
func initApp() *cli.App {
|
||||
return &cli.App{
|
||||
Usage: "Versity S3 Gateway",
|
||||
Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
|
||||
to access the supported backend storage as if it was a native S3 service.
|
||||
VersityGW is an open-source project licensed under the Apache 2.0 License. The
|
||||
source code is hosted on GitHub at https://github.com/versity/versitygw, and
|
||||
documentation can be found in the GitHub wiki.`,
|
||||
Copyright: "Copyright (c) 2023-2024 Versity Software",
|
||||
Name: "versitygw",
|
||||
Usage: "Start S3 gateway service with specified backend storage.",
|
||||
Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
|
||||
to access the supported backend storage as if it was a native S3 service.`,
|
||||
Action: func(ctx *cli.Context) error {
|
||||
return ctx.App.Command("help").Run(ctx)
|
||||
},
|
||||
@@ -210,7 +190,6 @@ func initFlags() []cli.Flag {
|
||||
&cli.BoolFlag{
|
||||
Name: "debug",
|
||||
Usage: "enable debug output",
|
||||
Value: false,
|
||||
EnvVars: []string{"VGW_DEBUG"},
|
||||
Destination: &debug,
|
||||
},
|
||||
@@ -233,12 +212,6 @@ func initFlags() []cli.Flag {
|
||||
EnvVars: []string{"LOGFILE", "VGW_ACCESS_LOG"},
|
||||
Destination: &accessLog,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "admin-access-log",
|
||||
Usage: "enable admin server access logging to specified file",
|
||||
EnvVars: []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
|
||||
Destination: &adminLogFile,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "log-webhook-url",
|
||||
Usage: "webhook url to send the audit logs",
|
||||
@@ -348,72 +321,6 @@ func initFlags() []cli.Flag {
|
||||
EnvVars: []string{"VGW_IAM_LDAP_ROLE_ATR"},
|
||||
Destination: &ldapRoleAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-user-id-atr",
|
||||
Usage: "ldap server user id attribute name",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_USER_ID_ATR"},
|
||||
Destination: &ldapUserIdAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-group-id-atr",
|
||||
Usage: "ldap server user group id attribute name",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
|
||||
Destination: &ldapGroupIdAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-endpoint-url",
|
||||
Usage: "vault server url",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
|
||||
Destination: &vaultEndpointURL,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-secret-storage-path",
|
||||
Usage: "vault server secret storage path",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
|
||||
Destination: &vaultSecretStoragePath,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-mount-path",
|
||||
Usage: "vault server mount path",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_MOUNT_PATH"},
|
||||
Destination: &vaultMountPath,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-root-token",
|
||||
Usage: "vault server root token",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
|
||||
Destination: &vaultRootToken,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-role-id",
|
||||
Usage: "vault server user role id",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ROLE_ID"},
|
||||
Destination: &vaultRoleId,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-role-secret",
|
||||
Usage: "vault server user role secret",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ROLE_SECRET"},
|
||||
Destination: &vaultRoleSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-server_cert",
|
||||
Usage: "vault server TLS certificate",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_SERVER_CERT"},
|
||||
Destination: &vaultServerCert,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-client_cert",
|
||||
Usage: "vault client TLS certificate",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT"},
|
||||
Destination: &vaultClientCert,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-client_cert_key",
|
||||
Usage: "vault client TLS certificate key",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
|
||||
Destination: &vaultClientCertKey,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-access",
|
||||
Usage: "s3 IAM access key",
|
||||
@@ -484,69 +391,6 @@ func initFlags() []cli.Flag {
|
||||
EnvVars: []string{"VGW_HEALTH"},
|
||||
Destination: &healthPath,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "readonly",
|
||||
Usage: "allow only read operations across all the gateway",
|
||||
EnvVars: []string{"VGW_READ_ONLY"},
|
||||
Destination: &readonly,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-service-name",
|
||||
Usage: "service name tag for metrics, hostname if blank",
|
||||
EnvVars: []string{"VGW_METRICS_SERVICE_NAME"},
|
||||
Aliases: []string{"msn"},
|
||||
Destination: &metricsService,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-statsd-servers",
|
||||
Usage: "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
|
||||
EnvVars: []string{"VGW_METRICS_STATSD_SERVERS"},
|
||||
Aliases: []string{"mss"},
|
||||
Destination: &statsdServers,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-dogstatsd-servers",
|
||||
Usage: "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
|
||||
EnvVars: []string{"VGW_METRICS_DOGSTATS_SERVERS"},
|
||||
Aliases: []string{"mds"},
|
||||
Destination: &dogstatsServers,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-host",
|
||||
Usage: "FreeIPA server url e.g. https://ipa.example.test",
|
||||
EnvVars: []string{"VGW_IPA_HOST"},
|
||||
Destination: &ipaHost,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-vault-name",
|
||||
Usage: "A name of the user vault containing their secret",
|
||||
EnvVars: []string{"VGW_IPA_VAULT_NAME"},
|
||||
Destination: &ipaVaultName,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-user",
|
||||
Usage: "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
|
||||
EnvVars: []string{"VGW_IPA_USER"},
|
||||
Destination: &ipaUser,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-password",
|
||||
Usage: "Password of the user used to connect to FreeIPA.",
|
||||
EnvVars: []string{"VGW_IPA_PASSWORD"},
|
||||
Destination: &ipaPassword,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "ipa-insecure",
|
||||
Usage: "Verify TLS certificate of FreeIPA server. Default is 'true'.",
|
||||
EnvVars: []string{"VGW_IPA_INSECURE"},
|
||||
Destination: &ipaInsecure,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "ipa-debug",
|
||||
Usage: "FreeIPA IAM debug output",
|
||||
EnvVars: []string{"VGW_IPA_DEBUG"},
|
||||
Destination: &ipaDebug,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -564,12 +408,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
StreamRequestBody: true,
|
||||
DisableKeepalive: true,
|
||||
Network: fiber.NetworkTCP,
|
||||
DisableStartupMessage: true,
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
StreamRequestBody: true,
|
||||
DisableKeepalive: true,
|
||||
})
|
||||
|
||||
var opts []s3api.Option
|
||||
@@ -600,15 +442,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
if healthPath != "" {
|
||||
opts = append(opts, s3api.WithHealth(healthPath))
|
||||
}
|
||||
if readonly {
|
||||
opts = append(opts, s3api.WithReadOnly())
|
||||
}
|
||||
|
||||
admApp := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
Network: fiber.NetworkTCP,
|
||||
DisableStartupMessage: true,
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
})
|
||||
|
||||
var admOpts []s3api.AdminOpt
|
||||
@@ -629,70 +466,38 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
}
|
||||
|
||||
iam, err := auth.New(&auth.Opts{
|
||||
RootAccount: auth.Account{
|
||||
Access: rootUserAccess,
|
||||
Secret: rootUserSecret,
|
||||
Role: auth.RoleAdmin,
|
||||
},
|
||||
Dir: iamDir,
|
||||
LDAPServerURL: ldapURL,
|
||||
LDAPBindDN: ldapBindDN,
|
||||
LDAPPassword: ldapPassword,
|
||||
LDAPQueryBase: ldapQueryBase,
|
||||
LDAPObjClasses: ldapObjClasses,
|
||||
LDAPAccessAtr: ldapAccessAtr,
|
||||
LDAPSecretAtr: ldapSecAtr,
|
||||
LDAPRoleAtr: ldapRoleAtr,
|
||||
LDAPUserIdAtr: ldapUserIdAtr,
|
||||
LDAPGroupIdAtr: ldapGroupIdAtr,
|
||||
VaultEndpointURL: vaultEndpointURL,
|
||||
VaultSecretStoragePath: vaultSecretStoragePath,
|
||||
VaultMountPath: vaultMountPath,
|
||||
VaultRootToken: vaultRootToken,
|
||||
VaultRoleId: vaultRoleId,
|
||||
VaultRoleSecret: vaultRoleSecret,
|
||||
VaultServerCert: vaultServerCert,
|
||||
VaultClientCert: vaultClientCert,
|
||||
VaultClientCertKey: vaultClientCertKey,
|
||||
S3Access: s3IamAccess,
|
||||
S3Secret: s3IamSecret,
|
||||
S3Region: s3IamRegion,
|
||||
S3Bucket: s3IamBucket,
|
||||
S3Endpoint: s3IamEndpoint,
|
||||
S3DisableSSlVerfiy: s3IamSslNoVerify,
|
||||
S3Debug: s3IamDebug,
|
||||
CacheDisable: iamCacheDisable,
|
||||
CacheTTL: iamCacheTTL,
|
||||
CachePrune: iamCachePrune,
|
||||
IpaHost: ipaHost,
|
||||
IpaVaultName: ipaVaultName,
|
||||
IpaUser: ipaUser,
|
||||
IpaPassword: ipaPassword,
|
||||
IpaInsecure: ipaInsecure,
|
||||
IpaDebug: ipaDebug,
|
||||
Dir: iamDir,
|
||||
LDAPServerURL: ldapURL,
|
||||
LDAPBindDN: ldapBindDN,
|
||||
LDAPPassword: ldapPassword,
|
||||
LDAPQueryBase: ldapQueryBase,
|
||||
LDAPObjClasses: ldapObjClasses,
|
||||
LDAPAccessAtr: ldapAccessAtr,
|
||||
LDAPSecretAtr: ldapSecAtr,
|
||||
LDAPRoleAtr: ldapRoleAtr,
|
||||
S3Access: s3IamAccess,
|
||||
S3Secret: s3IamSecret,
|
||||
S3Region: s3IamRegion,
|
||||
S3Bucket: s3IamBucket,
|
||||
S3Endpoint: s3IamEndpoint,
|
||||
S3DisableSSlVerfiy: s3IamSslNoVerify,
|
||||
S3Debug: s3IamDebug,
|
||||
CacheDisable: iamCacheDisable,
|
||||
CacheTTL: iamCacheTTL,
|
||||
CachePrune: iamCachePrune,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup iam: %w", err)
|
||||
}
|
||||
|
||||
loggers, err := s3log.InitLogger(&s3log.LogConfig{
|
||||
LogFile: accessLog,
|
||||
WebhookURL: logWebhookURL,
|
||||
AdminLogFile: adminLogFile,
|
||||
logger, err := s3log.InitLogger(&s3log.LogConfig{
|
||||
LogFile: accessLog,
|
||||
WebhookURL: logWebhookURL,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup logger: %w", err)
|
||||
}
|
||||
|
||||
metricsManager, err := metrics.NewManager(ctx, metrics.Config{
|
||||
ServiceName: metricsService,
|
||||
StatsdServers: statsdServers,
|
||||
DogStatsdServers: dogstatsServers,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("init metrics manager: %w", err)
|
||||
}
|
||||
|
||||
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
|
||||
KafkaURL: kafkaURL,
|
||||
KafkaTopic: kafkaTopic,
|
||||
@@ -709,16 +514,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
|
||||
Access: rootUserAccess,
|
||||
Secret: rootUserSecret,
|
||||
}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
|
||||
}, port, region, iam, logger, evSender, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init gateway: %v", err)
|
||||
}
|
||||
|
||||
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
|
||||
|
||||
if !quiet {
|
||||
printBanner(port, admPort, certFile != "", admCertFile != "")
|
||||
}
|
||||
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
|
||||
|
||||
c := make(chan error, 2)
|
||||
go func() { c <- srv.Serve() }()
|
||||
@@ -735,17 +536,10 @@ Loop:
|
||||
case err = <-c:
|
||||
break Loop
|
||||
case <-sigHup:
|
||||
if loggers.S3Logger != nil {
|
||||
err = loggers.S3Logger.HangUp()
|
||||
if logger != nil {
|
||||
err = logger.HangUp()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("HUP s3 logger: %w", err)
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
if loggers.AdminLogger != nil {
|
||||
err = loggers.AdminLogger.HangUp()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("HUP admin logger: %w", err)
|
||||
err = fmt.Errorf("HUP logger: %w", err)
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
@@ -763,22 +557,13 @@ Loop:
|
||||
fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
|
||||
}
|
||||
|
||||
if loggers.S3Logger != nil {
|
||||
err := loggers.S3Logger.Shutdown()
|
||||
if logger != nil {
|
||||
err := logger.Shutdown()
|
||||
if err != nil {
|
||||
if saveErr == nil {
|
||||
saveErr = err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
|
||||
}
|
||||
}
|
||||
if loggers.AdminLogger != nil {
|
||||
err := loggers.AdminLogger.Shutdown()
|
||||
if err != nil {
|
||||
if saveErr == nil {
|
||||
saveErr = err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -792,183 +577,5 @@ Loop:
|
||||
}
|
||||
}
|
||||
|
||||
if metricsManager != nil {
|
||||
metricsManager.Close()
|
||||
}
|
||||
|
||||
return saveErr
|
||||
}
|
||||
|
||||
func printBanner(port, admPort string, ssl, admSsl bool) {
|
||||
interfaces, err := getMatchingIPs(port)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var admInterfaces []string
|
||||
if admPort != "" {
|
||||
admInterfaces, err = getMatchingIPs(admPort)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
title := "VersityGW"
|
||||
version := fmt.Sprintf("Version %v, Build %v", Version, Build)
|
||||
urls := []string{}
|
||||
|
||||
hst, prt, err := net.SplitHostPort(port)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, ip := range interfaces {
|
||||
url := fmt.Sprintf("http://%s:%s", ip, prt)
|
||||
if ssl {
|
||||
url = fmt.Sprintf("https://%s:%s", ip, prt)
|
||||
}
|
||||
urls = append(urls, url)
|
||||
}
|
||||
|
||||
if hst == "" {
|
||||
hst = "0.0.0.0"
|
||||
}
|
||||
|
||||
boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
|
||||
|
||||
lines := []string{
|
||||
centerText(title),
|
||||
centerText(version),
|
||||
centerText(boundHost),
|
||||
centerText(""),
|
||||
}
|
||||
|
||||
if len(admInterfaces) > 0 {
|
||||
lines = append(lines,
|
||||
leftText("S3 service listening on:"),
|
||||
)
|
||||
} else {
|
||||
lines = append(lines,
|
||||
leftText("Admin/S3 service listening on:"),
|
||||
)
|
||||
}
|
||||
|
||||
for _, url := range urls {
|
||||
lines = append(lines, leftText(" "+url))
|
||||
}
|
||||
|
||||
if len(admInterfaces) > 0 {
|
||||
lines = append(lines,
|
||||
centerText(""),
|
||||
leftText("Admin service listening on:"),
|
||||
)
|
||||
|
||||
_, prt, err := net.SplitHostPort(admPort)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, ip := range admInterfaces {
|
||||
url := fmt.Sprintf("http://%s:%s", ip, prt)
|
||||
if admSsl {
|
||||
url = fmt.Sprintf("https://%s:%s", ip, prt)
|
||||
}
|
||||
lines = append(lines, leftText(" "+url))
|
||||
}
|
||||
}
|
||||
|
||||
// Print the top border
|
||||
fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
|
||||
|
||||
// Print each line
|
||||
for _, line := range lines {
|
||||
fmt.Printf("│%-*s│\n", columnWidth-2, line)
|
||||
}
|
||||
|
||||
// Print the bottom border
|
||||
fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
|
||||
}
|
||||
|
||||
// getMatchingIPs returns all IP addresses for local system interfaces that
|
||||
// match the input address specification.
|
||||
func getMatchingIPs(spec string) ([]string, error) {
|
||||
// Split the input spec into IP and port
|
||||
host, _, err := net.SplitHostPort(spec)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse address/port: %v", err)
|
||||
}
|
||||
|
||||
// Handle cases where IP is omitted (e.g., ":1234")
|
||||
if host == "" {
|
||||
host = "0.0.0.0"
|
||||
}
|
||||
|
||||
ipaddr, err := net.ResolveIPAddr("ip", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parsedInputIP := ipaddr.IP
|
||||
|
||||
var result []string
|
||||
|
||||
// Get all network interfaces
|
||||
interfaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, iface := range interfaces {
|
||||
// Get all addresses associated with the interface
|
||||
addrs, err := iface.Addrs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, addr := range addrs {
|
||||
// Parse the address to get the IP part
|
||||
ipAddr, _, err := net.ParseCIDR(addr.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ipAddr.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
if ipAddr.IsInterfaceLocalMulticast() {
|
||||
continue
|
||||
}
|
||||
if ipAddr.IsLinkLocalMulticast() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if the IP matches the input specification
|
||||
if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
|
||||
result = append(result, ipAddr.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const columnWidth = 70
|
||||
|
||||
func centerText(text string) string {
|
||||
padding := (columnWidth - 2 - len(text)) / 2
|
||||
if padding < 0 {
|
||||
padding = 0
|
||||
}
|
||||
return strings.Repeat(" ", padding) + text
|
||||
}
|
||||
|
||||
func leftText(text string) string {
|
||||
if len(text) > columnWidth-2 {
|
||||
return text
|
||||
}
|
||||
return text + strings.Repeat(" ", columnWidth-2-len(text))
|
||||
}
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
vgwplugin "github.com/versity/versitygw/backend/plugin"
|
||||
)
|
||||
|
||||
var (
|
||||
pluginPath string
|
||||
pluginConfig string
|
||||
)
|
||||
|
||||
func pluginCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "plugin",
|
||||
Usage: "plugin storage backend",
|
||||
Description: `This tells the gateway to load the backend from a dynamic runtime plugin.`,
|
||||
Action: runPlugin,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "file",
|
||||
Usage: "path to plugin shared object file",
|
||||
Value: "",
|
||||
Required: true,
|
||||
EnvVars: []string{"VGW_PLUGIN_FILE"},
|
||||
Destination: &pluginPath,
|
||||
Aliases: []string{"f"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "config",
|
||||
Usage: "configuration option for the plugin",
|
||||
Value: "",
|
||||
Required: true,
|
||||
EnvVars: []string{"VGW_PLUGIN_CONFIG"},
|
||||
Destination: &pluginConfig,
|
||||
Aliases: []string{"c"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runPlugin(ctx *cli.Context) error {
|
||||
be, err := vgwplugin.NewPluginBackend(pluginPath, pluginConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init plugin backend: %w", err)
|
||||
}
|
||||
return runGateway(ctx.Context, be)
|
||||
}
|
||||
@@ -16,21 +16,14 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"math"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
var (
|
||||
chownuid, chowngid bool
|
||||
bucketlinks bool
|
||||
versioningDir string
|
||||
dirPerms uint
|
||||
sidecar string
|
||||
nometa bool
|
||||
readonly bool
|
||||
)
|
||||
|
||||
func posixCommand() *cli.Command {
|
||||
@@ -62,36 +55,10 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
|
||||
Destination: &chowngid,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "bucketlinks",
|
||||
Usage: "allow symlinked directories at bucket level to be treated as buckets",
|
||||
EnvVars: []string{"VGW_BUCKET_LINKS"},
|
||||
Destination: &bucketlinks,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "versioning-dir",
|
||||
Usage: "the directory path to enable bucket versioning",
|
||||
EnvVars: []string{"VGW_VERSIONING_DIR"},
|
||||
Destination: &versioningDir,
|
||||
},
|
||||
&cli.UintFlag{
|
||||
Name: "dir-perms",
|
||||
Usage: "default directory permissions for new directories",
|
||||
EnvVars: []string{"VGW_DIR_PERMS"},
|
||||
Destination: &dirPerms,
|
||||
DefaultText: "0755",
|
||||
Value: 0755,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "sidecar",
|
||||
Usage: "use provided sidecar directory to store metadata",
|
||||
EnvVars: []string{"VGW_META_SIDECAR"},
|
||||
Destination: &sidecar,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "nometa",
|
||||
Usage: "disable metadata storage",
|
||||
EnvVars: []string{"VGW_META_NONE"},
|
||||
Destination: &nometa,
|
||||
Name: "readonly",
|
||||
Usage: "allow only read operations to backend",
|
||||
EnvVars: []string{"VGW_READ_ONLY"},
|
||||
Destination: &readonly,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -102,46 +69,13 @@ func runPosix(ctx *cli.Context) error {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
gwroot := (ctx.Args().Get(0))
|
||||
|
||||
if dirPerms > math.MaxUint32 {
|
||||
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
|
||||
}
|
||||
|
||||
if nometa && sidecar != "" {
|
||||
return fmt.Errorf("cannot use both nometa and sidecar metadata")
|
||||
}
|
||||
|
||||
opts := posix.PosixOpts{
|
||||
ChownUID: chownuid,
|
||||
ChownGID: chowngid,
|
||||
BucketLinks: bucketlinks,
|
||||
VersioningDir: versioningDir,
|
||||
NewDirPerm: fs.FileMode(dirPerms),
|
||||
}
|
||||
|
||||
var ms meta.MetadataStorer
|
||||
switch {
|
||||
case sidecar != "":
|
||||
sc, err := meta.NewSideCar(sidecar)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init sidecar metadata: %w", err)
|
||||
}
|
||||
ms = sc
|
||||
opts.SideCarDir = sidecar
|
||||
case nometa:
|
||||
ms = meta.NoMeta{}
|
||||
default:
|
||||
ms = meta.XattrMeta{}
|
||||
err := meta.XattrMeta{}.Test(gwroot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("xattr check failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
be, err := posix.New(gwroot, ms, opts)
|
||||
be, err := posix.New(ctx.Args().Get(0), posix.PosixOpts{
|
||||
ChownUID: chownuid,
|
||||
ChownGID: chowngid,
|
||||
ReadOnly: readonly,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init posix backend: %w", err)
|
||||
return fmt.Errorf("init posix: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx.Context, be)
|
||||
|
||||
@@ -16,16 +16,17 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"math"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/scoutfs"
|
||||
)
|
||||
|
||||
var (
|
||||
glacier bool
|
||||
disableNoArchive bool
|
||||
glacier bool
|
||||
|
||||
// defined in posix.go:
|
||||
// chownuid, chowngid bool
|
||||
// readonly bool
|
||||
)
|
||||
|
||||
func scoutfsCommand() *cli.Command {
|
||||
@@ -67,24 +68,10 @@ move interfaces as well as support for tiered filesystems.`,
|
||||
Destination: &chowngid,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "bucketlinks",
|
||||
Usage: "allow symlinked directories at bucket level to be treated as buckets",
|
||||
EnvVars: []string{"VGW_BUCKET_LINKS"},
|
||||
Destination: &bucketlinks,
|
||||
},
|
||||
&cli.UintFlag{
|
||||
Name: "dir-perms",
|
||||
Usage: "default directory permissions for new directories",
|
||||
EnvVars: []string{"VGW_DIR_PERMS"},
|
||||
Destination: &dirPerms,
|
||||
DefaultText: "0755",
|
||||
Value: 0755,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "disable-noarchive",
|
||||
Usage: "disable setting noarchive for multipart part uploads",
|
||||
EnvVars: []string{"VGW_DISABLE_NOARCHIVE"},
|
||||
Destination: &disableNoArchive,
|
||||
Name: "readonly",
|
||||
Usage: "allow only read operations to backend",
|
||||
EnvVars: []string{"VGW_READ_ONLY"},
|
||||
Destination: &readonly,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -95,17 +82,11 @@ func runScoutfs(ctx *cli.Context) error {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
if dirPerms > math.MaxUint32 {
|
||||
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
|
||||
}
|
||||
|
||||
var opts scoutfs.ScoutfsOpts
|
||||
opts.GlacierMode = glacier
|
||||
opts.ChownUID = chownuid
|
||||
opts.ChownGID = chowngid
|
||||
opts.BucketLinks = bucketlinks
|
||||
opts.NewDirPerm = fs.FileMode(dirPerms)
|
||||
opts.DisableNoArchive = disableNoArchive
|
||||
opts.ReadOnly = readonly
|
||||
|
||||
be, err := scoutfs.New(ctx.Args().Get(0), opts)
|
||||
if err != nil {
|
||||
|
||||
@@ -22,23 +22,20 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
awsID string
|
||||
awsSecret string
|
||||
endpoint string
|
||||
prefix string
|
||||
dstBucket string
|
||||
partSize int64
|
||||
objSize int64
|
||||
concurrency int
|
||||
files int
|
||||
totalReqs int
|
||||
upload bool
|
||||
download bool
|
||||
pathStyle bool
|
||||
checksumDisable bool
|
||||
versioningEnabled bool
|
||||
azureTests bool
|
||||
tlsStatus bool
|
||||
awsID string
|
||||
awsSecret string
|
||||
endpoint string
|
||||
prefix string
|
||||
dstBucket string
|
||||
partSize int64
|
||||
objSize int64
|
||||
concurrency int
|
||||
files int
|
||||
totalReqs int
|
||||
upload bool
|
||||
download bool
|
||||
pathStyle bool
|
||||
checksumDisable bool
|
||||
)
|
||||
|
||||
func testCommand() *cli.Command {
|
||||
@@ -80,12 +77,6 @@ func initTestFlags() []cli.Flag {
|
||||
Aliases: []string{"d"},
|
||||
Destination: &debug,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "allow-insecure",
|
||||
Usage: "skip tls verification",
|
||||
Aliases: []string{"ai"},
|
||||
Destination: &tlsStatus,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,33 +87,11 @@ func initTestCommands() []*cli.Command {
|
||||
Usage: "Tests the full flow of gateway.",
|
||||
Description: `Runs all the available tests to test the full flow of the gateway.`,
|
||||
Action: getAction(integration.TestFullFlow),
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "versioning-enabled",
|
||||
Usage: "Test the bucket object versioning, if the versioning is enabled",
|
||||
Destination: &versioningEnabled,
|
||||
Aliases: []string{"vs"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "azure-test-mode",
|
||||
Usage: "Skips tests that are not supported by Azure",
|
||||
Destination: &azureTests,
|
||||
Aliases: []string{"azure"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "posix",
|
||||
Usage: "Tests posix specific features",
|
||||
Action: getAction(integration.TestPosix),
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "versioning-enabled",
|
||||
Usage: "Test posix when versioning is enabled",
|
||||
Destination: &versioningEnabled,
|
||||
Aliases: []string{"vs"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "iam",
|
||||
@@ -218,7 +187,6 @@ func initTestCommands() []*cli.Command {
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
integration.WithPartSize(partSize),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
@@ -279,7 +247,6 @@ func initTestCommands() []*cli.Command {
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
@@ -305,17 +272,10 @@ func getAction(tf testFunc) func(*cli.Context) error {
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if versioningEnabled {
|
||||
opts = append(opts, integration.WithVersioningEnabled())
|
||||
}
|
||||
if azureTests {
|
||||
opts = append(opts, integration.WithAzureMode())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
tf(s)
|
||||
@@ -343,27 +303,15 @@ func extractIntTests() (commands []*cli.Command) {
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if versioningEnabled {
|
||||
opts = append(opts, integration.WithVersioningEnabled())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
err := testFunc(s)
|
||||
return err
|
||||
},
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "versioning-enabled",
|
||||
Usage: "Test the bucket object versioning, if the versioning is enabled",
|
||||
Destination: &versioningEnabled,
|
||||
Aliases: []string{"vs"},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
return
|
||||
|
||||
@@ -54,24 +54,22 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
|
||||
}
|
||||
|
||||
config := s3event.EventFilter{
|
||||
s3event.EventObjectCreated: true,
|
||||
s3event.EventObjectCreatedPut: true,
|
||||
s3event.EventObjectCreatedPost: true,
|
||||
s3event.EventObjectCreatedCopy: true,
|
||||
s3event.EventCompleteMultipartUpload: true,
|
||||
s3event.EventObjectRemoved: true,
|
||||
s3event.EventObjectRemovedDelete: true,
|
||||
s3event.EventObjectRemovedDeleteObjects: true,
|
||||
s3event.EventObjectTagging: true,
|
||||
s3event.EventObjectTaggingPut: true,
|
||||
s3event.EventObjectTaggingDelete: true,
|
||||
s3event.EventObjectAclPut: true,
|
||||
s3event.EventObjectRestore: true,
|
||||
s3event.EventObjectRestorePost: true,
|
||||
s3event.EventObjectRestoreCompleted: true,
|
||||
s3event.EventObjectCreated: true,
|
||||
s3event.EventObjectCreatedPut: true,
|
||||
s3event.EventObjectCreatedPost: true,
|
||||
s3event.EventObjectCreatedCopy: true,
|
||||
s3event.EventCompleteMultipartUpload: true,
|
||||
s3event.EventObjectDeleted: true,
|
||||
s3event.EventObjectTagging: true,
|
||||
s3event.EventObjectTaggingPut: true,
|
||||
s3event.EventObjectTaggingDelete: true,
|
||||
s3event.EventObjectAclPut: true,
|
||||
s3event.EventObjectRestore: true,
|
||||
s3event.EventObjectRestorePost: true,
|
||||
s3event.EventObjectRestoreCompleted: true,
|
||||
}
|
||||
|
||||
configBytes, err := json.MarshalIndent(config, "", " ")
|
||||
configBytes, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse event config: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
version: "3"
|
||||
services:
|
||||
posix:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: tests/Dockerfile.dev
|
||||
dockerfile: ./Dockerfile.dev
|
||||
args:
|
||||
- IAM_DIR=${IAM_DIR}
|
||||
- SETUP_DIR=${SETUP_DIR}
|
||||
@@ -14,7 +15,7 @@ services:
|
||||
proxy:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: tests/Dockerfile.dev
|
||||
dockerfile: ./Dockerfile.dev
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
@@ -30,13 +31,13 @@ services:
|
||||
hostname: azurite
|
||||
command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
|
||||
volumes:
|
||||
- ./tests/certs:/tests/certs
|
||||
- ./certs:/certs
|
||||
azuritegw:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: tests/Dockerfile.dev
|
||||
dockerfile: ./Dockerfile.dev
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
- 7070:7070
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
|
||||
@@ -1,19 +0,0 @@
|
||||
# Versity Gateway Dashboard
|
||||
|
||||
This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana.
|
||||
|
||||
The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
|
||||
|
||||
This implementation uses the influxql query language.
|
||||
|
||||
## Usage
|
||||
|
||||
From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
|
||||
|
||||
To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`.
|
||||
|
||||
The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
|
||||
|
||||
The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with username: `admin` and password: `admin`.
|
||||
|
||||
To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.
|
||||
@@ -1,4 +0,0 @@
|
||||
export AWS_SECRET_ACCESS_KEY=password
|
||||
export AWS_ACCESS_KEY_ID=user
|
||||
export AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
export AWS_REGION=us-east-1
|
||||
@@ -1,64 +0,0 @@
|
||||
services:
|
||||
telegraf:
|
||||
image: telegraf
|
||||
container_name: telegraf
|
||||
restart: always
|
||||
volumes:
|
||||
- ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
|
||||
depends_on:
|
||||
- influxdb
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '8125:8125/udp'
|
||||
|
||||
influxdb:
|
||||
image: influxdb
|
||||
container_name: influxdb
|
||||
restart: always
|
||||
environment:
|
||||
- DOCKER_INFLUXDB_INIT_MODE=setup
|
||||
- DOCKER_INFLUXDB_INIT_USERNAME=admin
|
||||
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
|
||||
- DOCKER_INFLUXDB_INIT_ORG=myorg
|
||||
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
|
||||
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
|
||||
ports:
|
||||
- '8086:8086'
|
||||
volumes:
|
||||
- influxdb_data:/var/lib/influxdb
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana
|
||||
container_name: grafana-server
|
||||
restart: always
|
||||
depends_on:
|
||||
- influxdb
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
- GF_INSTALL_PLUGINS=
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./metrics-exploration/grafana_data:/var/lib/grafana
|
||||
- ./metrics-exploration/provisioning:/etc/grafana/provisioning
|
||||
|
||||
versitygw:
|
||||
image: versity/versitygw:latest
|
||||
container_name: versitygw
|
||||
ports:
|
||||
- "7070:7070"
|
||||
environment:
|
||||
- ROOT_ACCESS_KEY=user
|
||||
- ROOT_SECRET_KEY=password
|
||||
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
|
||||
depends_on:
|
||||
- telegraf
|
||||
command: >
|
||||
posix /tmp/vgw
|
||||
|
||||
volumes:
|
||||
influxdb_data: {}
|
||||
@@ -1,64 +0,0 @@
|
||||
services:
|
||||
telegraf:
|
||||
image: telegraf
|
||||
container_name: telegraf
|
||||
restart: always
|
||||
volumes:
|
||||
- ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
|
||||
depends_on:
|
||||
- influxdb
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '8125:8125/udp'
|
||||
|
||||
influxdb:
|
||||
image: influxdb
|
||||
container_name: influxdb
|
||||
restart: always
|
||||
environment:
|
||||
- DOCKER_INFLUXDB_INIT_MODE=setup
|
||||
- DOCKER_INFLUXDB_INIT_USERNAME=admin
|
||||
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
|
||||
- DOCKER_INFLUXDB_INIT_ORG=myorg
|
||||
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
|
||||
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
|
||||
ports:
|
||||
- '8086:8086'
|
||||
volumes:
|
||||
- influxdb_data:/var/lib/influxdb
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana
|
||||
container_name: grafana-server
|
||||
restart: always
|
||||
depends_on:
|
||||
- influxdb
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
- GF_INSTALL_PLUGINS=
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./grafana_data:/var/lib/grafana
|
||||
- ./provisioning:/etc/grafana/provisioning
|
||||
|
||||
versitygw:
|
||||
image: versity/versitygw:latest
|
||||
container_name: versitygw
|
||||
ports:
|
||||
- "7070:7070"
|
||||
environment:
|
||||
- ROOT_ACCESS_KEY=user
|
||||
- ROOT_SECRET_KEY=password
|
||||
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
|
||||
depends_on:
|
||||
- telegraf
|
||||
command: >
|
||||
posix /tmp/vgw
|
||||
|
||||
volumes:
|
||||
influxdb_data: {}
|
||||
@@ -1,25 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
# <string> an unique provider name. Required
|
||||
- name: 'influxql'
|
||||
# <int> Org id. Default to 1
|
||||
orgId: 1
|
||||
# <string> name of the dashboard folder.
|
||||
folder: 'influxql'
|
||||
# <string> folder UID. will be automatically generated if not specified
|
||||
folderUid: ''
|
||||
# <string> provider type. Default to 'file'
|
||||
type: file
|
||||
# <bool> disable dashboard deletion
|
||||
disableDeletion: false
|
||||
# <int> how often Grafana will scan for changed dashboards
|
||||
updateIntervalSeconds: 10
|
||||
# <bool> allow updating provisioned dashboards from the UI
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
# <string, required> path to dashboard files on disk. Required when using the 'file' type
|
||||
path: /etc/grafana/provisioning/dashboards/influxql
|
||||
# <bool> use folder names from filesystem to create folders in Grafana
|
||||
foldersFromFilesStructure: true
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,13 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: influxdb
|
||||
type: influxdb
|
||||
isDefault: true
|
||||
access: proxy
|
||||
url: http://influxdb:8086
|
||||
jsonData:
|
||||
dbName: 'metrics'
|
||||
httpHeaderName1: 'Authorization'
|
||||
secureJsonData:
|
||||
httpHeaderValue1: 'Token my-super-secret-auth-token'
|
||||
@@ -1,34 +0,0 @@
|
||||
[global_tags]
|
||||
|
||||
[agent]
|
||||
debug = true
|
||||
quiet = false
|
||||
interval = "60s"
|
||||
round_interval = true
|
||||
metric_batch_size = 1000
|
||||
metric_buffer_limit = 10000
|
||||
collection_jitter = "0s"
|
||||
flush_interval = "10s"
|
||||
flush_jitter = "0s"
|
||||
precision = ""
|
||||
hostname = "versitygw"
|
||||
omit_hostname = false
|
||||
|
||||
[[outputs.file]]
|
||||
files = ["stdout"]
|
||||
|
||||
[[outputs.influxdb_v2]]
|
||||
urls = ["http://influxdb:8086"]
|
||||
timeout = "5s"
|
||||
token = "my-super-secret-auth-token"
|
||||
organization = "myorg"
|
||||
bucket = "metrics"
|
||||
|
||||
[[inputs.statsd]]
|
||||
protocol = "udp4"
|
||||
service_address = ":8125"
|
||||
percentiles = [90]
|
||||
metric_separator = "_"
|
||||
datadog_extensions = false
|
||||
allowed_pending_messages = 10000
|
||||
percentile_limit = 1000
|
||||
@@ -1,6 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
. ./aws_env_setup.sh
|
||||
|
||||
aws s3 mb s3://test
|
||||
aws s3 cp docker-compose.yml s3://test/test.yaml
|
||||
@@ -153,14 +153,6 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
# specified, all configured bucket events will be sent to the webhook.
|
||||
#VGW_EVENT_WEBHOOK_URL=
|
||||
|
||||
# Bucket events can be filtered for any of the above event types. The
|
||||
# VGW_EVENT_FILTER option specifies a config file that contains the event
|
||||
# filter rules. The event filter rules are used to determine which events are
|
||||
# sent to the configured event services. Run:
|
||||
# versitygw utils gen-event-filter-config --path .
|
||||
# to generate a default rules file "event_config.json" in the current directory.
|
||||
#VGW_EVENT_FILTER=
|
||||
|
||||
#######################
|
||||
# Debug / Diagnostics #
|
||||
#######################
|
||||
@@ -190,40 +182,7 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
# as a dedicated IAM service.
|
||||
#VGW_IAM_DIR=
|
||||
|
||||
# The Vault options will enable the Vault IAM service with accounts stored in
|
||||
# the HashiCorp Vault service. The Vault URL is the address and port of the
|
||||
# Vault server with the format <IP/host>:<port>. A root taken can be used for
|
||||
# testing, but it is recommended to use the role based authentication in
|
||||
# production. The Vault server certificate, client certificate, and client
|
||||
# certificate key are optional, and will default to not verifying the server
|
||||
# certificate and not using client certificates. The Vault server certificate
|
||||
# is used to verify the Vault server, and the client certificate and key are
|
||||
# used to authenticate the gateway to the Vault server. See wiki documentation
|
||||
# for an example of using Vault in dev mode with the gateway.
|
||||
#VGW_IAM_VAULT_ENDPOINT_URL=
|
||||
#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
|
||||
#VGW_IAM_VAULT_MOUNT_PATH=
|
||||
#VGW_IAM_VAULT_ROOT_TOKEN=
|
||||
#VGW_IAM_VAULT_ROLE_ID=
|
||||
#VGW_IAM_VAULT_ROLE_SECRET=
|
||||
#VGW_IAM_VAULT_SERVER_CERT=
|
||||
#VGW_IAM_VAULT_CLIENT_CERT=
|
||||
#VGW_IAM_VAULT_CLIENT_CERT_KEY=
|
||||
|
||||
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
|
||||
# stores the account information JSON encoded in an S3 object. This should use
|
||||
# a bucket that is not accessible to general users when using s3 backend to
|
||||
# prevent access to account credentials. This IAM service is added for
|
||||
# convenience, but is not considered as secure or scalable as a dedicated IAM
|
||||
# service.
|
||||
#VGW_S3_IAM_ACCESS_KEY=
|
||||
#VGW_S3_IAM_SECRET_KEY=
|
||||
#VGW_S3_IAM_REGION=
|
||||
#VGW_S3_IAM_ENDPOINT=
|
||||
#VGW_S3_IAM_BUCKET=
|
||||
#VGW_S3_IAM_NO_VERIFY=
|
||||
|
||||
# The LDAP options will enable the LDAP IAM service with accounts stored in an
|
||||
# The ldap options will enable the LDAP IAM service with accounts stored in an
|
||||
# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
|
||||
# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
|
||||
# secret credentials and role respectively. The other options are used to
|
||||
@@ -237,8 +196,19 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_IAM_LDAP_ACCESS_ATR=
|
||||
#VGW_IAM_LDAP_SECRET_ATR=
|
||||
#VGW_IAM_LDAP_ROLE_ATR=
|
||||
#VGW_IAM_LDAP_USER_ID_ATR=
|
||||
#VGW_IAM_LDAP_GROUP_ID_ATR=
|
||||
|
||||
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
|
||||
# stores the account information JSON encoded in an S3 object. This should use
|
||||
# a bucket that is not accessible to general users when using s3 backend to
|
||||
# prevent access to account credentials. This IAM service is added for
|
||||
# convenience, but is not considered as secure or scalable as a dedicated IAM
|
||||
# service.
|
||||
#VGW_S3_IAM_ACCESS_KEY=
|
||||
#VGW_S3_IAM_SECRET_KEY=
|
||||
#VGW_S3_IAM_REGION=
|
||||
#VGW_S3_IAM_ENDPOINT=
|
||||
#VGW_S3_IAM_BUCKET=
|
||||
#VGW_S3_IAM_NO_VERIFY=
|
||||
|
||||
###############
|
||||
# IAM caching #
|
||||
@@ -258,29 +228,6 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_IAM_CACHE_TTL=120
|
||||
#VGW_IAM_CACHE_PRUNE=3600
|
||||
|
||||
###########
|
||||
# Metrics #
|
||||
###########
|
||||
|
||||
# The metrics service name is a tag that is added to all metrics to help
|
||||
# identify the source of the metrics. This is especially useful when multiple
|
||||
# gateways are running. The default is the hostname of the system.
|
||||
#VGW_METRICS_SERVICE_NAME=$HOSTNAME
|
||||
|
||||
# The metrics service will send metrics to the configured statsd servers. The
|
||||
# servers are specified as a comma separated list of host:port pairs. The
|
||||
# default is to not send metrics to any statsd servers. The gateway uses
|
||||
# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
|
||||
#VGW_METRICS_STATSD_SERVERS=
|
||||
|
||||
# The metrics service will send metrics to the configured dogstatsd servers.
|
||||
# The servers are specified as a comma separated list of host:port pairs. The
|
||||
# default is to not send metrics to any dogstatsd servers. Generally
|
||||
# DataDog recommends installing a local agent to collect metrics and forward
|
||||
# them to the DataDog service. In this case the option value would be the
|
||||
# local agent address: 127.0.0.1:8125.
|
||||
#VGW_METRICS_DOGSTATS_SERVERS=
|
||||
|
||||
######################################
|
||||
# VersityGW Backend Specific Options #
|
||||
######################################
|
||||
@@ -307,15 +254,11 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_CHOWN_UID=false
|
||||
#VGW_CHOWN_GID=false
|
||||
|
||||
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
|
||||
# to directories at the top level gateway directory as buckets.
|
||||
#VGW_BUCKET_LINKS=false
|
||||
|
||||
# The default permissions mode when creating new directories is 0755. Use
|
||||
# VGW_DIR_PERMS option to set a different mode for any new directory that the
|
||||
# gateway creates. This applies to buckets created through the gateway as well
|
||||
# as any parent directories automatically created with object uploads.
|
||||
#VGW_DIR_PERMS=0755
|
||||
# The VGW_READ_ONLY option will disable all write operations to the backend
|
||||
# filesystem. This is useful for creating a read-only gateway for clients.
|
||||
# This will prevent any PUT, POST, DELETE, and multipart upload operations
|
||||
# for objects and buckets as well as preventing updating metadata for objects.
|
||||
#VGW_READ_ONLY=false
|
||||
|
||||
###########
|
||||
# scoutfs #
|
||||
@@ -348,22 +291,11 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_CHOWN_UID=false
|
||||
#VGW_CHOWN_GID=false
|
||||
|
||||
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
|
||||
# to directories at the top level gateway directory as buckets.
|
||||
#VGW_BUCKET_LINKS=false
|
||||
|
||||
# The default permissions mode when creating new directories is 0755. Use
|
||||
# VGW_DIR_PERMS option to set a different mode for any new directory that the
|
||||
# gateway creates. This applies to buckets created through the gateway as well
|
||||
# as any parent directories automatically created with object uploads.
|
||||
#VGW_DIR_PERMS=0755
|
||||
|
||||
# The default behavior of the gateway is to automatically set the noarchive
|
||||
# flag on the multipart upload parts while the multipart upload is in progress.
|
||||
# This is to prevent the parts from being archived since they are temporary
|
||||
# and will be deleted after the multipart upload is completed or aborted. The
|
||||
# VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
|
||||
#VGW_DISABLE_NOARCHIVE=false
|
||||
# The VGW_READ_ONLY option will disable all write operations to the backend
|
||||
# filesystem. This is useful for creating a read-only gateway for clients.
|
||||
# This will prevent any PUT, POST, DELETE, and multipart upload operations
|
||||
# for objects and buckets as well as preventing updating metadata for objects.
|
||||
#VGW_READ_ONLY=false
|
||||
|
||||
######
|
||||
# s3 #
|
||||
|
||||
109
go.mod
109
go.mod
@@ -1,83 +1,70 @@
|
||||
module github.com/versity/versitygw
|
||||
|
||||
go 1.23.0
|
||||
|
||||
toolchain go1.24.1
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
|
||||
github.com/DataDog/datadog-go/v5 v5.6.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1
|
||||
github.com/aws/smithy-go v1.22.3
|
||||
github.com/go-ldap/ldap/v3 v3.4.10
|
||||
github.com/gofiber/fiber/v2 v2.52.6
|
||||
github.com/google/go-cmp v0.7.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
|
||||
github.com/aws/smithy-go v1.20.2
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/gofiber/fiber/v2 v2.52.4
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/vault-client-go v0.4.3
|
||||
github.com/nats-io/nats.go v1.41.0
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/pkg/xattr v0.4.10
|
||||
github.com/nats-io/nats.go v1.34.1
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.47
|
||||
github.com/smira/go-statsd v1.3.4
|
||||
github.com/urfave/cli/v2 v2.27.6
|
||||
github.com/valyala/fasthttp v1.60.0
|
||||
github.com/urfave/cli/v2 v2.27.1
|
||||
github.com/valyala/fasthttp v1.52.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
|
||||
golang.org/x/sync v0.13.0
|
||||
golang.org/x/sys v0.32.0
|
||||
golang.org/x/sys v0.19.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.10 // indirect
|
||||
github.com/nats-io/nkeys v0.4.7 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
golang.org/x/crypto v0.37.0 // indirect
|
||||
golang.org/x/net v0.38.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
golang.org/x/time v0.11.0 // indirect
|
||||
golang.org/x/crypto v0.22.0 // indirect
|
||||
golang.org/x/net v0.24.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.13
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.66
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
|
||||
github.com/klauspost/compress v1.17.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
|
||||
)
|
||||
|
||||
318
go.sum
318
go.sum
@@ -1,187 +1,131 @@
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2/go.mod h1:SqINnQ9lVVdRlyC8cd1lCI0SdX4n2paeABd2K8ggfnE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 h1:Bg8m3nq/X1DeePkAbCfb6ml6F3F0IunEhE8TMh+lY48=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
|
||||
github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
|
||||
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.13 h1:RgdPqWoE8nPpIekpVpDJsBckbqT4Liiaq9f35pbTh1Y=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.13/go.mod h1:NI28qs/IOUIRhsR7GQ/JdexoqRN9tDxkIrYZq0SOF44=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.66 h1:aKpEKaTy6n4CEJeYI1MNj97oSDLi4xro3UzQfwf5RWE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.66/go.mod h1:xQ5SusDmHb/fy55wU0QqTy0yNfLqxzec59YcsRZB+rI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 h1:s43gLuY+zGmtpx+KybfFP4IckopmTfDOPdlf/L++N5I=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71/go.mod h1:KH6wWmY3O3c/jVAjHk0MGzVAFDxkOSt42Eoe4ZO4ge0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 h1:2Ku1xwAohSSXHR1tpAnyVDSQSxoDMA+/NZBytW+f4qg=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3QasqBfuAFHY4Pk5g=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
|
||||
github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
|
||||
github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
|
||||
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
|
||||
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
|
||||
github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
|
||||
github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
|
||||
github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
|
||||
github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
|
||||
github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
|
||||
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
|
||||
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
|
||||
github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
|
||||
github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
|
||||
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
|
||||
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/nats-io/nats.go v1.41.0 h1:PzxEva7fflkd+n87OtQTXqCTyLfIIMFJBpyccHLE2Ko=
|
||||
github.com/nats-io/nats.go v1.41.0/go.mod h1:wV73x0FSI/orHPSYoyMeJB+KajMDoWyXmFaRrrYaaTo=
|
||||
github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc=
|
||||
github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
|
||||
github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
|
||||
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
|
||||
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
|
||||
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
|
||||
github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
|
||||
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
|
||||
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
|
||||
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
|
||||
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
|
||||
github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
|
||||
github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
|
||||
github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.60.0 h1:kBRYS0lOhVJ6V+bYN8PqAHELKHtXqwq9zNMLKx1MBsw=
|
||||
github.com/valyala/fasthttp v1.60.0/go.mod h1:iY4kDgV3Gc6EqhRZ8icqcmlG6bqhcDXfuHgTO4FXCvc=
|
||||
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
|
||||
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
@@ -190,87 +134,49 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
|
||||
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
|
||||
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
@@ -278,25 +184,17 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
|
||||
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,284 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
type Action struct {
|
||||
Name string
|
||||
Service string
|
||||
}
|
||||
|
||||
var (
|
||||
ActionMap map[string]Action
|
||||
)
|
||||
|
||||
var (
|
||||
ActionUndetected = "ActionUnDetected"
|
||||
ActionAbortMultipartUpload = "s3_AbortMultipartUpload"
|
||||
ActionCompleteMultipartUpload = "s3_CompleteMultipartUpload"
|
||||
ActionCopyObject = "s3_CopyObject"
|
||||
ActionCreateBucket = "s3_CreateBucket"
|
||||
ActionCreateMultipartUpload = "s3_CreateMultipartUpload"
|
||||
ActionDeleteBucket = "s3_DeleteBucket"
|
||||
ActionDeleteBucketPolicy = "s3_DeleteBucketPolicy"
|
||||
ActionDeleteBucketTagging = "s3_DeleteBucketTagging"
|
||||
ActionDeleteObject = "s3_DeleteObject"
|
||||
ActionDeleteObjectTagging = "s3_DeleteObjectTagging"
|
||||
ActionDeleteObjects = "s3_DeleteObjects"
|
||||
ActionGetBucketAcl = "s3_GetBucketAcl"
|
||||
ActionGetBucketPolicy = "s3_GetBucketPolicy"
|
||||
ActionGetBucketTagging = "s3_GetBucketTagging"
|
||||
ActionGetBucketVersioning = "s3_GetBucketVersioning"
|
||||
ActionGetObject = "s3_GetObject"
|
||||
ActionGetObjectAcl = "s3_GetObjectAcl"
|
||||
ActionGetObjectAttributes = "s3_GetObjectAttributes"
|
||||
ActionGetObjectLegalHold = "s3_GetObjectLegalHold"
|
||||
ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
|
||||
ActionGetObjectRetention = "s3_GetObjectRetention"
|
||||
ActionGetObjectTagging = "s3_GetObjectTagging"
|
||||
ActionHeadBucket = "s3_HeadBucket"
|
||||
ActionHeadObject = "s3_HeadObject"
|
||||
ActionListAllMyBuckets = "s3_ListAllMyBuckets"
|
||||
ActionListMultipartUploads = "s3_ListMultipartUploads"
|
||||
ActionListObjectVersions = "s3_ListObjectVersions"
|
||||
ActionListObjects = "s3_ListObjects"
|
||||
ActionListObjectsV2 = "s3_ListObjectsV2"
|
||||
ActionListParts = "s3_ListParts"
|
||||
ActionPutBucketAcl = "s3_PutBucketAcl"
|
||||
ActionPutBucketPolicy = "s3_PutBucketPolicy"
|
||||
ActionPutBucketTagging = "s3_PutBucketTagging"
|
||||
ActionPutBucketVersioning = "s3_PutBucketVersioning"
|
||||
ActionPutObject = "s3_PutObject"
|
||||
ActionPutObjectAcl = "s3_PutObjectAcl"
|
||||
ActionPutObjectLegalHold = "s3_PutObjectLegalHold"
|
||||
ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
|
||||
ActionPutObjectRetention = "s3_PutObjectRetention"
|
||||
ActionPutObjectTagging = "s3_PutObjectTagging"
|
||||
ActionRestoreObject = "s3_RestoreObject"
|
||||
ActionSelectObjectContent = "s3_SelectObjectContent"
|
||||
ActionUploadPart = "s3_UploadPart"
|
||||
ActionUploadPartCopy = "s3_UploadPartCopy"
|
||||
ActionPutBucketOwnershipControls = "s3_PutBucketOwnershipControls"
|
||||
ActionGetBucketOwnershipControls = "s3_GetBucketOwnershipControls"
|
||||
ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
|
||||
ActionPutBucketCors = "s3_PutBucketCors"
|
||||
ActionGetBucketCors = "s3_GetBucketCors"
|
||||
ActionDeleteBucketCors = "s3_DeleteBucketCors"
|
||||
|
||||
// Admin actions
|
||||
ActionAdminCreateUser = "admin_CreateUser"
|
||||
ActionAdminUpdateUser = "admin_UpdateUser"
|
||||
ActionAdminDeleteUser = "admin_DeleteUser"
|
||||
ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
|
||||
ActionAdminListUsers = "admin_ListUsers"
|
||||
ActionAdminListBuckets = "admin_ListBuckets"
|
||||
)
|
||||
|
||||
func init() {
|
||||
ActionMap = make(map[string]Action)
|
||||
|
||||
ActionMap[ActionUndetected] = Action{
|
||||
Name: "ActionUnDetected",
|
||||
Service: "unknown",
|
||||
}
|
||||
|
||||
ActionMap[ActionAbortMultipartUpload] = Action{
|
||||
Name: "AbortMultipartUpload",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCompleteMultipartUpload] = Action{
|
||||
Name: "CompleteMultipartUpload",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCopyObject] = Action{
|
||||
Name: "CopyObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCreateBucket] = Action{
|
||||
Name: "CreateBucket",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCreateMultipartUpload] = Action{
|
||||
Name: "CreateMultipartUpload",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteBucket] = Action{
|
||||
Name: "DeleteBucket",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteBucketPolicy] = Action{
|
||||
Name: "DeleteBucketPolicy",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteBucketTagging] = Action{
|
||||
Name: "DeleteBucketTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteObject] = Action{
|
||||
Name: "DeleteObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteObjectTagging] = Action{
|
||||
Name: "DeleteObjectTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteObjects] = Action{
|
||||
Name: "DeleteObjects",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketAcl] = Action{
|
||||
Name: "GetBucketAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketPolicy] = Action{
|
||||
Name: "GetBucketPolicy",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketTagging] = Action{
|
||||
Name: "GetBucketTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketVersioning] = Action{
|
||||
Name: "GetBucketVersioning",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObject] = Action{
|
||||
Name: "GetObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectAcl] = Action{
|
||||
Name: "GetObjectAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectAttributes] = Action{
|
||||
Name: "GetObjectAttributes",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectLegalHold] = Action{
|
||||
Name: "GetObjectLegalHold",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectLockConfiguration] = Action{
|
||||
Name: "GetObjectLockConfiguration",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectRetention] = Action{
|
||||
Name: "GetObjectRetention",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectTagging] = Action{
|
||||
Name: "GetObjectTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionHeadBucket] = Action{
|
||||
Name: "HeadBucket",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionHeadObject] = Action{
|
||||
Name: "HeadObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListAllMyBuckets] = Action{
|
||||
Name: "ListAllMyBuckets",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListMultipartUploads] = Action{
|
||||
Name: "ListMultipartUploads",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListObjectVersions] = Action{
|
||||
Name: "ListObjectVersions",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListObjects] = Action{
|
||||
Name: "ListObjects",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListObjectsV2] = Action{
|
||||
Name: "ListObjectsV2",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListParts] = Action{
|
||||
Name: "ListParts",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketAcl] = Action{
|
||||
Name: "PutBucketAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketPolicy] = Action{
|
||||
Name: "PutBucketPolicy",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketTagging] = Action{
|
||||
Name: "PutBucketTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketVersioning] = Action{
|
||||
Name: "PutBucketVersioning",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObject] = Action{
|
||||
Name: "PutObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectAcl] = Action{
|
||||
Name: "PutObjectAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectLegalHold] = Action{
|
||||
Name: "PutObjectLegalHold",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectLockConfiguration] = Action{
|
||||
Name: "PutObjectLockConfiguration",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectRetention] = Action{
|
||||
Name: "PutObjectRetention",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectTagging] = Action{
|
||||
Name: "PutObjectTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionRestoreObject] = Action{
|
||||
Name: "RestoreObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionSelectObjectContent] = Action{
|
||||
Name: "SelectObjectContent",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionUploadPart] = Action{
|
||||
Name: "UploadPart",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionUploadPartCopy] = Action{
|
||||
Name: "UploadPartCopy",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketCors] = Action{
|
||||
Name: "PutBucketCors",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketCors] = Action{
|
||||
Name: "GetBucketCors",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteBucketCors] = Action{
|
||||
Name: "DeleteBucketCors",
|
||||
Service: "s3",
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
dogstats "github.com/DataDog/datadog-go/v5/statsd"
|
||||
)
|
||||
|
||||
// vgwDogStatsd metrics type
|
||||
type vgwDogStatsd struct {
|
||||
c *dogstats.Client
|
||||
}
|
||||
|
||||
var (
|
||||
rateSampleAlways = 1.0
|
||||
)
|
||||
|
||||
// newDogStatsd takes a server address and returns a statsd merics
|
||||
func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
|
||||
c, err := dogstats.New(server,
|
||||
dogstats.WithMaxMessagesPerPayload(1000),
|
||||
dogstats.WithNamespace("versitygw"),
|
||||
dogstats.WithTags([]string{
|
||||
"service:" + service,
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &vgwDogStatsd{c: c}, nil
|
||||
}
|
||||
|
||||
// Close closes statsd connections
|
||||
func (s *vgwDogStatsd) Close() {
|
||||
s.c.Close()
|
||||
}
|
||||
|
||||
func (t Tag) ddString() string {
|
||||
if t.Value == "" {
|
||||
return t.Key
|
||||
}
|
||||
return fmt.Sprintf("%v:%v", t.Key, t.Value)
|
||||
}
|
||||
|
||||
// Add adds value to key
|
||||
func (s *vgwDogStatsd) Add(key string, value int64, tags ...Tag) {
|
||||
stags := make([]string, len(tags))
|
||||
for i, t := range tags {
|
||||
stags[i] = t.ddString()
|
||||
}
|
||||
s.c.Count(key, value, stags, rateSampleAlways)
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
var (
|
||||
// max size of data items to buffer before dropping
|
||||
// new incoming data items
|
||||
dataItemCount = 100000
|
||||
)
|
||||
|
||||
// Tag is added metadata for metrics
|
||||
type Tag struct {
|
||||
// Key is tag name
|
||||
Key string
|
||||
// Value is tag data
|
||||
Value string
|
||||
}
|
||||
|
||||
// Manager is a manager of metrics plugins
|
||||
type Manager struct {
|
||||
wg sync.WaitGroup
|
||||
ctx context.Context
|
||||
|
||||
config Config
|
||||
|
||||
publishers []publisher
|
||||
addDataChan chan datapoint
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ServiceName string
|
||||
StatsdServers string
|
||||
DogStatsdServers string
|
||||
}
|
||||
|
||||
// NewManager initializes metrics plugins and returns a new metrics manager
|
||||
func NewManager(ctx context.Context, conf Config) (*Manager, error) {
|
||||
if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if conf.ServiceName == "" {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
||||
}
|
||||
conf.ServiceName = hostname
|
||||
}
|
||||
|
||||
addDataChan := make(chan datapoint, dataItemCount)
|
||||
|
||||
mgr := &Manager{
|
||||
addDataChan: addDataChan,
|
||||
ctx: ctx,
|
||||
config: conf,
|
||||
}
|
||||
|
||||
// setup statsd endpoints
|
||||
if len(conf.StatsdServers) > 0 {
|
||||
statsdServers := strings.Split(conf.StatsdServers, ",")
|
||||
|
||||
for _, server := range statsdServers {
|
||||
statsd, err := newStatsd(server, conf.ServiceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mgr.publishers = append(mgr.publishers, statsd)
|
||||
}
|
||||
}
|
||||
|
||||
// setup dogstatsd endpoints
|
||||
if len(conf.DogStatsdServers) > 0 {
|
||||
dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
|
||||
|
||||
for _, server := range dogStatsdServers {
|
||||
dogStatsd, err := newDogStatsd(server, conf.ServiceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mgr.publishers = append(mgr.publishers, dogStatsd)
|
||||
}
|
||||
}
|
||||
|
||||
mgr.wg.Add(1)
|
||||
go mgr.addForwarder(addDataChan)
|
||||
|
||||
return mgr, nil
|
||||
}
|
||||
|
||||
func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) {
|
||||
// In case of Authentication failures, url parsing ...
|
||||
if action == "" {
|
||||
action = ActionUndetected
|
||||
}
|
||||
|
||||
a := ActionMap[action]
|
||||
reqTags := []Tag{
|
||||
{Key: "method", Value: ctx.Method()},
|
||||
{Key: "api", Value: a.Service},
|
||||
{Key: "action", Value: a.Name},
|
||||
}
|
||||
|
||||
reqStatus := status
|
||||
|
||||
if err != nil {
|
||||
var apierr s3err.APIError
|
||||
if errors.As(err, &apierr) {
|
||||
reqStatus = apierr.HTTPStatusCode
|
||||
} else {
|
||||
reqStatus = http.StatusInternalServerError
|
||||
}
|
||||
}
|
||||
if reqStatus == 0 {
|
||||
reqStatus = http.StatusOK
|
||||
}
|
||||
|
||||
reqTags = append(reqTags, Tag{
|
||||
Key: "status",
|
||||
Value: fmt.Sprintf("%v", reqStatus),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
m.increment("failed_count", reqTags...)
|
||||
} else {
|
||||
m.increment("success_count", reqTags...)
|
||||
}
|
||||
|
||||
switch action {
|
||||
case ActionPutObject:
|
||||
m.add("bytes_written", count, reqTags...)
|
||||
m.increment("object_created_count", reqTags...)
|
||||
case ActionCompleteMultipartUpload:
|
||||
m.increment("object_created_count", reqTags...)
|
||||
case ActionUploadPart:
|
||||
m.add("bytes_written", count, reqTags...)
|
||||
case ActionGetObject:
|
||||
m.add("bytes_read", count, reqTags...)
|
||||
case ActionDeleteObject:
|
||||
m.increment("object_removed_count", reqTags...)
|
||||
case ActionDeleteObjects:
|
||||
m.add("object_removed_count", count, reqTags...)
|
||||
}
|
||||
}
|
||||
|
||||
// increment increments the key by one
|
||||
func (m *Manager) increment(key string, tags ...Tag) {
|
||||
m.add(key, 1, tags...)
|
||||
}
|
||||
|
||||
// add adds value to key
|
||||
func (m *Manager) add(key string, value int64, tags ...Tag) {
|
||||
if m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
d := datapoint{
|
||||
key: key,
|
||||
value: value,
|
||||
tags: tags,
|
||||
}
|
||||
|
||||
select {
|
||||
case m.addDataChan <- d:
|
||||
default:
|
||||
// channel full, drop the updates
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes metrics channels, waits for data to complete, closes all plugins
|
||||
func (m *Manager) Close() {
|
||||
// drain the datapoint channels
|
||||
close(m.addDataChan)
|
||||
m.wg.Wait()
|
||||
|
||||
// close all publishers
|
||||
for _, p := range m.publishers {
|
||||
p.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// publisher is the interface for interacting with the metrics plugins
|
||||
type publisher interface {
|
||||
Add(key string, value int64, tags ...Tag)
|
||||
Close()
|
||||
}
|
||||
|
||||
func (m *Manager) addForwarder(addChan <-chan datapoint) {
|
||||
for data := range addChan {
|
||||
for _, s := range m.publishers {
|
||||
s.Add(data.key, data.value, data.tags...)
|
||||
}
|
||||
}
|
||||
m.wg.Done()
|
||||
}
|
||||
|
||||
type datapoint struct {
|
||||
key string
|
||||
value int64
|
||||
tags []Tag
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/smira/go-statsd"
|
||||
)
|
||||
|
||||
// vgwStatsd metrics type
|
||||
type vgwStatsd struct {
|
||||
c *statsd.Client
|
||||
}
|
||||
|
||||
// newStatsd takes a server address and returns a statsd merics
|
||||
// Supply service name to be used as a tag to identify the spcific
|
||||
// gateway instance, this may typically be the gateway hostname
|
||||
func newStatsd(server string, service string) (*vgwStatsd, error) {
|
||||
c := statsd.NewClient(
|
||||
server,
|
||||
statsd.MetricPrefix("versitygw."),
|
||||
statsd.TagStyle(statsd.TagFormatInfluxDB),
|
||||
statsd.DefaultTags(statsd.StringTag("service", service)),
|
||||
)
|
||||
return &vgwStatsd{c: c}, nil
|
||||
}
|
||||
|
||||
// Close closes statsd connections
|
||||
func (s *vgwStatsd) Close() {
|
||||
s.c.Close()
|
||||
}
|
||||
|
||||
// Add adds value to key
|
||||
func (s *vgwStatsd) Add(key string, value int64, tags ...Tag) {
|
||||
stags := make([]statsd.Tag, len(tags))
|
||||
for i, t := range tags {
|
||||
stags[i] = statsd.StringTag(t.Key, t.Value)
|
||||
}
|
||||
s.c.Incr(key, value, stags...)
|
||||
}
|
||||
122
runtests.sh
122
runtests.sh
@@ -5,27 +5,15 @@ rm -rf /tmp/gw
|
||||
mkdir /tmp/gw
|
||||
rm -rf /tmp/covdata
|
||||
mkdir /tmp/covdata
|
||||
rm -rf /tmp/versioing.covdata
|
||||
mkdir /tmp/versioning.covdata
|
||||
rm -rf /tmp/versioningdir
|
||||
mkdir /tmp/versioningdir
|
||||
|
||||
# setup tls certificate and key
|
||||
ECHO "Generating TLS certificate and key in the cert.pem and key.pem files"
|
||||
|
||||
openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
|
||||
openssl req -new -x509 -key key.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
|
||||
|
||||
ECHO "Running the sdk test over http"
|
||||
# run server in background not versioning-enabled
|
||||
# port: 7070(default)
|
||||
# run server in background
|
||||
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
|
||||
GW_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if gateway process is still running
|
||||
# check if server is still running
|
||||
if ! kill -0 $GW_PID; then
|
||||
echo "server no longer running"
|
||||
exit 1
|
||||
@@ -51,110 +39,8 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_PID
|
||||
|
||||
ECHO "Running the sdk test over https"
|
||||
|
||||
# run server in background with TLS certificate
|
||||
# port: 7071(default)
|
||||
GOCOVERDIR=/tmp/https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7071 -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
|
||||
GW_HTTPS_PID=$!
|
||||
|
||||
sleep 1
|
||||
|
||||
# check if https gateway process is still running
|
||||
if ! kill -0 $GW_HTTPS_PID; then
|
||||
echo "server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
# full flow tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow; then
|
||||
echo "full flow tests failed"
|
||||
kill $GW_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
# posix tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 posix; then
|
||||
echo "posix tests failed"
|
||||
kill $GW_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
# iam tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 iam; then
|
||||
echo "iam tests failed"
|
||||
kill $GW_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kill $GW_HTTPS_PID
|
||||
|
||||
|
||||
ECHO "Running the sdk test over http against the versioning-enabled gateway"
|
||||
# run server in background versioning-enabled
|
||||
# port: 7072
|
||||
GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7072 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
|
||||
GW_VS_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if versioning-enabled gateway process is still running
|
||||
if ! kill -0 $GW_VS_PID; then
|
||||
echo "versioning-enabled server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
# full flow tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs; then
|
||||
echo "versioning-enabled full-flow tests failed"
|
||||
kill $GW_VS_PID
|
||||
exit 1
|
||||
fi
|
||||
# posix tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 posix -vs; then
|
||||
echo "versiongin-enabled posix tests failed"
|
||||
kill $GW_VS_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_VS_PID
|
||||
|
||||
ECHO "Running the sdk test over https against the versioning-enabled gateway"
|
||||
# run server in background versioning-enabled
|
||||
# port: 7073
|
||||
GOCOVERDIR=/tmp/versioning.https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7073 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
|
||||
GW_VS_HTTPS_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if versioning-enabled gateway process is still running
|
||||
if ! kill -0 $GW_VS_HTTPS_PID; then
|
||||
echo "versioning-enabled server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
# full flow tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs; then
|
||||
echo "versioning-enabled full-flow tests failed"
|
||||
kill $GW_VS_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
# posix tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 posix -vs; then
|
||||
echo "versiongin-enabled posix tests failed"
|
||||
kill $GW_VS_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_VS_HTTPS_PID
|
||||
|
||||
exit 0
|
||||
|
||||
# if the above binary was built with -cover enabled (make testbin),
|
||||
|
||||
@@ -19,13 +19,12 @@ import (
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
type S3AdminRouter struct{}
|
||||
|
||||
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
|
||||
controller := controllers.NewAdminController(iam, be, logger)
|
||||
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
|
||||
controller := controllers.NewAdminController(iam, be)
|
||||
|
||||
// CreateUser admin api
|
||||
app.Patch("/create-user", controller.CreateUser)
|
||||
@@ -33,9 +32,6 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
|
||||
// DeleteUsers admin api
|
||||
app.Patch("/delete-user", controller.DeleteUser)
|
||||
|
||||
// UpdateUser admin api
|
||||
app.Patch("/update-user", controller.UpdateUser)
|
||||
|
||||
// ListUsers admin api
|
||||
app.Patch("/list-users", controller.ListUsers)
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
type S3AdminServer struct {
|
||||
@@ -33,7 +32,7 @@ type S3AdminServer struct {
|
||||
cert *tls.Certificate
|
||||
}
|
||||
|
||||
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
|
||||
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
|
||||
server := &S3AdminServer{
|
||||
app: app,
|
||||
backend: be,
|
||||
@@ -47,16 +46,14 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
|
||||
|
||||
// Logging middlewares
|
||||
app.Use(logger.New())
|
||||
app.Use(middlewares.DecodeURL(l, nil))
|
||||
app.Use(middlewares.DecodeURL(nil))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
|
||||
app.Use(middlewares.VerifyMD5Body(nil))
|
||||
app.Use(middlewares.AclParser(be, nil))
|
||||
|
||||
// Admin role checker
|
||||
app.Use(middlewares.IsAdmin(l))
|
||||
|
||||
server.router.Init(app, be, iam, l)
|
||||
server.router.Init(app, be, iam)
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
@@ -16,188 +16,107 @@ package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type AdminController struct {
|
||||
iam auth.IAMService
|
||||
be backend.Backend
|
||||
l s3log.AuditLogger
|
||||
}
|
||||
|
||||
func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
|
||||
return AdminController{iam: iam, be: be, l: l}
|
||||
func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
|
||||
return AdminController{iam: iam, be: be}
|
||||
}
|
||||
|
||||
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
var usr auth.Account
|
||||
err := xml.Unmarshal(ctx.Body(), &usr)
|
||||
err := json.Unmarshal(ctx.Body(), &usr)
|
||||
if err != nil {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
})
|
||||
return fmt.Errorf("failed to parse request body: %w", err)
|
||||
}
|
||||
|
||||
if !usr.Role.IsValid() {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
})
|
||||
if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
|
||||
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
|
||||
}
|
||||
|
||||
err = c.iam.CreateAccount(usr)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "user already exists") {
|
||||
err = s3err.GetAPIError(s3err.ErrAdminUserExists)
|
||||
}
|
||||
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
})
|
||||
return fmt.Errorf("failed to create a user: %w", err)
|
||||
}
|
||||
|
||||
return SendResponse(ctx, nil,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
Status: http.StatusCreated,
|
||||
})
|
||||
}
|
||||
|
||||
func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
|
||||
access := ctx.Query("access")
|
||||
if access == "" {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
}
|
||||
|
||||
var props auth.MutableProps
|
||||
if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
}
|
||||
|
||||
err := c.iam.UpdateUserAccount(access, props)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "user not found") {
|
||||
err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
|
||||
}
|
||||
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
}
|
||||
|
||||
return SendResponse(ctx, nil,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
return ctx.SendString("The user has been created successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
|
||||
access := ctx.Query("access")
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
|
||||
err := c.iam.DeleteUserAccount(access)
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminDeleteUser,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.SendString("The user has been deleted successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
accs, err := c.iam.ListUserAccounts()
|
||||
return SendXMLResponse(ctx,
|
||||
auth.ListUserAccountsResult{
|
||||
Accounts: accs,
|
||||
}, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminListUsers,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.JSON(accs)
|
||||
}
|
||||
|
||||
func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
owner := ctx.Query("owner")
|
||||
bucket := ctx.Query("bucket")
|
||||
|
||||
accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
|
||||
if err != nil {
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return err
|
||||
}
|
||||
if len(accs) > 0 {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return fmt.Errorf("user specified as the new bucket owner does not exist")
|
||||
}
|
||||
|
||||
acl := auth.ACL{
|
||||
Owner: owner,
|
||||
Grantees: []auth.Grantee{
|
||||
{
|
||||
Permission: auth.PermissionFullControl,
|
||||
Access: owner,
|
||||
Type: types.TypeCanonicalUser,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
aclParsed, err := json.Marshal(acl)
|
||||
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
|
||||
if err != nil {
|
||||
return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return ctx.Status(201).SendString("Bucket owner has been updated successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
|
||||
buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
|
||||
return SendXMLResponse(ctx,
|
||||
s3response.ListBucketsResult{
|
||||
Buckets: buckets,
|
||||
}, err, &MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminListBuckets,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.JSON(buckets)
|
||||
}
|
||||
|
||||
@@ -15,11 +15,12 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
@@ -42,26 +43,33 @@ func TestAdminController_CreateUser(t *testing.T) {
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
succUser := `
|
||||
<Account>
|
||||
<Access>access</Access>
|
||||
<Secret>secret</Secret>
|
||||
<Role>admin</Role>
|
||||
<UserID>0</UserID>
|
||||
<GroupID>0</GroupID>
|
||||
</Account>
|
||||
`
|
||||
invuser := `
|
||||
<Account>
|
||||
<Access>access</Access>
|
||||
<Secret>secret</Secret>
|
||||
<Role>invalid_role</Role>
|
||||
<UserID>0</UserID>
|
||||
<GroupID>0</GroupID>
|
||||
</Account>
|
||||
`
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
usr := auth.Account{
|
||||
Access: "access",
|
||||
Secret: "secret",
|
||||
Role: "invalid role",
|
||||
}
|
||||
|
||||
user, _ := json.Marshal(&usr)
|
||||
|
||||
usr.Role = "admin"
|
||||
|
||||
succUsr, _ := json.Marshal(&usr)
|
||||
|
||||
appErr.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -71,31 +79,31 @@ func TestAdminController_CreateUser(t *testing.T) {
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-create-user-malformed-body",
|
||||
name: "Admin-create-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-user-role",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-requester-role",
|
||||
app: appErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-requester-role",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 201,
|
||||
statusCode: 500,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -111,100 +119,6 @@ func TestAdminController_CreateUser(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminController_UpdateUser(t *testing.T) {
|
||||
type args struct {
|
||||
req *http.Request
|
||||
}
|
||||
|
||||
adminController := AdminController{
|
||||
iam: &IAMServiceMock{
|
||||
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Patch("/update-user", adminController.UpdateUser)
|
||||
|
||||
adminControllerErr := AdminController{
|
||||
iam: &IAMServiceMock{
|
||||
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
|
||||
return auth.ErrNoSuchUser
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
appNotFound := fiber.New()
|
||||
|
||||
appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
|
||||
|
||||
succUser := `
|
||||
<Account>
|
||||
<Secret>secret</Secret>
|
||||
<UserID>0</UserID>
|
||||
<GroupID>0</GroupID>
|
||||
</Account>
|
||||
`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
args args
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-update-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-update-user-missing-access",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
},
|
||||
{
|
||||
name: "Admin-update-user-invalid-request-body",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Admin-update-user-not-found",
|
||||
app: appNotFound,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
if resp.StatusCode != tt.statusCode {
|
||||
t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminController_DeleteUser(t *testing.T) {
|
||||
type args struct {
|
||||
req *http.Request
|
||||
@@ -220,8 +134,22 @@ func TestAdminController_DeleteUser(t *testing.T) {
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appErr.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -238,6 +166,15 @@ func TestAdminController_DeleteUser(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-delete-user-invalid-requester-role",
|
||||
app: appErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
@@ -274,9 +211,30 @@ func TestAdminController_ListUsers(t *testing.T) {
|
||||
}
|
||||
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appErr.Patch("/list-users", adminControllerErr.ListUsers)
|
||||
|
||||
appRoleErr := fiber.New()
|
||||
|
||||
appRoleErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appRoleErr.Patch("/list-users", adminController.ListUsers)
|
||||
|
||||
appSucc := fiber.New()
|
||||
|
||||
appSucc.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appSucc.Patch("/list-users", adminController.ListUsers)
|
||||
|
||||
tests := []struct {
|
||||
@@ -286,6 +244,15 @@ func TestAdminController_ListUsers(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-list-users-access-denied",
|
||||
app: appRoleErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Admin-list-users-iam-error",
|
||||
app: appErr,
|
||||
@@ -324,7 +291,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
}
|
||||
adminController := AdminController{
|
||||
be: &BackendMock{
|
||||
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
|
||||
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
@@ -352,12 +319,39 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
|
||||
|
||||
appRoleErr := fiber.New()
|
||||
|
||||
appRoleErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
|
||||
|
||||
appIamErr := fiber.New()
|
||||
|
||||
appIamErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
|
||||
|
||||
appIamNoSuchUser := fiber.New()
|
||||
|
||||
appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
|
||||
|
||||
tests := []struct {
|
||||
@@ -367,6 +361,15 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Change-bucket-owner-access-denied",
|
||||
app: appRoleErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Change-bucket-owner-check-account-server-error",
|
||||
app: appIamErr,
|
||||
@@ -383,7 +386,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Change-bucket-owner-success",
|
||||
@@ -392,7 +395,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
statusCode: 201,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -421,8 +424,23 @@ func TestAdminController_ListBuckets(t *testing.T) {
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/list-buckets", adminController.ListBuckets)
|
||||
|
||||
appRoleErr := fiber.New()
|
||||
|
||||
appRoleErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -430,6 +448,15 @@ func TestAdminController_ListBuckets(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "List-buckets-incorrect-role",
|
||||
app: appRoleErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "List-buckets-success",
|
||||
app: app,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
@@ -76,7 +77,7 @@ func TestNew(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false)
|
||||
got := New(tt.args.be, tt.args.iam, nil, nil, false)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("New() = %v, want %v", got, tt.want)
|
||||
}
|
||||
@@ -92,7 +93,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, nil
|
||||
},
|
||||
},
|
||||
@@ -109,7 +110,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
|
||||
appErr := fiber.New()
|
||||
s3ApiControllerErr := S3ApiController{
|
||||
be: &BackendMock{
|
||||
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
},
|
||||
},
|
||||
@@ -187,10 +188,10 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
|
||||
return &s3.GetObjectAclOutput{}, nil
|
||||
},
|
||||
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
|
||||
return s3response.GetObjectAttributesResponse{}, nil
|
||||
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
return &s3.GetObjectAttributesOutput{}, nil
|
||||
},
|
||||
GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
|
||||
return &s3.GetObjectOutput{
|
||||
Metadata: map[string]string{"hello": "world"},
|
||||
ContentType: getPtr("application/xml"),
|
||||
@@ -204,19 +205,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
|
||||
return map[string]string{"hello": "world"}, nil
|
||||
},
|
||||
GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
result, err := json.Marshal(types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeCompliance,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
},
|
||||
GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
result := true
|
||||
return &result, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
@@ -232,9 +220,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
|
||||
getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
|
||||
|
||||
invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
|
||||
invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -251,24 +236,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-get-object-retention-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?retention", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-get-object-legal-hold-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?legal-hold", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-invalid-max-parts-string",
|
||||
app: app,
|
||||
@@ -332,15 +299,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-get-object-invalid-checksum-mode",
|
||||
app: app,
|
||||
args: args{
|
||||
req: invalidChecksumMode,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-get-object-success",
|
||||
app: app,
|
||||
@@ -371,11 +329,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
req *http.Request
|
||||
}
|
||||
|
||||
objectLockResult, err := json.Marshal(auth.BucketLockConfig{})
|
||||
if err != nil {
|
||||
t.Errorf("failed to parse object lock result %v", err)
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
@@ -385,30 +338,24 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
return s3response.ListMultipartUploadsResult{}, nil
|
||||
},
|
||||
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
|
||||
return s3response.ListObjectsV2Result{}, nil
|
||||
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return &s3.ListObjectsV2Output{}, nil
|
||||
},
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
return s3response.ListObjectsResult{}, nil
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return &s3.ListObjectsOutput{}, nil
|
||||
},
|
||||
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
return map[string]string{}, nil
|
||||
},
|
||||
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
return s3response.GetBucketVersioningOutput{}, nil
|
||||
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
|
||||
return &s3.GetBucketVersioningOutput{}, nil
|
||||
},
|
||||
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
|
||||
return s3response.ListVersionsResult{}, nil
|
||||
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
|
||||
return &s3.ListObjectVersionsOutput{}, nil
|
||||
},
|
||||
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return []byte{}, nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return objectLockResult, nil
|
||||
},
|
||||
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
return types.ObjectOwnershipBucketOwnerEnforced, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -422,14 +369,14 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
|
||||
app.Get("/:bucket", s3ApiController.ListActions)
|
||||
|
||||
// Error case
|
||||
//Error case
|
||||
s3ApiControllerError := S3ApiController{
|
||||
be: &BackendMock{
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return acldata, nil
|
||||
},
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
@@ -462,15 +409,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-ownership-control-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-tagging-success",
|
||||
app: app,
|
||||
@@ -480,15 +418,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-object-lock-configuration-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket?object-lock", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-acl-success",
|
||||
app: app,
|
||||
@@ -585,7 +514,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
app := fiber.New()
|
||||
|
||||
// Mock valid acl
|
||||
acl := auth.ACL{Owner: "valid access"}
|
||||
acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
|
||||
acldata, err := json.Marshal(acl)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to parse the params: %v", err.Error())
|
||||
@@ -616,6 +545,14 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
</AccessControlPolicy>
|
||||
`
|
||||
|
||||
succBody := `
|
||||
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Owner>
|
||||
<ID>valid access</ID>
|
||||
</Owner>
|
||||
</AccessControlPolicy>
|
||||
`
|
||||
|
||||
tagBody := `
|
||||
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<TagSet>
|
||||
@@ -634,7 +571,8 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
</VersioningConfiguration>
|
||||
`
|
||||
|
||||
policyBody := `{
|
||||
policyBody := `
|
||||
{
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
@@ -646,34 +584,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
}
|
||||
`
|
||||
|
||||
objectLockBody := `
|
||||
<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<ObjectLockEnabled>Enabled</ObjectLockEnabled>
|
||||
<Rule>
|
||||
<DefaultRetention>
|
||||
<Mode>GOVERNANCE</Mode>
|
||||
<Years>2</Years>
|
||||
</DefaultRetention>
|
||||
</Rule>
|
||||
</ObjectLockConfiguration>
|
||||
`
|
||||
|
||||
ownershipBody := `
|
||||
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Rule>
|
||||
<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
|
||||
</Rule>
|
||||
</OwnershipControls>
|
||||
`
|
||||
|
||||
invalidOwnershipBody := `
|
||||
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Rule>
|
||||
<ObjectOwnership>invalid_value</ObjectOwnership>
|
||||
</Rule>
|
||||
</OwnershipControls>
|
||||
`
|
||||
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
@@ -688,21 +598,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
|
||||
return nil
|
||||
},
|
||||
PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
|
||||
PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
|
||||
return nil
|
||||
},
|
||||
PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
|
||||
return nil
|
||||
},
|
||||
PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
|
||||
return nil
|
||||
},
|
||||
PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
|
||||
return nil
|
||||
},
|
||||
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
return types.ObjectOwnershipBucketOwnerPreferred, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
// Mock ctx.Locals
|
||||
@@ -726,18 +627,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
|
||||
// PutBucketAcl incorrect bucket owner case
|
||||
incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
|
||||
incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
|
||||
|
||||
// PutBucketAcl acl success
|
||||
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
|
||||
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
|
||||
aclSuccReq.Header.Set("X-Amz-Acl", "private")
|
||||
|
||||
// Invalid acl body case
|
||||
errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
|
||||
errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
|
||||
|
||||
invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
|
||||
invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -761,42 +660,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-ownership-controls-invalid-ownership",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-ownership-controls-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Put-object-lock-configuration-invalid-body",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Put-object-lock-configuration-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", strings.NewReader(objectLockBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
@@ -869,7 +732,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
req: incorrectBucketOwner,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
statusCode: 403,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-acl-success",
|
||||
@@ -881,16 +744,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Create-bucket-invalid-acl-ownership-combination",
|
||||
app: app,
|
||||
args: args{
|
||||
req: invAclOwnershipReq,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Create-bucket-invalid-bucket-name",
|
||||
name: "Put-bucket-invalid-bucket-name",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/aa", nil),
|
||||
@@ -899,7 +753,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Create-bucket-success",
|
||||
name: "Put-bucket-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
|
||||
@@ -952,19 +806,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
</Tagging>
|
||||
`
|
||||
|
||||
//retentionBody := `
|
||||
//<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
// <Mode>GOVERNANCE</Mode>
|
||||
// <RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
|
||||
//</Retention>
|
||||
//`
|
||||
|
||||
legalHoldBody := `
|
||||
<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>ON</Status>
|
||||
</LegalHold>
|
||||
`
|
||||
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
@@ -974,31 +815,22 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
|
||||
return nil
|
||||
},
|
||||
CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
CopyObjectFunc: func(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
return &s3.CopyObjectOutput{
|
||||
CopyObjectResult: &types.CopyObjectResult{},
|
||||
}, nil
|
||||
},
|
||||
PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
return s3response.PutObjectOutput{}, nil
|
||||
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
|
||||
return "ETag", nil
|
||||
},
|
||||
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
|
||||
return &s3.UploadPartOutput{}, nil
|
||||
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
|
||||
return "hello", nil
|
||||
},
|
||||
PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
|
||||
return nil
|
||||
},
|
||||
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
|
||||
return s3response.CopyPartResult{}, nil
|
||||
},
|
||||
PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
|
||||
return nil
|
||||
},
|
||||
PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
|
||||
return nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
return s3response.CopyObjectResult{}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1023,11 +855,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
|
||||
|
||||
// CopyObject invalid checksum algorithm
|
||||
cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
|
||||
cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
|
||||
|
||||
// PutObjectAcl success
|
||||
aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
aclReq.Header.Set("X-Amz-Acl", "private")
|
||||
@@ -1049,40 +876,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
|
||||
invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
|
||||
|
||||
// PutObject invalid checksum algorithm
|
||||
invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
|
||||
|
||||
// PutObject invalid base64 checksum
|
||||
invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
|
||||
|
||||
// PutObject invalid crc32
|
||||
invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
|
||||
|
||||
// PutObject invalid crc32c
|
||||
invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
|
||||
|
||||
// PutObject invalid sha1
|
||||
invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
|
||||
|
||||
// PutObject invalid sha256
|
||||
invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
|
||||
|
||||
// PutObject multiple checksum headers
|
||||
mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
|
||||
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
|
||||
|
||||
// PutObject checksum algorithm and header mismatch
|
||||
checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
|
||||
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
|
||||
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -1117,42 +910,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "put-object-retention-invalid-request",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
//{
|
||||
// name: "put-object-retention-success",
|
||||
// app: app,
|
||||
// args: args{
|
||||
// req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
|
||||
// },
|
||||
// wantErr: false,
|
||||
// statusCode: 200,
|
||||
//},
|
||||
{
|
||||
name: "put-legal-hold-invalid-request",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "put-legal-hold-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", strings.NewReader(legalHoldBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Put-object-acl-invalid-acl",
|
||||
app: app,
|
||||
@@ -1225,15 +982,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Copy-object-invalid-checksum-algorithm",
|
||||
app: app,
|
||||
args: args{
|
||||
req: cpyInvChecksumAlgo,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Copy-object-success",
|
||||
app: app,
|
||||
@@ -1276,18 +1024,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
DeleteBucketFunc: func(_ context.Context, bucket string) error {
|
||||
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
|
||||
return nil
|
||||
},
|
||||
DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
return nil
|
||||
},
|
||||
DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
return nil
|
||||
},
|
||||
DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1326,23 +1068,6 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
{
|
||||
name: "Delete-bucket-ownership-controls-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
}, {
|
||||
name: "Delete-bucket-policy-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
@@ -1371,9 +1096,6 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
|
||||
DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
return s3response.DeleteResult{}, nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1442,8 +1164,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return acldata, nil
|
||||
},
|
||||
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
|
||||
return nil
|
||||
},
|
||||
AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
|
||||
return nil
|
||||
@@ -1451,9 +1173,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
|
||||
DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error {
|
||||
return nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1473,11 +1192,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return acldata, nil
|
||||
},
|
||||
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
|
||||
return s3err.GetAPIError(7)
|
||||
},
|
||||
}}
|
||||
|
||||
@@ -1569,7 +1285,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
|
||||
ctx.Locals("isRoot", true)
|
||||
ctx.Locals("isDebug", false)
|
||||
ctx.Locals("parsedAcl", auth.ACL{})
|
||||
ctx.Locals("region", "us-east-1")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
@@ -1593,7 +1308,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
|
||||
ctx.Locals("isRoot", true)
|
||||
ctx.Locals("isDebug", false)
|
||||
ctx.Locals("parsedAcl", auth.ACL{})
|
||||
ctx.Locals("region", "us-east-1")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
@@ -1687,7 +1401,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
|
||||
return acldata, nil
|
||||
},
|
||||
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
return nil, s3err.GetAPIError(42)
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1701,9 +1415,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
|
||||
})
|
||||
appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
|
||||
|
||||
invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
|
||||
invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -1720,15 +1431,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Head-object-invalid-checksum-mode",
|
||||
app: app,
|
||||
args: args{
|
||||
req: invChecksumMode,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Head-object-error",
|
||||
app: appErr,
|
||||
@@ -1768,8 +1470,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
return &s3.CompleteMultipartUploadOutput{}, nil
|
||||
},
|
||||
CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
|
||||
return s3response.InitiateMultipartUploadResult{}, nil
|
||||
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return &s3.CreateMultipartUploadOutput{}, nil
|
||||
},
|
||||
SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
return func(w *bufio.Writer) {}
|
||||
@@ -1784,19 +1486,6 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
</SelectObjectContentRequest>
|
||||
`
|
||||
|
||||
completMpBody := `
|
||||
<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Part>
|
||||
<ETag>etag</ETag>
|
||||
<PartNumber>1</PartNumber>
|
||||
</Part>
|
||||
</CompleteMultipartUpload>
|
||||
`
|
||||
|
||||
completMpEmptyBody := `
|
||||
<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></CompleteMultipartUpload>
|
||||
`
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "valid access"})
|
||||
ctx.Locals("isRoot", true)
|
||||
@@ -1806,9 +1495,6 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
})
|
||||
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
|
||||
|
||||
invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
|
||||
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -1820,11 +1506,20 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
name: "Restore-object-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Restore-object-error",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Select-object-content-invalid-body",
|
||||
app: app,
|
||||
@@ -1852,33 +1547,15 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Complete-multipart-upload-empty-parts",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpEmptyBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Complete-multipart-upload-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpBody)),
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(`<root><key>body</key></root>`)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Create-multipart-upload-invalid-checksum-algorithm",
|
||||
app: app,
|
||||
args: args{
|
||||
req: invChecksumAlgo,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Create-multipart-upload-success",
|
||||
app: app,
|
||||
|
||||
@@ -33,9 +33,6 @@ var _ auth.IAMService = &IAMServiceMock{}
|
||||
// ShutdownFunc: func() error {
|
||||
// panic("mock out the Shutdown method")
|
||||
// },
|
||||
// UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
|
||||
// panic("mock out the UpdateUserAccount method")
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// // use mockedIAMService in code that requires auth.IAMService
|
||||
@@ -58,9 +55,6 @@ type IAMServiceMock struct {
|
||||
// ShutdownFunc mocks the Shutdown method.
|
||||
ShutdownFunc func() error
|
||||
|
||||
// UpdateUserAccountFunc mocks the UpdateUserAccount method.
|
||||
UpdateUserAccountFunc func(access string, props auth.MutableProps) error
|
||||
|
||||
// calls tracks calls to the methods.
|
||||
calls struct {
|
||||
// CreateAccount holds details about calls to the CreateAccount method.
|
||||
@@ -84,20 +78,12 @@ type IAMServiceMock struct {
|
||||
// Shutdown holds details about calls to the Shutdown method.
|
||||
Shutdown []struct {
|
||||
}
|
||||
// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
|
||||
UpdateUserAccount []struct {
|
||||
// Access is the access argument value.
|
||||
Access string
|
||||
// Props is the props argument value.
|
||||
Props auth.MutableProps
|
||||
}
|
||||
}
|
||||
lockCreateAccount sync.RWMutex
|
||||
lockDeleteUserAccount sync.RWMutex
|
||||
lockGetUserAccount sync.RWMutex
|
||||
lockListUserAccounts sync.RWMutex
|
||||
lockShutdown sync.RWMutex
|
||||
lockUpdateUserAccount sync.RWMutex
|
||||
}
|
||||
|
||||
// CreateAccount calls CreateAccountFunc.
|
||||
@@ -249,39 +235,3 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
|
||||
mock.lockShutdown.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
// UpdateUserAccount calls UpdateUserAccountFunc.
|
||||
func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
|
||||
if mock.UpdateUserAccountFunc == nil {
|
||||
panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
Access string
|
||||
Props auth.MutableProps
|
||||
}{
|
||||
Access: access,
|
||||
Props: props,
|
||||
}
|
||||
mock.lockUpdateUserAccount.Lock()
|
||||
mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
|
||||
mock.lockUpdateUserAccount.Unlock()
|
||||
return mock.UpdateUserAccountFunc(access, props)
|
||||
}
|
||||
|
||||
// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedIAMService.UpdateUserAccountCalls())
|
||||
func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
|
||||
Access string
|
||||
Props auth.MutableProps
|
||||
} {
|
||||
var calls []struct {
|
||||
Access string
|
||||
Props auth.MutableProps
|
||||
}
|
||||
mock.lockUpdateUserAccount.RLock()
|
||||
calls = mock.calls.UpdateUserAccount
|
||||
mock.lockUpdateUserAccount.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
@@ -32,7 +31,7 @@ var (
|
||||
singlePath = regexp.MustCompile(`^/[^/]+/?$`)
|
||||
)
|
||||
|
||||
func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
|
||||
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
|
||||
path := ctx.Path()
|
||||
@@ -49,20 +48,10 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
|
||||
!ctx.Request().URI().QueryArgs().Has("acl") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("tagging") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("versioning") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("policy") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("object-lock") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("ownershipControls") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("cors") {
|
||||
!ctx.Request().URI().QueryArgs().Has("policy") {
|
||||
if err := auth.MayCreateBucket(acct, isRoot); err != nil {
|
||||
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
|
||||
}
|
||||
if readonly {
|
||||
return controllers.SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrAccessDenied),
|
||||
&controllers.MetaOpts{
|
||||
Logger: logger,
|
||||
Action: "CreateBucket",
|
||||
})
|
||||
}
|
||||
return ctx.Next()
|
||||
}
|
||||
data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
|
||||
@@ -75,11 +64,6 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
// if owner is not set, set default owner to root account
|
||||
if parsedAcl.Owner == "" {
|
||||
parsedAcl.Owner = ctx.Locals("rootAccess").(string)
|
||||
}
|
||||
|
||||
ctx.Locals("parsedAcl", parsedAcl)
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != auth.RoleAdmin {
|
||||
path := ctx.Path()
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
|
||||
&controllers.MetaOpts{
|
||||
Logger: logger,
|
||||
Action: detectAction(path),
|
||||
})
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func detectAction(path string) (action string) {
|
||||
if strings.Contains(path, "create-user") {
|
||||
action = metrics.ActionAdminCreateUser
|
||||
} else if strings.Contains(path, "update-user") {
|
||||
action = metrics.ActionAdminUpdateUser
|
||||
} else if strings.Contains(path, "delete-user") {
|
||||
action = metrics.ActionAdminDeleteUser
|
||||
} else if strings.Contains(path, "list-user") {
|
||||
action = metrics.ActionAdminListUsers
|
||||
} else if strings.Contains(path, "list-buckets") {
|
||||
action = metrics.ActionAdminListBuckets
|
||||
} else if strings.Contains(path, "change-bucket-owner") {
|
||||
action = metrics.ActionAdminChangeBucketOwner
|
||||
}
|
||||
return action
|
||||
}
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
@@ -41,7 +40,7 @@ type RootUserConfig struct {
|
||||
Secret string
|
||||
}
|
||||
|
||||
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
|
||||
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
@@ -55,12 +54,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
ctx.Locals("startTime", time.Now())
|
||||
authorization := ctx.Get("Authorization")
|
||||
if authorization == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
|
||||
}
|
||||
|
||||
authData, err := utils.ParseAuthorization(authorization)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
if authData.Algorithm != "AWS4-HMAC-SHA256" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
|
||||
}
|
||||
|
||||
if authData.Region != region {
|
||||
@@ -68,44 +71,42 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}, logger, mm)
|
||||
}, logger)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
ctx.Locals("rootAccess", root.Access)
|
||||
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
// Check X-Amz-Date header
|
||||
date := ctx.Get("X-Amz-Date")
|
||||
if date == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
|
||||
}
|
||||
|
||||
// Parse the date and check the date validity
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
|
||||
}
|
||||
|
||||
if date[:8] != authData.Date {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
|
||||
}
|
||||
|
||||
// Validate the dates difference
|
||||
err = utils.ValidateDate(tdate)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
hashPayload := ctx.Get("X-Amz-Content-Sha256")
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
// for streaming PUT actions, authorization is deferred
|
||||
// until end of stream due to need to get length and
|
||||
@@ -113,24 +114,10 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
|
||||
})
|
||||
|
||||
// wrap the io.Reader with ChunkReader if x-amz-content-sha256
|
||||
// provide chunk encoding value
|
||||
if utils.IsStreamingPayload(hashPayload) {
|
||||
var err error
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
var cr io.Reader
|
||||
cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate)
|
||||
return cr
|
||||
})
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
hashPayload := ctx.Get("X-Amz-Content-Sha256")
|
||||
if !utils.IsSpecialPayload(hashPayload) {
|
||||
// Calculate the hash of the request payload
|
||||
hashedPayload := sha256.Sum256(ctx.Body())
|
||||
@@ -138,7 +125,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
|
||||
// Compare the calculated hash with the hash provided
|
||||
if hashPayload != hexPayload {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,13 +134,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
|
||||
}
|
||||
}
|
||||
|
||||
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
@@ -177,6 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
|
||||
return a.iam.GetUserAccount(access)
|
||||
}
|
||||
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
61
s3api/middlewares/chunk.go
Normal file
61
s3api/middlewares/chunk.go
Normal file
@@ -0,0 +1,61 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
// ProcessChunkedBody initializes the chunked upload stream if the
|
||||
// request appears to be a chunked upload
|
||||
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
|
||||
if decodedLength == "" {
|
||||
return ctx.Next()
|
||||
}
|
||||
// TODO: validate content length
|
||||
|
||||
authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
amzdate := ctx.Get("X-Amz-Date")
|
||||
date, _ := time.Parse(iso8601Format, amzdate)
|
||||
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
var err error
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
var cr *utils.ChunkReader
|
||||
cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
|
||||
return cr
|
||||
})
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
|
||||
}
|
||||
|
||||
sum := md5.Sum(ctx.Body())
|
||||
calculatedSum := utils.Base64SumString(sum[:])
|
||||
calculatedSum := utils.Md5SumString(sum[:])
|
||||
|
||||
if incomingSum != calculatedSum {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
|
||||
|
||||
@@ -20,13 +20,12 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
@@ -39,18 +38,16 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
|
||||
|
||||
authData, err := utils.ParsePresignedURIParts(ctx)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
ctx.Locals("rootAccess", root.Access)
|
||||
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
@@ -64,7 +61,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
|
||||
|
||||
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
|
||||
@@ -18,19 +18,19 @@ import (
|
||||
"net/url"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
|
||||
func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
|
||||
reqURL := ctx.Request().URI().String()
|
||||
decoded, err := url.Parse(reqURL)
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
ctx.Path(unescp)
|
||||
ctx.Path(decoded.Path)
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,9 +18,7 @@ import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
@@ -29,29 +27,26 @@ type S3ApiRouter struct {
|
||||
WithAdmSrv bool
|
||||
}
|
||||
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
|
||||
s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
|
||||
s3ApiController := controllers.New(be, iam, logger, evs, debug)
|
||||
|
||||
if sa.WithAdmSrv {
|
||||
adminController := controllers.NewAdminController(iam, be, aLogger)
|
||||
adminController := controllers.NewAdminController(iam, be)
|
||||
|
||||
// CreateUser admin api
|
||||
app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
|
||||
app.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
// DeleteUsers admin api
|
||||
app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
|
||||
|
||||
// UpdateUser admin api
|
||||
app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
|
||||
app.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
// ListUsers admin api
|
||||
app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
|
||||
app.Patch("/list-users", adminController.ListUsers)
|
||||
|
||||
// ChangeBucketOwner admin api
|
||||
app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
|
||||
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
|
||||
|
||||
// ListBucketsAndOwners admin api
|
||||
app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
|
||||
app.Patch("/list-buckets", adminController.ListBuckets)
|
||||
}
|
||||
|
||||
// ListBuckets action
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false)
|
||||
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,36 +22,23 @@ import (
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
type S3ApiServer struct {
|
||||
app *fiber.App
|
||||
backend backend.Backend
|
||||
router *S3ApiRouter
|
||||
port string
|
||||
cert *tls.Certificate
|
||||
quiet bool
|
||||
debug bool
|
||||
readonly bool
|
||||
health string
|
||||
app *fiber.App
|
||||
backend backend.Backend
|
||||
router *S3ApiRouter
|
||||
port string
|
||||
cert *tls.Certificate
|
||||
quiet bool
|
||||
debug bool
|
||||
health string
|
||||
}
|
||||
|
||||
func New(
|
||||
app *fiber.App,
|
||||
be backend.Backend,
|
||||
root middlewares.RootUserConfig,
|
||||
port, region string,
|
||||
iam auth.IAMService,
|
||||
l s3log.AuditLogger,
|
||||
adminLogger s3log.AuditLogger,
|
||||
evs s3event.S3EventSender,
|
||||
mm *metrics.Manager,
|
||||
opts ...Option,
|
||||
) (*S3ApiServer, error) {
|
||||
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
|
||||
server := &S3ApiServer{
|
||||
app: app,
|
||||
backend: be,
|
||||
@@ -65,9 +52,7 @@ func New(
|
||||
|
||||
// Logging middlewares
|
||||
if !server.quiet {
|
||||
app.Use(logger.New(logger.Config{
|
||||
Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
|
||||
}))
|
||||
app.Use(logger.New())
|
||||
}
|
||||
// Set up health endpoint if specified
|
||||
if server.health != "" {
|
||||
@@ -75,16 +60,17 @@ func New(
|
||||
return ctx.SendStatus(http.StatusOK)
|
||||
})
|
||||
}
|
||||
app.Use(middlewares.DecodeURL(l, mm))
|
||||
app.Use(middlewares.DecodeURL(l))
|
||||
app.Use(middlewares.RequestLogger(server.debug))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
app.Use(middlewares.AclParser(be, l, server.readonly))
|
||||
app.Use(middlewares.AclParser(be, l))
|
||||
|
||||
server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly)
|
||||
server.router.Init(app, be, iam, l, evs, server.debug)
|
||||
|
||||
return server, nil
|
||||
}
|
||||
@@ -117,10 +103,6 @@ func WithHealth(health string) Option {
|
||||
return func(s *S3ApiServer) { s.health = health }
|
||||
}
|
||||
|
||||
func WithReadOnly() Option {
|
||||
return func(s *S3ApiServer) { s.readonly = true }
|
||||
}
|
||||
|
||||
func (sa *S3ApiServer) Serve() (err error) {
|
||||
if sa.cert != nil {
|
||||
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
|
||||
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil, nil)
|
||||
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user