mirror of
https://github.com/versity/versitygw.git
synced 2026-01-28 14:02:03 +00:00
Compare commits
1 Commits
ben/chunk-
...
ben/read_o
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9b41d53b6 |
18
.github/ISSUE_TEMPLATE/bug_report.md
vendored
18
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -1,23 +1,27 @@
|
||||
---
|
||||
name: Bug Report
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: '[Bug] - <Short Description>'
|
||||
title: ''
|
||||
labels: bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
<!-- A clear and concise description of what the bug is. -->
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
<!-- Steps to reproduce the behavior. -->
|
||||
Steps to reproduce the behavior.
|
||||
|
||||
**Expected behavior**
|
||||
<!-- A clear and concise description of what you expected to happen. -->
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Server Version**
|
||||
<!-- output of: './versitygw -version && uname -a' -->
|
||||
output of
|
||||
```
|
||||
./versitygw -version
|
||||
uname -a
|
||||
```
|
||||
|
||||
**Additional context**
|
||||
<!-- Describe s3 client and version if applicable.
|
||||
Describe s3 client and version if applicable.
|
||||
|
||||
8
.github/ISSUE_TEMPLATE/feature_request.md
vendored
8
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -1,14 +1,14 @@
|
||||
---
|
||||
name: Feature Request
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: '[Feature] - <Short Description>'
|
||||
title: ''
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the solution you'd like**
|
||||
<!-- A clear and concise description of what you want to happen. -->
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Additional context**
|
||||
<!-- Add any other context or screenshots about the feature request here. -->
|
||||
Add any other context or screenshots about the feature request here.
|
||||
|
||||
33
.github/ISSUE_TEMPLATE/test_case.md
vendored
33
.github/ISSUE_TEMPLATE/test_case.md
vendored
@@ -1,33 +0,0 @@
|
||||
---
|
||||
name: Test Case Request
|
||||
about: Request new test cases or additional test coverage
|
||||
title: '[Test Case] - <Short Description>'
|
||||
labels: 'testcase'
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
<!-- Please provide a detailed description of the test case or test coverage request. -->
|
||||
|
||||
## Purpose
|
||||
<!-- Explain why this test case is important and what it aims to achieve. -->
|
||||
|
||||
## Scope
|
||||
<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
|
||||
|
||||
## Acceptance Criteria
|
||||
<!-- List the criteria that must be met for the test case to be considered complete. -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
## Additional Context
|
||||
<!-- Add any other context or screenshots about the feature request here. -->
|
||||
|
||||
## Resources
|
||||
<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
|
||||
|
||||
|
||||
**Thank you for contributing to our project!**
|
||||
37
.github/workflows/azurite.yml
vendored
37
.github/workflows/azurite.yml
vendored
@@ -1,37 +0,0 @@
|
||||
name: azurite functional tests
|
||||
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Compose
|
||||
run: |
|
||||
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
|
||||
|
||||
- name: Wait for Azurite to be ready
|
||||
run: sleep 40
|
||||
|
||||
- name: Get Dependencies
|
||||
run: |
|
||||
go mod download
|
||||
|
||||
- name: Build and Run
|
||||
run: |
|
||||
make
|
||||
./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
|
||||
|
||||
- name: Shut down services
|
||||
run: |
|
||||
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw
|
||||
29
.github/workflows/docker-bats.yml
vendored
29
.github/workflows/docker-bats.yml
vendored
@@ -1,29 +0,0 @@
|
||||
name: docker bats tests
|
||||
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
cp tests/.env.docker.default tests/.env.docker
|
||||
cp tests/.secrets.default tests/.secrets
|
||||
# see https://github.com/versity/versitygw/issues/1034
|
||||
docker build \
|
||||
--build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
|
||||
--build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
|
||||
--build-arg="MC_FOLDER=linux-amd64" \
|
||||
--progress=plain \
|
||||
-f tests/Dockerfile_test_bats \
|
||||
-t bats_test .
|
||||
|
||||
- name: Run Docker Container
|
||||
run: |
|
||||
docker compose -f tests/docker-compose-bats.yml --project-directory . \
|
||||
up --exit-code-from s3api_np_only s3api_np_only
|
||||
@@ -15,13 +15,6 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -50,7 +43,6 @@ jobs:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build-args: |
|
||||
VERSION=${{ github.event.release.tag_name }}
|
||||
TIME=${{ github.event.release.published_at }}
|
||||
5
.github/workflows/functional.yml
vendored
5
.github/workflows/functional.yml
vendored
@@ -1,8 +1,7 @@
|
||||
name: functional tests
|
||||
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
@@ -19,7 +18,7 @@ jobs:
|
||||
|
||||
- name: Get Dependencies
|
||||
run: |
|
||||
go mod download
|
||||
go get -v -t -d ./...
|
||||
|
||||
- name: Build and Run
|
||||
run: |
|
||||
|
||||
16
.github/workflows/shellcheck.yml
vendored
16
.github/workflows/shellcheck.yml
vendored
@@ -1,16 +0,0 @@
|
||||
name: shellcheck
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: Run shellcheck
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run checks
|
||||
run: |
|
||||
shellcheck --version
|
||||
shellcheck -e SC1091 tests/*.sh tests/*/*.sh
|
||||
178
.github/workflows/system.yml
vendored
178
.github/workflows/system.yml
vendored
@@ -4,105 +4,16 @@ jobs:
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- set: "mc, posix, non-file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "mc-non-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "mc, posix, file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "mc-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "REST, posix, non-static, all, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "rest"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3, posix, non-file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3-non-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3, posix, file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, policy, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-policy"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, user, non-static, s3 IAM"
|
||||
IAM_TYPE: s3
|
||||
RUN_SET: "s3api-user"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, bucket, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-bucket"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, multipart, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-multipart"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, object, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-object"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, policy, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-policy"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
- set: "s3api, posix, user, static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3api-user"
|
||||
RECREATE_BUCKETS: "false"
|
||||
BACKEND: "posix"
|
||||
# TODO fix/debug s3 gateway
|
||||
#- set: "s3api, s3, multipart|object, non-static, folder IAM"
|
||||
# IAM_TYPE: folder
|
||||
# RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
|
||||
# RECREATE_BUCKETS: "true"
|
||||
# BACKEND: "s3"
|
||||
#- set: "s3api, s3, policy|user, non-static, folder IAM"
|
||||
# IAM_TYPE: folder
|
||||
# RUN_SET: "s3api-policy,s3api-user"
|
||||
# RECREATE_BUCKETS: "true"
|
||||
# BACKEND: "s3"
|
||||
- set: "s3cmd, posix, file count, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3cmd-file-count"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3cmd, posix, non-user, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3cmd-non-user"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
- set: "s3cmd, posix, user, non-static, folder IAM"
|
||||
IAM_TYPE: folder
|
||||
RUN_SET: "s3cmd-user"
|
||||
RECREATE_BUCKETS: "true"
|
||||
BACKEND: "posix"
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install ShellCheck
|
||||
run: sudo apt-get install shellcheck
|
||||
|
||||
- name: Run ShellCheck
|
||||
run: shellcheck -S warning ./tests/*.sh
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
@@ -117,8 +28,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/bats-core/bats-core.git
|
||||
cd bats-core && ./install.sh $HOME
|
||||
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
|
||||
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
|
||||
|
||||
- name: Install s3cmd
|
||||
run: |
|
||||
@@ -129,72 +38,35 @@ jobs:
|
||||
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
|
||||
chmod 755 /usr/local/bin/mc
|
||||
|
||||
- name: Install xmllint (for rest)
|
||||
run: |
|
||||
sudo apt-get install libxml2-utils
|
||||
|
||||
# see https://github.com/versity/versitygw/issues/1034
|
||||
- name: Install AWS cli
|
||||
run: |
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip"
|
||||
unzip -o awscliv2.zip
|
||||
./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
|
||||
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Build and run
|
||||
env:
|
||||
IAM_TYPE: ${{ matrix.IAM_TYPE }}
|
||||
RUN_SET: ${{ matrix.RUN_SET }}
|
||||
AWS_PROFILE: versity
|
||||
VERSITY_EXE: ${{ github.workspace }}/versitygw
|
||||
RUN_VERSITYGW: true
|
||||
BACKEND: ${{ matrix.BACKEND }}
|
||||
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
|
||||
CERT: ${{ github.workspace }}/cert.pem
|
||||
KEY: ${{ github.workspace }}/versitygw.pem
|
||||
LOCAL_FOLDER: /tmp/gw
|
||||
BUCKET_ONE_NAME: versity-gwtest-bucket-one
|
||||
BUCKET_TWO_NAME: versity-gwtest-bucket-two
|
||||
USERS_FOLDER: /tmp/iam
|
||||
USERS_BUCKET: versity-gwtest-iam
|
||||
AWS_ENDPOINT_URL: https://127.0.0.1:7070
|
||||
PORT: 7070
|
||||
S3CMD_CONFIG: tests/s3cfg.local.default
|
||||
MC_ALIAS: versity
|
||||
LOG_LEVEL: 4
|
||||
GOCOVERDIR: ${{ github.workspace }}/cover
|
||||
USERNAME_ONE: ABCDEFG
|
||||
PASSWORD_ONE: 1234567
|
||||
USERNAME_TWO: HIJKLMN
|
||||
PASSWORD_TWO: 8901234
|
||||
TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
|
||||
REMOVE_TEST_FILE_FOLDER: true
|
||||
VERSIONING_DIR: ${{ github.workspace }}/versioning
|
||||
COMMAND_LOG: command.log
|
||||
TIME_LOG: time.log
|
||||
- name: Build and run, posix backend
|
||||
run: |
|
||||
make testbin
|
||||
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
|
||||
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
export AWS_REGION=us-east-1
|
||||
export AWS_ACCESS_KEY_ID_TWO=user
|
||||
export AWS_SECRET_ACCESS_KEY_TWO=pass
|
||||
export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
|
||||
aws configure set aws_region $AWS_REGION --profile versity
|
||||
mkdir $LOCAL_FOLDER
|
||||
mkdir /tmp/gw
|
||||
export WORKSPACE=$GITHUB_WORKSPACE
|
||||
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
|
||||
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
mkdir $GOCOVERDIR $USERS_FOLDER
|
||||
if [[ $RECREATE_BUCKETS == "false" ]]; then
|
||||
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
|
||||
fi
|
||||
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
|
||||
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
|
||||
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
mkdir cover
|
||||
VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
|
||||
|
||||
- name: Time report
|
||||
run: cat ${{ github.workspace }}/time.log
|
||||
#- name: Build and run, s3 backend
|
||||
# run: |
|
||||
# make testbin
|
||||
# export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
|
||||
# export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
# export AWS_REGION=us-east-1
|
||||
# aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
|
||||
# aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
|
||||
# aws configure set aws_region $AWS_REGION --profile versity_s3
|
||||
# export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
|
||||
# export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
# export WORKSPACE=$GITHUB_WORKSPACE
|
||||
# VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
|
||||
|
||||
- name: Coverage report
|
||||
run: |
|
||||
|
||||
22
.gitignore
vendored
22
.gitignore
vendored
@@ -45,25 +45,7 @@ tests/.secrets*
|
||||
|
||||
# IAM users files often created in testing
|
||||
users.json
|
||||
users.json.backup
|
||||
|
||||
# env files for testing
|
||||
**/.env*
|
||||
**/!.env.default
|
||||
|
||||
# s3cmd config files (testing)
|
||||
tests/s3cfg.local*
|
||||
tests/!s3cfg.local.default
|
||||
|
||||
# keys
|
||||
*.pem
|
||||
|
||||
# patches
|
||||
*.patch
|
||||
|
||||
# grafana's local database (kept on filesystem for survival between instantiations)
|
||||
metrics-exploration/grafana_data/**
|
||||
|
||||
# bats tools
|
||||
/tests/bats-assert
|
||||
/tests/bats-support
|
||||
.env*
|
||||
!.env.default
|
||||
|
||||
@@ -32,4 +32,4 @@ RUN mkdir -p $SETUP_DIR
|
||||
|
||||
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
|
||||
|
||||
ENTRYPOINT [ "/app/versitygw" ]
|
||||
ENTRYPOINT [ "/app/versitygw" ]
|
||||
@@ -6,7 +6,7 @@ COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY ./ ./
|
||||
COPY ./tests/certs/* /etc/pki/tls/certs/
|
||||
COPY certs/* /etc/pki/tls/certs/
|
||||
|
||||
ARG IAM_DIR=/tmp/vgw
|
||||
ARG SETUP_DIR=/tmp/vgw
|
||||
@@ -1,12 +1,8 @@
|
||||
FROM ubuntu:latest
|
||||
FROM --platform=linux/arm64 ubuntu:latest
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG SECRETS_FILE=tests/.secrets
|
||||
ARG CONFIG_FILE=tests/.env.docker
|
||||
ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz
|
||||
# see https://github.com/versity/versitygw/issues/1034
|
||||
ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip
|
||||
ARG MC_FOLDER=linux-arm64
|
||||
|
||||
ENV TZ=Etc/UTC
|
||||
RUN apt-get update && \
|
||||
@@ -20,7 +16,6 @@ RUN apt-get update && \
|
||||
s3cmd \
|
||||
jq \
|
||||
bc \
|
||||
libxml2-utils \
|
||||
ca-certificates && \
|
||||
update-ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -29,20 +24,20 @@ RUN apt-get update && \
|
||||
WORKDIR /tmp
|
||||
|
||||
# Install AWS cli
|
||||
RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
|
||||
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
|
||||
|
||||
# Install mc
|
||||
RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
|
||||
RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
|
||||
--create-dirs \
|
||||
-o /usr/local/minio-binaries/mc && \
|
||||
chmod -R 755 /usr/local/minio-binaries
|
||||
ENV PATH=/usr/local/minio-binaries:${PATH}
|
||||
ENV PATH="/usr/local/minio-binaries":${PATH}
|
||||
|
||||
# Download Go 1.21 (adjust the version and platform as needed)
|
||||
RUN wget https://golang.org/dl/${GO_LIBRARY}
|
||||
RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
|
||||
|
||||
# Extract the downloaded archive
|
||||
RUN tar -xvf $GO_LIBRARY -C /usr/local
|
||||
RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
|
||||
|
||||
# Set Go environment variables
|
||||
ENV PATH="/usr/local/go/bin:${PATH}"
|
||||
@@ -65,11 +60,9 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
|
||||
USER tester
|
||||
COPY --chown=tester:tester . /home/tester
|
||||
|
||||
# add bats support libraries
|
||||
RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
|
||||
RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
|
||||
|
||||
WORKDIR /home/tester
|
||||
#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
|
||||
RUN cp tests/s3cfg.local.default tests/s3cfg.local
|
||||
RUN make
|
||||
|
||||
RUN . $SECRETS_FILE && \
|
||||
@@ -86,7 +79,5 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
|
||||
|
||||
ENV WORKSPACE=.
|
||||
ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
|
||||
#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
|
||||
|
||||
ENTRYPOINT ["tests/run.sh"]
|
||||
CMD ["s3api,s3,s3cmd,mc,rest"]
|
||||
CMD ["tests/run_all.sh"]
|
||||
12
Makefile
12
Makefile
@@ -18,10 +18,6 @@ GOBUILD=$(GOCMD) build
|
||||
GOCLEAN=$(GOCMD) clean
|
||||
GOTEST=$(GOCMD) test
|
||||
|
||||
# docker-compose
|
||||
DCCMD=docker-compose
|
||||
DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
|
||||
|
||||
BIN=versitygw
|
||||
|
||||
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
|
||||
@@ -75,19 +71,19 @@ dist:
|
||||
# Creates and runs S3 gateway instance in a docker container
|
||||
.PHONY: up-posix
|
||||
up-posix:
|
||||
$(DOCKERCOMPOSE) up posix
|
||||
docker compose --env-file .env.dev up posix
|
||||
|
||||
# Creates and runs S3 gateway proxy instance in a docker container
|
||||
.PHONY: up-proxy
|
||||
up-proxy:
|
||||
$(DOCKERCOMPOSE) up proxy
|
||||
docker compose --env-file .env.dev up proxy
|
||||
|
||||
# Creates and runs S3 gateway to azurite instance in a docker container
|
||||
.PHONY: up-azurite
|
||||
up-azurite:
|
||||
$(DOCKERCOMPOSE) up azurite azuritegw
|
||||
docker compose --env-file .env.dev up azurite azuritegw
|
||||
|
||||
# Creates and runs both S3 gateway and proxy server instances in docker containers
|
||||
.PHONY: up-app
|
||||
up-app:
|
||||
$(DOCKERCOMPOSE) up
|
||||
docker compose --env-file .env.dev up
|
||||
|
||||
19
README.md
19
README.md
@@ -6,7 +6,7 @@
|
||||
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
|
||||
</picture>
|
||||
|
||||
[](https://github.com/versity/versitygw/blob/main/LICENSE) [](https://goreportcard.com/report/github.com/versity/versitygw) [](https://pkg.go.dev/github.com/versity/versitygw)
|
||||
[](https://github.com/versity/versitygw/blob/main/LICENSE)
|
||||
|
||||
### Binary release builds
|
||||
Download [latest release](https://github.com/versity/versitygw/releases)
|
||||
@@ -14,15 +14,8 @@ Download [latest release](https://github.com/versity/versitygw/releases)
|
||||
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
|
||||
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
|
||||
|
||||
### Use Cases
|
||||
* Turn your local filesystem into an S3 server with a single command!
|
||||
* Proxy S3 requests to S3 storage
|
||||
* Simple to deploy S3 server with a single command
|
||||
* Protocol compatibility in `posix` allows common access to files via posix or S3
|
||||
* Simplified interface for adding new storage system support
|
||||
|
||||
### News
|
||||
Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
|
||||
* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
|
||||
|
||||
### Mailing List
|
||||
Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
|
||||
@@ -35,6 +28,12 @@ Ask questions in the [community discussions](https://github.com/versity/versityg
|
||||
<br>
|
||||
Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
|
||||
|
||||
### Use Cases
|
||||
* Share filesystem directory via S3 protocol
|
||||
* Proxy S3 requests to S3 storage
|
||||
* Simple to deploy S3 server with a single command
|
||||
* Protocol compatibility in `posix` allows common access to files via posix or S3
|
||||
|
||||
### Overview
|
||||
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
|
||||
|
||||
@@ -68,7 +67,7 @@ The command format is
|
||||
```
|
||||
versitygw [global options] command [command options] [arguments...]
|
||||
```
|
||||
The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
|
||||
The global options are specified before the backend type and the backend options are specified after.
|
||||
|
||||
***
|
||||
|
||||
|
||||
328
auth/acl.go
328
auth/acl.go
@@ -17,8 +17,6 @@ package auth
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
@@ -29,156 +27,27 @@ import (
|
||||
)
|
||||
|
||||
type ACL struct {
|
||||
ACL types.BucketCannedACL
|
||||
Owner string
|
||||
Grantees []Grantee
|
||||
}
|
||||
|
||||
type Grantee struct {
|
||||
Permission Permission
|
||||
Permission types.Permission
|
||||
Access string
|
||||
Type types.Type
|
||||
}
|
||||
|
||||
type GetBucketAclOutput struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
|
||||
Owner *types.Owner
|
||||
AccessControlList AccessControlList
|
||||
}
|
||||
|
||||
type PutBucketAclInput struct {
|
||||
Bucket *string
|
||||
ACL types.BucketCannedACL
|
||||
AccessControlPolicy *AccessControlPolicy
|
||||
GrantFullControl *string
|
||||
GrantRead *string
|
||||
GrantReadACP *string
|
||||
GrantWrite *string
|
||||
GrantWriteACP *string
|
||||
type AccessControlList struct {
|
||||
Grants []types.Grant `xml:"Grant"`
|
||||
}
|
||||
|
||||
type AccessControlPolicy struct {
|
||||
AccessControlList AccessControlList `xml:"AccessControlList"`
|
||||
Owner *types.Owner
|
||||
}
|
||||
|
||||
func (acp *AccessControlPolicy) Validate() error {
|
||||
if !acp.AccessControlList.isValid() {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedACL)
|
||||
}
|
||||
|
||||
// The Owner can't be nil
|
||||
if acp.Owner == nil {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedACL)
|
||||
}
|
||||
|
||||
// The Owner ID can't be empty
|
||||
if acp.Owner.ID == nil || *acp.Owner.ID == "" {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedACL)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type AccessControlList struct {
|
||||
Grants []Grant `xml:"Grant"`
|
||||
}
|
||||
|
||||
// Validates the AccessControlList
|
||||
func (acl *AccessControlList) isValid() bool {
|
||||
for _, el := range acl.Grants {
|
||||
if !el.isValid() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
type Permission string
|
||||
|
||||
const (
|
||||
PermissionFullControl Permission = "FULL_CONTROL"
|
||||
PermissionWrite Permission = "WRITE"
|
||||
PermissionWriteAcp Permission = "WRITE_ACP"
|
||||
PermissionRead Permission = "READ"
|
||||
PermissionReadAcp Permission = "READ_ACP"
|
||||
)
|
||||
|
||||
// Check if the permission is valid
|
||||
func (p Permission) isValid() bool {
|
||||
return p == PermissionFullControl ||
|
||||
p == PermissionRead ||
|
||||
p == PermissionReadAcp ||
|
||||
p == PermissionWrite ||
|
||||
p == PermissionWriteAcp
|
||||
}
|
||||
|
||||
type Grant struct {
|
||||
Grantee *Grt `xml:"Grantee"`
|
||||
Permission Permission `xml:"Permission"`
|
||||
}
|
||||
|
||||
// Checks if Grant is valid
|
||||
func (g *Grant) isValid() bool {
|
||||
return g.Permission.isValid() && g.Grantee.isValid()
|
||||
}
|
||||
|
||||
type Grt struct {
|
||||
XMLNS string `xml:"xmlns:xsi,attr"`
|
||||
Type types.Type `xml:"xsi:type,attr"`
|
||||
ID string `xml:"ID"`
|
||||
}
|
||||
|
||||
// Custom Unmarshalling for Grt to parse xsi:type properly
|
||||
func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
// Iterate through the XML tokens to process the attributes
|
||||
for _, attr := range start.Attr {
|
||||
// Check if the attribute is xsi:type and belongs to the xsi namespace
|
||||
if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
|
||||
g.Type = types.Type(attr.Value)
|
||||
}
|
||||
// Handle xmlns:xsi
|
||||
if attr.Name.Local == "xmlns:xsi" {
|
||||
g.XMLNS = attr.Value
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the inner XML elements like ID
|
||||
for {
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch se := t.(type) {
|
||||
case xml.StartElement:
|
||||
if se.Name.Local == "ID" {
|
||||
if err := d.DecodeElement(&g.ID, &se); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case xml.EndElement:
|
||||
if se.Name.Local == start.Name.Local {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validates Grt
|
||||
func (g *Grt) isValid() bool {
|
||||
// Validate the Type
|
||||
// Only these 2 types are supported in the gateway
|
||||
if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
|
||||
return false
|
||||
}
|
||||
|
||||
// The ID prop shouldn't be empty
|
||||
if g.ID == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
Owner types.Owner
|
||||
}
|
||||
|
||||
func ParseACL(data []byte) (ACL, error) {
|
||||
@@ -199,18 +68,11 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
|
||||
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
|
||||
}
|
||||
|
||||
grants := []Grant{}
|
||||
grants := []types.Grant{}
|
||||
|
||||
for _, elem := range acl.Grantees {
|
||||
acs := elem.Access
|
||||
grants = append(grants, Grant{
|
||||
Grantee: &Grt{
|
||||
XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
|
||||
ID: acs,
|
||||
Type: elem.Type,
|
||||
},
|
||||
Permission: elem.Permission,
|
||||
})
|
||||
grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
|
||||
}
|
||||
|
||||
return GetBucketAclOutput{
|
||||
@@ -223,43 +85,20 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
|
||||
func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
|
||||
if input == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
defaultGrantees := []Grantee{
|
||||
{
|
||||
Permission: PermissionFullControl,
|
||||
Access: acl.Owner,
|
||||
Type: types.TypeCanonicalUser,
|
||||
},
|
||||
if acl.Owner != *input.AccessControlPolicy.Owner.ID {
|
||||
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
// if the ACL is specified, set the ACL, else replace the grantees
|
||||
if input.ACL != "" {
|
||||
switch input.ACL {
|
||||
case types.BucketCannedACLPublicRead:
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Permission: PermissionRead,
|
||||
Access: "all-users",
|
||||
Type: types.TypeGroup,
|
||||
})
|
||||
case types.BucketCannedACLPublicReadWrite:
|
||||
defaultGrantees = append(defaultGrantees, []Grantee{
|
||||
{
|
||||
Permission: PermissionRead,
|
||||
Access: "all-users",
|
||||
Type: types.TypeGroup,
|
||||
},
|
||||
{
|
||||
Permission: PermissionWrite,
|
||||
Access: "all-users",
|
||||
Type: types.TypeGroup,
|
||||
},
|
||||
}...)
|
||||
}
|
||||
acl.ACL = input.ACL
|
||||
acl.Grantees = []Grantee{}
|
||||
} else {
|
||||
grantees := []Grantee{}
|
||||
accs := []string{}
|
||||
|
||||
if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
|
||||
@@ -268,71 +107,45 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
|
||||
if input.GrantFullControl != nil && *input.GrantFullControl != "" {
|
||||
fullControlList = splitUnique(*input.GrantFullControl, ",")
|
||||
for _, str := range fullControlList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionFullControl,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
|
||||
}
|
||||
}
|
||||
if input.GrantRead != nil && *input.GrantRead != "" {
|
||||
readList = splitUnique(*input.GrantRead, ",")
|
||||
for _, str := range readList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionRead,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
|
||||
}
|
||||
}
|
||||
if input.GrantReadACP != nil && *input.GrantReadACP != "" {
|
||||
readACPList = splitUnique(*input.GrantReadACP, ",")
|
||||
for _, str := range readACPList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionReadAcp,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
|
||||
}
|
||||
}
|
||||
if input.GrantWrite != nil && *input.GrantWrite != "" {
|
||||
writeList = splitUnique(*input.GrantWrite, ",")
|
||||
for _, str := range writeList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionWrite,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
|
||||
}
|
||||
}
|
||||
if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
|
||||
writeACPList = splitUnique(*input.GrantWriteACP, ",")
|
||||
for _, str := range writeACPList {
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: str,
|
||||
Permission: PermissionWriteAcp,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
|
||||
}
|
||||
}
|
||||
|
||||
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
|
||||
} else {
|
||||
cache := make(map[string]bool)
|
||||
for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
|
||||
if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
|
||||
for _, grt := range input.AccessControlPolicy.Grants {
|
||||
if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
access := grt.Grantee.ID
|
||||
defaultGrantees = append(defaultGrantees, Grantee{
|
||||
Access: access,
|
||||
Permission: grt.Permission,
|
||||
Type: types.TypeCanonicalUser,
|
||||
})
|
||||
if _, ok := cache[access]; !ok {
|
||||
cache[access] = true
|
||||
accs = append(accs, access)
|
||||
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
|
||||
if _, ok := cache[*grt.Grantee.ID]; !ok {
|
||||
cache[*grt.Grantee.ID] = true
|
||||
accs = append(accs, *grt.Grantee.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -345,9 +158,10 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
|
||||
if len(accList) > 0 {
|
||||
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
acl.Grantees = defaultGrantees
|
||||
acl.Grantees = grantees
|
||||
acl.ACL = ""
|
||||
}
|
||||
|
||||
result, err := json.Marshal(acl)
|
||||
if err != nil {
|
||||
@@ -367,8 +181,8 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
|
||||
result = append(result, acc)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
|
||||
return nil, err
|
||||
if err == ErrNotSupported {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
return nil, fmt.Errorf("check user account: %w", err)
|
||||
}
|
||||
@@ -391,34 +205,35 @@ func splitUnique(s, divider string) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
func verifyACL(acl ACL, access string, permission Permission) error {
|
||||
grantee := Grantee{
|
||||
Access: access,
|
||||
Permission: permission,
|
||||
Type: types.TypeCanonicalUser,
|
||||
}
|
||||
granteeFullCtrl := Grantee{
|
||||
Access: access,
|
||||
Permission: PermissionFullControl,
|
||||
Type: types.TypeCanonicalUser,
|
||||
}
|
||||
granteeAllUsers := Grantee{
|
||||
Access: "all-users",
|
||||
Permission: permission,
|
||||
Type: types.TypeGroup,
|
||||
}
|
||||
|
||||
isFound := false
|
||||
|
||||
for _, grt := range acl.Grantees {
|
||||
if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
|
||||
isFound = true
|
||||
break
|
||||
func verifyACL(acl ACL, access string, permission types.Permission) error {
|
||||
if acl.ACL != "" {
|
||||
if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
}
|
||||
|
||||
if isFound {
|
||||
return nil
|
||||
} else {
|
||||
if len(acl.Grantees) == 0 {
|
||||
return nil
|
||||
}
|
||||
grantee := Grantee{Access: access, Permission: permission}
|
||||
granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
|
||||
|
||||
isFound := false
|
||||
|
||||
for _, grt := range acl.Grantees {
|
||||
if grt == grantee || grt == granteeFullCtrl {
|
||||
isFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if isFound {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
@@ -458,37 +273,38 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
|
||||
|
||||
type AccessOptions struct {
|
||||
Acl ACL
|
||||
AclPermission Permission
|
||||
AclPermission types.Permission
|
||||
IsRoot bool
|
||||
Acc Account
|
||||
Bucket string
|
||||
Object string
|
||||
Action Action
|
||||
Readonly bool
|
||||
}
|
||||
|
||||
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
|
||||
if opts.Readonly {
|
||||
if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
}
|
||||
if opts.IsRoot {
|
||||
return nil
|
||||
}
|
||||
if opts.Acc.Role == RoleAdmin {
|
||||
return nil
|
||||
}
|
||||
|
||||
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
|
||||
if policyErr != nil {
|
||||
if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return policyErr
|
||||
}
|
||||
} else {
|
||||
return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
|
||||
if opts.Acc.Access == opts.Acl.Owner {
|
||||
return nil
|
||||
}
|
||||
|
||||
policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If bucket policy is not set and the ACL is default, only the owner has access
|
||||
if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -527,7 +343,7 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource
|
||||
|
||||
if err := VerifyAccess(ctx, be, AccessOptions{
|
||||
Acl: srcBucketAcl,
|
||||
AclPermission: PermissionRead,
|
||||
AclPermission: types.PermissionRead,
|
||||
IsRoot: opts.IsRoot,
|
||||
Acc: opts.Acc,
|
||||
Bucket: srcBucket,
|
||||
|
||||
@@ -16,22 +16,12 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
var (
|
||||
errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
|
||||
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
|
||||
errInvalidResource = errors.New("Policy has invalid resource")
|
||||
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
|
||||
errInvalidPrincipal = errors.New("Invalid principal in policy")
|
||||
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
|
||||
errInvalidAction = errors.New("Policy has invalid action")
|
||||
)
|
||||
|
||||
type BucketPolicy struct {
|
||||
Statement []BucketPolicyItem `json:"Statement"`
|
||||
}
|
||||
@@ -85,14 +75,11 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
|
||||
|
||||
for action := range bpi.Actions {
|
||||
isObjectAction := action.IsObjectAction()
|
||||
if isObjectAction == nil {
|
||||
break
|
||||
if isObjectAction && !containsObjectAction {
|
||||
return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
|
||||
}
|
||||
if *isObjectAction && !containsObjectAction {
|
||||
return errResourceMismatch
|
||||
}
|
||||
if !*isObjectAction && !containsBucketAction {
|
||||
return errResourceMismatch
|
||||
if !isObjectAction && !containsBucketAction {
|
||||
return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,11 +108,6 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
|
||||
return getMalformedPolicyError(err)
|
||||
}
|
||||
|
||||
if len(policy.Statement) == 0 {
|
||||
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
|
||||
return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
|
||||
}
|
||||
|
||||
if err := policy.Validate(bucket, iam); err != nil {
|
||||
return getMalformedPolicyError(err)
|
||||
}
|
||||
@@ -133,7 +115,12 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
|
||||
func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
|
||||
// If bucket policy is not set
|
||||
if len(policy) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var bucketPolicy BucketPolicy
|
||||
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
|
||||
return err
|
||||
@@ -144,6 +131,7 @@ func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Act
|
||||
resource += "/" + object
|
||||
}
|
||||
|
||||
fmt.Println(access, action, resource)
|
||||
if !bucketPolicy.isAllowed(access, action, resource) {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
@@ -16,116 +16,92 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
GetBucketAclAction Action = "s3:GetBucketAcl"
|
||||
CreateBucketAction Action = "s3:CreateBucket"
|
||||
PutBucketAclAction Action = "s3:PutBucketAcl"
|
||||
DeleteBucketAction Action = "s3:DeleteBucket"
|
||||
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
|
||||
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
|
||||
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
|
||||
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
|
||||
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
|
||||
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
|
||||
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
|
||||
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
|
||||
PutObjectAction Action = "s3:PutObject"
|
||||
GetObjectAction Action = "s3:GetObject"
|
||||
GetObjectVersionAction Action = "s3:GetObjectVersion"
|
||||
DeleteObjectAction Action = "s3:DeleteObject"
|
||||
GetObjectAclAction Action = "s3:GetObjectAcl"
|
||||
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
|
||||
PutObjectAclAction Action = "s3:PutObjectAcl"
|
||||
RestoreObjectAction Action = "s3:RestoreObject"
|
||||
GetBucketTaggingAction Action = "s3:GetBucketTagging"
|
||||
PutBucketTaggingAction Action = "s3:PutBucketTagging"
|
||||
GetObjectTaggingAction Action = "s3:GetObjectTagging"
|
||||
PutObjectTaggingAction Action = "s3:PutObjectTagging"
|
||||
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
|
||||
ListBucketVersionsAction Action = "s3:ListBucketVersions"
|
||||
ListBucketAction Action = "s3:ListBucket"
|
||||
GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
|
||||
PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
|
||||
GetObjectLegalHoldAction Action = "s3:GetObjectLegalHold"
|
||||
PutObjectLegalHoldAction Action = "s3:PutObjectLegalHold"
|
||||
GetObjectRetentionAction Action = "s3:GetObjectRetention"
|
||||
PutObjectRetentionAction Action = "s3:PutObjectRetention"
|
||||
BypassGovernanceRetentionAction Action = "s3:BypassGovernanceRetention"
|
||||
PutBucketOwnershipControlsAction Action = "s3:PutBucketOwnershipControls"
|
||||
GetBucketOwnershipControlsAction Action = "s3:GetBucketOwnershipControls"
|
||||
AllActions Action = "s3:*"
|
||||
GetBucketAclAction Action = "s3:GetBucketAcl"
|
||||
CreateBucketAction Action = "s3:CreateBucket"
|
||||
PutBucketAclAction Action = "s3:PutBucketAcl"
|
||||
DeleteBucketAction Action = "s3:DeleteBucket"
|
||||
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
|
||||
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
|
||||
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
|
||||
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
|
||||
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
|
||||
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
|
||||
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
|
||||
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
|
||||
PutObjectAction Action = "s3:PutObject"
|
||||
GetObjectAction Action = "s3:GetObject"
|
||||
DeleteObjectAction Action = "s3:DeleteObject"
|
||||
GetObjectAclAction Action = "s3:GetObjectAcl"
|
||||
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
|
||||
PutObjectAclAction Action = "s3:PutObjectAcl"
|
||||
RestoreObjectAction Action = "s3:RestoreObject"
|
||||
GetBucketTaggingAction Action = "s3:GetBucketTagging"
|
||||
PutBucketTaggingAction Action = "s3:PutBucketTagging"
|
||||
GetObjectTaggingAction Action = "s3:GetObjectTagging"
|
||||
PutObjectTaggingAction Action = "s3:PutObjectTagging"
|
||||
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
|
||||
ListBucketVersionsAction Action = "s3:ListBucketVersions"
|
||||
ListBucketAction Action = "s3:ListBucket"
|
||||
AllActions Action = "s3:*"
|
||||
)
|
||||
|
||||
var supportedActionList = map[Action]struct{}{
|
||||
GetBucketAclAction: {},
|
||||
CreateBucketAction: {},
|
||||
PutBucketAclAction: {},
|
||||
DeleteBucketAction: {},
|
||||
PutBucketVersioningAction: {},
|
||||
GetBucketVersioningAction: {},
|
||||
PutBucketPolicyAction: {},
|
||||
GetBucketPolicyAction: {},
|
||||
DeleteBucketPolicyAction: {},
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
ListBucketMultipartUploadsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
GetObjectVersionAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetBucketTaggingAction: {},
|
||||
PutBucketTaggingAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
ListBucketVersionsAction: {},
|
||||
ListBucketAction: {},
|
||||
PutBucketObjectLockConfigurationAction: {},
|
||||
GetObjectLegalHoldAction: {},
|
||||
PutObjectLegalHoldAction: {},
|
||||
GetObjectRetentionAction: {},
|
||||
PutObjectRetentionAction: {},
|
||||
BypassGovernanceRetentionAction: {},
|
||||
PutBucketOwnershipControlsAction: {},
|
||||
GetBucketOwnershipControlsAction: {},
|
||||
AllActions: {},
|
||||
GetBucketAclAction: {},
|
||||
CreateBucketAction: {},
|
||||
PutBucketAclAction: {},
|
||||
DeleteBucketAction: {},
|
||||
PutBucketVersioningAction: {},
|
||||
GetBucketVersioningAction: {},
|
||||
PutBucketPolicyAction: {},
|
||||
GetBucketPolicyAction: {},
|
||||
DeleteBucketPolicyAction: {},
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
ListBucketMultipartUploadsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetBucketTaggingAction: {},
|
||||
PutBucketTaggingAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
ListBucketVersionsAction: {},
|
||||
ListBucketAction: {},
|
||||
AllActions: {},
|
||||
}
|
||||
|
||||
var supportedObjectActionList = map[Action]struct{}{
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
GetObjectVersionAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
GetObjectLegalHoldAction: {},
|
||||
PutObjectLegalHoldAction: {},
|
||||
GetObjectRetentionAction: {},
|
||||
PutObjectRetentionAction: {},
|
||||
BypassGovernanceRetentionAction: {},
|
||||
AllActions: {},
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
AllActions: {},
|
||||
}
|
||||
|
||||
// Validates Action: it should either wildcard match with supported actions list or be in it
|
||||
func (a Action) IsValid() error {
|
||||
if !strings.HasPrefix(string(a), "s3:") {
|
||||
return errInvalidAction
|
||||
return fmt.Errorf("invalid action: %v", a)
|
||||
}
|
||||
|
||||
if a == AllActions {
|
||||
@@ -140,39 +116,31 @@ func (a Action) IsValid() error {
|
||||
}
|
||||
}
|
||||
|
||||
return errInvalidAction
|
||||
return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
|
||||
}
|
||||
|
||||
_, found := supportedActionList[a]
|
||||
if !found {
|
||||
return errInvalidAction
|
||||
return fmt.Errorf("unsupported action: %v", a)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getBoolPtr(bl bool) *bool {
|
||||
return &bl
|
||||
}
|
||||
|
||||
// Checks if the action is object action
|
||||
// nil points to 's3:*'
|
||||
func (a Action) IsObjectAction() *bool {
|
||||
if a == AllActions {
|
||||
return nil
|
||||
}
|
||||
func (a Action) IsObjectAction() bool {
|
||||
if a[len(a)-1] == '*' {
|
||||
pattern := strings.TrimSuffix(string(a), "*")
|
||||
for act := range supportedObjectActionList {
|
||||
if strings.HasPrefix(string(act), pattern) {
|
||||
return getBoolPtr(true)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return getBoolPtr(false)
|
||||
return false
|
||||
}
|
||||
|
||||
_, found := supportedObjectActionList[a]
|
||||
return &found
|
||||
return found
|
||||
}
|
||||
|
||||
func (a Action) WildCardMatch(act Action) bool {
|
||||
@@ -191,7 +159,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
|
||||
var err error
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return errInvalidAction
|
||||
return fmt.Errorf("actions can't be empty")
|
||||
}
|
||||
*a = make(Actions)
|
||||
for _, s := range ss {
|
||||
@@ -204,7 +172,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return errInvalidAction
|
||||
return fmt.Errorf("actions can't be empty")
|
||||
}
|
||||
*a = make(Actions)
|
||||
err = a.Add(s)
|
||||
|
||||
@@ -30,6 +30,5 @@ func (bpat BucketPolicyAccessType) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
|
||||
return fmt.Errorf("Invalid effect: %v", bpat)
|
||||
return fmt.Errorf("invalid effect: %v", bpat)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type Principals map[string]struct{}
|
||||
@@ -27,50 +28,23 @@ func (p Principals) Add(key string) {
|
||||
// Override UnmarshalJSON method to decode both []string and string properties
|
||||
func (p *Principals) UnmarshalJSON(data []byte) error {
|
||||
ss := []string{}
|
||||
var s string
|
||||
var k struct {
|
||||
AWS string
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return errInvalidPrincipal
|
||||
return fmt.Errorf("principals can't be empty")
|
||||
}
|
||||
*p = make(Principals)
|
||||
for _, s := range ss {
|
||||
p.Add(s)
|
||||
}
|
||||
return nil
|
||||
} else if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return errInvalidPrincipal
|
||||
}
|
||||
*p = make(Principals)
|
||||
p.Add(s)
|
||||
|
||||
return nil
|
||||
} else if err = json.Unmarshal(data, &k); err == nil {
|
||||
if k.AWS == "" {
|
||||
return errInvalidPrincipal
|
||||
}
|
||||
*p = make(Principals)
|
||||
p.Add(k.AWS)
|
||||
|
||||
return nil
|
||||
} else {
|
||||
var sk struct {
|
||||
AWS []string
|
||||
}
|
||||
if err = json.Unmarshal(data, &sk); err == nil {
|
||||
if len(sk.AWS) == 0 {
|
||||
return errInvalidPrincipal
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return fmt.Errorf("principals can't be empty")
|
||||
}
|
||||
*p = make(Principals)
|
||||
for _, s := range sk.AWS {
|
||||
p.Add(s)
|
||||
}
|
||||
p.Add(s)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,7 +71,7 @@ func (p Principals) Validate(iam IAMService) error {
|
||||
if len(p) == 1 {
|
||||
return nil
|
||||
}
|
||||
return errInvalidPrincipal
|
||||
return fmt.Errorf("principals should either contain * or user access keys")
|
||||
}
|
||||
|
||||
accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
|
||||
@@ -105,7 +79,7 @@ func (p Principals) Validate(iam IAMService) error {
|
||||
return err
|
||||
}
|
||||
if len(accs) > 0 {
|
||||
return errInvalidPrincipal
|
||||
return fmt.Errorf("user accounts don't exist: %v", accs)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -16,6 +16,7 @@ package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -29,7 +30,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
|
||||
var err error
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return errInvalidResource
|
||||
return fmt.Errorf("resources can't be empty")
|
||||
}
|
||||
*r = make(Resources)
|
||||
for _, s := range ss {
|
||||
@@ -42,7 +43,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return errInvalidResource
|
||||
return fmt.Errorf("resources can't be empty")
|
||||
}
|
||||
*r = make(Resources)
|
||||
err = r.Add(s)
|
||||
@@ -59,7 +60,12 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
|
||||
func (r Resources) Add(rc string) error {
|
||||
ok, pattern := isValidResource(rc)
|
||||
if !ok {
|
||||
return errInvalidResource
|
||||
return fmt.Errorf("invalid resource: %v", rc)
|
||||
}
|
||||
|
||||
_, found := r[pattern]
|
||||
if found {
|
||||
return fmt.Errorf("duplicate resource: %v", rc)
|
||||
}
|
||||
|
||||
r[pattern] = struct{}{}
|
||||
@@ -93,7 +99,7 @@ func (r Resources) ContainsBucketPattern() bool {
|
||||
func (r Resources) Validate(bucket string) error {
|
||||
for resource := range r {
|
||||
if !strings.HasPrefix(resource, bucket) {
|
||||
return errInvalidResource
|
||||
return fmt.Errorf("incorrect bucket name in %v", resource)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
129
auth/iam.go
129
auth/iam.go
@@ -28,49 +28,14 @@ const (
|
||||
RoleUserPlus Role = "userplus"
|
||||
)
|
||||
|
||||
func (r Role) IsValid() bool {
|
||||
switch r {
|
||||
case RoleAdmin:
|
||||
return true
|
||||
case RoleUser:
|
||||
return true
|
||||
case RoleUserPlus:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Account is a gateway IAM account
|
||||
type Account struct {
|
||||
Access string `json:"access"`
|
||||
Secret string `json:"secret"`
|
||||
Role Role `json:"role"`
|
||||
UserID int `json:"userID"`
|
||||
GroupID int `json:"groupID"`
|
||||
}
|
||||
|
||||
type ListUserAccountsResult struct {
|
||||
Accounts []Account
|
||||
}
|
||||
|
||||
// Mutable props, which could be changed when updating an IAM account
|
||||
type MutableProps struct {
|
||||
Secret *string `json:"secret"`
|
||||
UserID *int `json:"userID"`
|
||||
GroupID *int `json:"groupID"`
|
||||
}
|
||||
|
||||
func updateAcc(acc *Account, props MutableProps) {
|
||||
if props.Secret != nil {
|
||||
acc.Secret = *props.Secret
|
||||
}
|
||||
if props.GroupID != nil {
|
||||
acc.GroupID = *props.GroupID
|
||||
}
|
||||
if props.UserID != nil {
|
||||
acc.UserID = *props.UserID
|
||||
}
|
||||
Access string `json:"access"`
|
||||
Secret string `json:"secret"`
|
||||
Role Role `json:"role"`
|
||||
UserID int `json:"userID"`
|
||||
GroupID int `json:"groupID"`
|
||||
ProjectID int `json:"projectID"`
|
||||
}
|
||||
|
||||
// IAMService is the interface for all IAM service implementations
|
||||
@@ -79,57 +44,33 @@ func updateAcc(acc *Account, props MutableProps) {
|
||||
type IAMService interface {
|
||||
CreateAccount(account Account) error
|
||||
GetUserAccount(access string) (Account, error)
|
||||
UpdateUserAccount(access string, props MutableProps) error
|
||||
DeleteUserAccount(access string) error
|
||||
ListUserAccounts() ([]Account, error)
|
||||
Shutdown() error
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrUserExists is returned when the user already exists
|
||||
ErrUserExists = errors.New("user already exists")
|
||||
// ErrNoSuchUser is returned when the user does not exist
|
||||
ErrNoSuchUser = errors.New("user not found")
|
||||
)
|
||||
var ErrNoSuchUser = errors.New("user not found")
|
||||
|
||||
type Opts struct {
|
||||
RootAccount Account
|
||||
Dir string
|
||||
LDAPServerURL string
|
||||
LDAPBindDN string
|
||||
LDAPPassword string
|
||||
LDAPQueryBase string
|
||||
LDAPObjClasses string
|
||||
LDAPAccessAtr string
|
||||
LDAPSecretAtr string
|
||||
LDAPRoleAtr string
|
||||
LDAPUserIdAtr string
|
||||
LDAPGroupIdAtr string
|
||||
VaultEndpointURL string
|
||||
VaultSecretStoragePath string
|
||||
VaultMountPath string
|
||||
VaultRootToken string
|
||||
VaultRoleId string
|
||||
VaultRoleSecret string
|
||||
VaultServerCert string
|
||||
VaultClientCert string
|
||||
VaultClientCertKey string
|
||||
S3Access string
|
||||
S3Secret string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3Endpoint string
|
||||
S3DisableSSlVerfiy bool
|
||||
S3Debug bool
|
||||
CacheDisable bool
|
||||
CacheTTL int
|
||||
CachePrune int
|
||||
IpaHost string
|
||||
IpaVaultName string
|
||||
IpaUser string
|
||||
IpaPassword string
|
||||
IpaInsecure bool
|
||||
IpaDebug bool
|
||||
Dir string
|
||||
LDAPServerURL string
|
||||
LDAPBindDN string
|
||||
LDAPPassword string
|
||||
LDAPQueryBase string
|
||||
LDAPObjClasses string
|
||||
LDAPAccessAtr string
|
||||
LDAPSecretAtr string
|
||||
LDAPRoleAtr string
|
||||
S3Access string
|
||||
S3Secret string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3Endpoint string
|
||||
S3DisableSSlVerfiy bool
|
||||
S3Debug bool
|
||||
CacheDisable bool
|
||||
CacheTTL int
|
||||
CachePrune int
|
||||
}
|
||||
|
||||
func New(o *Opts) (IAMService, error) {
|
||||
@@ -138,26 +79,18 @@ func New(o *Opts) (IAMService, error) {
|
||||
|
||||
switch {
|
||||
case o.Dir != "":
|
||||
svc, err = NewInternal(o.RootAccount, o.Dir)
|
||||
svc, err = NewInternal(o.Dir)
|
||||
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
|
||||
case o.LDAPServerURL != "":
|
||||
svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
|
||||
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
|
||||
o.LDAPGroupIdAtr, o.LDAPObjClasses)
|
||||
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
|
||||
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
|
||||
o.LDAPObjClasses)
|
||||
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
|
||||
case o.S3Endpoint != "":
|
||||
svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
|
||||
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
|
||||
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
|
||||
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
|
||||
o.S3Endpoint, o.S3Bucket)
|
||||
case o.VaultEndpointURL != "":
|
||||
svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
|
||||
o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
|
||||
o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
|
||||
fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
|
||||
case o.IpaHost != "":
|
||||
svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure, o.IpaDebug)
|
||||
fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
|
||||
default:
|
||||
// if no iam options selected, default to the single user mode
|
||||
fmt.Println("No IAM service configured, enabling single account mode")
|
||||
|
||||
@@ -66,21 +66,6 @@ func (i *icache) get(k string) (Account, bool) {
|
||||
return v.value, true
|
||||
}
|
||||
|
||||
func (i *icache) update(k string, props MutableProps) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
item, found := i.items[k]
|
||||
if found {
|
||||
updateAcc(&item.value, props)
|
||||
|
||||
// refresh the expiration date
|
||||
item.exp = time.Now().Add(i.expire)
|
||||
|
||||
i.items[k] = item
|
||||
}
|
||||
}
|
||||
|
||||
func (i *icache) Delete(k string) {
|
||||
i.Lock()
|
||||
delete(i.items, k)
|
||||
@@ -181,16 +166,6 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
|
||||
err := c.service.UpdateUserAccount(access, props)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.iamcache.update(access, props)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListUserAccounts is a passthrough to the underlying service and
|
||||
// does not make use of the cache
|
||||
func (c *IAMCache) ListUserAccounts() ([]Account, error) {
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -33,15 +32,7 @@ const (
|
||||
|
||||
// IAMServiceInternal manages the internal IAM service
|
||||
type IAMServiceInternal struct {
|
||||
// This mutex will help with racing updates to the IAM data
|
||||
// from multiple requests to this gateway instance, but
|
||||
// will not help with racing updates to multiple load balanced
|
||||
// gateway instances. This is a limitation of the internal
|
||||
// IAM service. All account updates should be sent to a single
|
||||
// gateway instance if possible.
|
||||
sync.RWMutex
|
||||
dir string
|
||||
rootAcc Account
|
||||
dir string
|
||||
}
|
||||
|
||||
// UpdateAcctFunc accepts the current data and returns the new data to be stored
|
||||
@@ -55,10 +46,9 @@ type iAMConfig struct {
|
||||
var _ IAMService = &IAMServiceInternal{}
|
||||
|
||||
// NewInternal creates a new instance for the Internal IAM service
|
||||
func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
|
||||
func NewInternal(dir string) (*IAMServiceInternal, error) {
|
||||
i := &IAMServiceInternal{
|
||||
dir: dir,
|
||||
rootAcc: rootAcc,
|
||||
dir: dir,
|
||||
}
|
||||
|
||||
err := i.initIAM()
|
||||
@@ -72,13 +62,6 @@ func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
|
||||
// CreateAccount creates a new IAM account. Returns an error if the account
|
||||
// already exists.
|
||||
func (s *IAMServiceInternal) CreateAccount(account Account) error {
|
||||
if account.Access == s.rootAcc.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
@@ -87,7 +70,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
|
||||
|
||||
_, ok := conf.AccessAccounts[account.Access]
|
||||
if ok {
|
||||
return nil, ErrUserExists
|
||||
return nil, fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[account.Access] = account
|
||||
|
||||
@@ -103,13 +86,6 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
|
||||
// GetUserAccount retrieves account info for the requested user. Returns
|
||||
// ErrNoSuchUser if the account does not exist.
|
||||
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
|
||||
if access == s.rootAcc.Access {
|
||||
return s.rootAcc, nil
|
||||
}
|
||||
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getIAM()
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("get iam data: %w", err)
|
||||
@@ -123,41 +99,9 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
// UpdateUserAccount updates the specified user account fields. Returns
|
||||
// ErrNoSuchUser if the account does not exist.
|
||||
func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
acc, found := conf.AccessAccounts[access]
|
||||
if !found {
|
||||
return nil, ErrNoSuchUser
|
||||
}
|
||||
|
||||
updateAcc(&acc, props)
|
||||
conf.AccessAccounts[access] = acc
|
||||
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteUserAccount deletes the specified user account. Does not check if
|
||||
// account exists.
|
||||
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
@@ -177,9 +121,6 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
|
||||
|
||||
// ListUserAccounts lists all the user accounts stored.
|
||||
func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getIAM()
|
||||
if err != nil {
|
||||
return []Account{}, fmt.Errorf("get iam data: %w", err)
|
||||
@@ -194,11 +135,12 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
|
||||
var accs []Account
|
||||
for _, k := range keys {
|
||||
accs = append(accs, Account{
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
ProjectID: conf.AccessAccounts[k].ProjectID,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -247,10 +189,6 @@ func parseIAM(b []byte) (iAMConfig, error) {
|
||||
return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
|
||||
}
|
||||
|
||||
if conf.AccessAccounts == nil {
|
||||
conf.AccessAccounts = make(map[string]Account)
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
|
||||
446
auth/iam_ipa.go
446
auth/iam_ipa.go
@@ -1,446 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const IpaVersion = "2.254"
|
||||
|
||||
type IpaIAMService struct {
|
||||
client http.Client
|
||||
id int
|
||||
version string
|
||||
host string
|
||||
vaultName string
|
||||
username string
|
||||
password string
|
||||
kraTransportKey *rsa.PublicKey
|
||||
debug bool
|
||||
rootAcc Account
|
||||
}
|
||||
|
||||
var _ IAMService = &IpaIAMService{}
|
||||
|
||||
func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
|
||||
|
||||
ipa := IpaIAMService{
|
||||
id: 0,
|
||||
version: IpaVersion,
|
||||
host: host,
|
||||
vaultName: vaultName,
|
||||
username: username,
|
||||
password: password,
|
||||
debug: debug,
|
||||
rootAcc: rootAcc,
|
||||
}
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
// this should never happen
|
||||
return nil, fmt.Errorf("cookie jar creation: %w", err)
|
||||
}
|
||||
|
||||
mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: mTLSConfig,
|
||||
}
|
||||
ipa.client = http.Client{Jar: jar, Transport: tr}
|
||||
|
||||
err = ipa.login()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa login failed: %w", err)
|
||||
}
|
||||
|
||||
req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
|
||||
}
|
||||
vaultConfig := struct {
|
||||
Kra_Server_Server []string
|
||||
Transport_Cert Base64EncodedWrapped
|
||||
Wrapping_default_algorithm string
|
||||
Wrapping_supported_algorithms []string
|
||||
}{}
|
||||
err = ipa.rpc(req, &vaultConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa vault config: %w", err)
|
||||
}
|
||||
|
||||
cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
|
||||
}
|
||||
|
||||
ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
|
||||
|
||||
isSupported := false
|
||||
for _, algo := range vaultConfig.Wrapping_supported_algorithms {
|
||||
if algo == "aes-128-cbc" {
|
||||
isSupported = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !isSupported {
|
||||
return nil,
|
||||
fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
|
||||
vaultConfig.Wrapping_supported_algorithms)
|
||||
}
|
||||
return &ipa, nil
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) CreateAccount(account Account) error {
|
||||
return fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
|
||||
if access == ipa.rootAcc.Access {
|
||||
return ipa.rootAcc, nil
|
||||
}
|
||||
|
||||
req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa user_show: %w", err)
|
||||
}
|
||||
|
||||
userResult := struct {
|
||||
Gidnumber []string
|
||||
Uidnumber []string
|
||||
}{}
|
||||
|
||||
err = ipa.rpc(req, &userResult)
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
uid, err := strconv.Atoi(userResult.Uidnumber[0])
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
|
||||
}
|
||||
gid, err := strconv.Atoi(userResult.Gidnumber[0])
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
|
||||
}
|
||||
|
||||
account := Account{
|
||||
Access: access,
|
||||
Role: RoleUser,
|
||||
UserID: uid,
|
||||
GroupID: gid,
|
||||
}
|
||||
|
||||
session_key := make([]byte, 16)
|
||||
|
||||
_, err = rand.Read(session_key)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa cannot generate session key: %w", err)
|
||||
}
|
||||
|
||||
encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
|
||||
}
|
||||
|
||||
req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
|
||||
map[string]any{"username": access,
|
||||
"session_key": Base64EncodedWrapped(encryptedKey),
|
||||
"wrapping_algo": "aes-128-cbc"})
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
|
||||
}
|
||||
|
||||
data := struct {
|
||||
Vault_data Base64EncodedWrapped
|
||||
Nonce Base64EncodedWrapped
|
||||
}{}
|
||||
|
||||
err = ipa.rpc(req, &data)
|
||||
if err != nil {
|
||||
return account, err
|
||||
}
|
||||
|
||||
aes, err := aes.NewCipher(session_key)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
|
||||
}
|
||||
cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
|
||||
cbc.CryptBlocks(data.Vault_data, data.Vault_data)
|
||||
secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
|
||||
if err != nil {
|
||||
return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
|
||||
}
|
||||
|
||||
secret := struct {
|
||||
Data Base64Encoded
|
||||
}{}
|
||||
json.Unmarshal(secretUnpaddedJson, &secret)
|
||||
account.Secret = string(secret.Data)
|
||||
|
||||
return account, nil
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
|
||||
return fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
|
||||
return fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
|
||||
return []Account{}, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Implementation
|
||||
|
||||
func (ipa *IpaIAMService) login() error {
|
||||
form := url.Values{}
|
||||
form.Set("user", ipa.username)
|
||||
form.Set("password", ipa.password)
|
||||
|
||||
req, err := http.NewRequest(
|
||||
"POST",
|
||||
fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
|
||||
strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
resp, err := ipa.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode == 401 {
|
||||
return errors.New("cannot login to FreeIPA: invalid credentials")
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type rpcRequest = string
|
||||
|
||||
type rpcResponse struct {
|
||||
Result json.RawMessage
|
||||
Principal string
|
||||
Id int
|
||||
Version string
|
||||
}
|
||||
|
||||
func (p rpcResponse) String() string {
|
||||
return string(p.Result)
|
||||
}
|
||||
|
||||
var errRpc = errors.New("IPA RPC error")
|
||||
|
||||
func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
|
||||
err := ipa.login()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res, err := ipa.rpcInternal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return json.Unmarshal(res.Result, value)
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
|
||||
httpReq, err := http.NewRequest("POST",
|
||||
fmt.Sprintf("%s/ipa/session/json", ipa.host),
|
||||
strings.NewReader(req))
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
|
||||
ipa.log(fmt.Sprintf("%v", req))
|
||||
httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
httpResp, err := ipa.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
|
||||
bytes, err := io.ReadAll(httpResp.Body)
|
||||
ipa.log(string(bytes))
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
|
||||
result := struct {
|
||||
Result struct {
|
||||
Json json.RawMessage `json:"result"`
|
||||
Value string `json:"value"`
|
||||
Summary any `json:"summary"`
|
||||
} `json:"result"`
|
||||
Error json.RawMessage `json:"error"`
|
||||
Id int `json:"id"`
|
||||
Principal string `json:"principal"`
|
||||
Version string `json:"version"`
|
||||
}{}
|
||||
|
||||
err = json.Unmarshal(bytes, &result)
|
||||
if err != nil {
|
||||
return rpcResponse{}, err
|
||||
}
|
||||
if string(result.Error) != "null" {
|
||||
return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
|
||||
}
|
||||
|
||||
return rpcResponse{
|
||||
Result: result.Result.Json,
|
||||
Principal: result.Principal,
|
||||
Id: result.Id,
|
||||
Version: result.Version,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
|
||||
|
||||
id := ipa.id
|
||||
ipa.id++
|
||||
|
||||
dict["version"] = ipa.version
|
||||
|
||||
jmethod, errMethod := json.Marshal(method)
|
||||
jargs, errArgs := json.Marshal(args)
|
||||
jdict, errDict := json.Marshal(dict)
|
||||
|
||||
err := errors.Join(errMethod, errArgs, errDict)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ipa request invalid: %w", err)
|
||||
}
|
||||
|
||||
request := map[string]interface{}{
|
||||
"id": id,
|
||||
"method": json.RawMessage(jmethod),
|
||||
"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
|
||||
}
|
||||
|
||||
requestJSON, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
return string(requestJSON), nil
|
||||
}
|
||||
|
||||
// pkcs7Unpad validates and unpads data from the given bytes slice.
|
||||
// The returned value will be 1 to n bytes smaller depending on the
|
||||
// amount of padding, where n is the block size.
|
||||
func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
|
||||
if blocksize <= 0 {
|
||||
return nil, errors.New("invalid blocksize")
|
||||
}
|
||||
if len(b) == 0 {
|
||||
return nil, errors.New("invalid PKCS7 data (empty or not padded)")
|
||||
}
|
||||
if len(b)%blocksize != 0 {
|
||||
return nil, errors.New("invalid padding on input")
|
||||
}
|
||||
c := b[len(b)-1]
|
||||
n := int(c)
|
||||
if n == 0 || n > len(b) {
|
||||
return nil, errors.New("invalid padding on input")
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if b[len(b)-n+i] != c {
|
||||
return nil, errors.New("invalid padding on input")
|
||||
}
|
||||
}
|
||||
return b[:len(b)-n], nil
|
||||
}
|
||||
|
||||
/*
|
||||
e.g.
|
||||
|
||||
"value" {
|
||||
"__base64__": "aGVsbG93b3JsZAo="
|
||||
}
|
||||
*/
|
||||
type Base64EncodedWrapped []byte
|
||||
|
||||
func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
|
||||
intermediate := struct {
|
||||
Base64 string `json:"__base64__"`
|
||||
}{}
|
||||
err := json.Unmarshal(data, &intermediate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
|
||||
intermediate := struct {
|
||||
Base64 string `json:"__base64__"`
|
||||
}{Base64: base64.StdEncoding.EncodeToString(*b)}
|
||||
return json.Marshal(intermediate)
|
||||
}
|
||||
|
||||
/*
|
||||
e.g.
|
||||
|
||||
"value": "aGVsbG93b3JsZAo="
|
||||
*/
|
||||
type Base64Encoded []byte
|
||||
|
||||
func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
|
||||
var intermediate string
|
||||
err := json.Unmarshal(data, &intermediate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*b, err = base64.StdEncoding.DecodeString(intermediate)
|
||||
return err
|
||||
}
|
||||
|
||||
func (ipa *IpaIAMService) log(msg string) {
|
||||
if ipa.debug {
|
||||
log.Println(msg)
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,7 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-ldap/ldap/v3"
|
||||
@@ -29,19 +14,15 @@ type LdapIAMService struct {
|
||||
accessAtr string
|
||||
secretAtr string
|
||||
roleAtr string
|
||||
groupIdAtr string
|
||||
userIdAtr string
|
||||
rootAcc Account
|
||||
}
|
||||
|
||||
var _ IAMService = &LdapIAMService{}
|
||||
|
||||
func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
|
||||
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
|
||||
secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
|
||||
func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
|
||||
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
|
||||
return nil, fmt.Errorf("required parameters list not fully provided")
|
||||
}
|
||||
conn, err := ldap.DialURL(url)
|
||||
conn, err := ldap.Dial("tcp", url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
|
||||
}
|
||||
@@ -57,23 +38,15 @@ func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAt
|
||||
accessAtr: accAtr,
|
||||
secretAtr: secAtr,
|
||||
roleAtr: roleAtr,
|
||||
userIdAtr: userIdAtr,
|
||||
groupIdAtr: groupIdAtr,
|
||||
rootAcc: rootAcc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) CreateAccount(account Account) error {
|
||||
if ld.rootAcc.Access == account.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
|
||||
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
|
||||
userEntry.Attribute("objectClass", ld.objClasses)
|
||||
userEntry.Attribute(ld.accessAtr, []string{account.Access})
|
||||
userEntry.Attribute(ld.secretAtr, []string{account.Secret})
|
||||
userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
|
||||
userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
|
||||
userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
|
||||
|
||||
err := ld.conn.Add(userEntry)
|
||||
if err != nil {
|
||||
@@ -84,9 +57,6 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
|
||||
if access == ld.rootAcc.Access {
|
||||
return ld.rootAcc, nil
|
||||
}
|
||||
searchRequest := ldap.NewSearchRequest(
|
||||
ld.queryBase,
|
||||
ldap.ScopeWholeSubtree,
|
||||
@@ -95,7 +65,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
|
||||
0,
|
||||
false,
|
||||
fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
|
||||
nil,
|
||||
)
|
||||
|
||||
@@ -104,48 +74,14 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
if len(result.Entries) == 0 {
|
||||
return Account{}, ErrNoSuchUser
|
||||
}
|
||||
|
||||
entry := result.Entries[0]
|
||||
groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
|
||||
}
|
||||
userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
|
||||
}
|
||||
return Account{
|
||||
Access: entry.GetAttributeValue(ld.accessAtr),
|
||||
Secret: entry.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
|
||||
GroupID: groupId,
|
||||
UserID: userId,
|
||||
Access: entry.GetAttributeValue(ld.accessAtr),
|
||||
Secret: entry.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
|
||||
req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
|
||||
if props.Secret != nil {
|
||||
req.Replace(ld.secretAtr, []string{*props.Secret})
|
||||
}
|
||||
if props.GroupID != nil {
|
||||
req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
|
||||
}
|
||||
if props.UserID != nil {
|
||||
req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
|
||||
}
|
||||
|
||||
err := ld.conn.Modify(req)
|
||||
//TODO: Handle non existing user case
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) DeleteUserAccount(access string) error {
|
||||
delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
|
||||
|
||||
@@ -170,7 +106,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
|
||||
0,
|
||||
false,
|
||||
fmt.Sprintf("(&%v)", searchFilter),
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
|
||||
nil,
|
||||
)
|
||||
|
||||
@@ -181,20 +117,10 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
|
||||
|
||||
result := []Account{}
|
||||
for _, el := range resp.Entries {
|
||||
groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
|
||||
}
|
||||
userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
|
||||
}
|
||||
result = append(result, Account{
|
||||
Access: el.GetAttributeValue(ld.accessAtr),
|
||||
Secret: el.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(el.GetAttributeValue(ld.roleAtr)),
|
||||
GroupID: groupId,
|
||||
UserID: userId,
|
||||
Access: el.GetAttributeValue(ld.accessAtr),
|
||||
Secret: el.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(el.GetAttributeValue(ld.roleAtr)),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
@@ -42,14 +41,6 @@ import (
|
||||
// coming from iAMConfig and iamFile in iam_internal.
|
||||
|
||||
type IAMServiceS3 struct {
|
||||
// This mutex will help with racing updates to the IAM data
|
||||
// from multiple requests to this gateway instance, but
|
||||
// will not help with racing updates to multiple load balanced
|
||||
// gateway instances. This is a limitation of the internal
|
||||
// IAM service. All account updates should be sent to a single
|
||||
// gateway instance if possible.
|
||||
sync.RWMutex
|
||||
|
||||
access string
|
||||
secret string
|
||||
region string
|
||||
@@ -57,13 +48,12 @@ type IAMServiceS3 struct {
|
||||
endpoint string
|
||||
sslSkipVerify bool
|
||||
debug bool
|
||||
rootAcc Account
|
||||
client *s3.Client
|
||||
}
|
||||
|
||||
var _ IAMService = &IAMServiceS3{}
|
||||
|
||||
func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
|
||||
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
|
||||
if access == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service access key")
|
||||
}
|
||||
@@ -88,7 +78,6 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
|
||||
endpoint: endpoint,
|
||||
sslSkipVerify: sslSkipVerify,
|
||||
debug: debug,
|
||||
rootAcc: rootAcc,
|
||||
}
|
||||
|
||||
cfg, err := i.getConfig()
|
||||
@@ -96,25 +85,11 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
|
||||
return nil, fmt.Errorf("init s3 IAM: %v", err)
|
||||
}
|
||||
|
||||
if endpoint != "" {
|
||||
i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = &endpoint
|
||||
})
|
||||
return i, nil
|
||||
}
|
||||
|
||||
i.client = s3.NewFromConfig(cfg)
|
||||
return i, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
if s.rootAcc.Access == account.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -122,7 +97,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
|
||||
_, ok := conf.AccessAccounts[account.Access]
|
||||
if ok {
|
||||
return ErrUserExists
|
||||
return fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[account.Access] = account
|
||||
|
||||
@@ -130,13 +105,6 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
|
||||
if access == s.rootAcc.Access {
|
||||
return s.rootAcc, nil
|
||||
}
|
||||
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
@@ -150,30 +118,7 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
acc, ok := conf.AccessAccounts[access]
|
||||
if !ok {
|
||||
return ErrNoSuchUser
|
||||
}
|
||||
|
||||
updateAcc(&acc, props)
|
||||
conf.AccessAccounts[access] = acc
|
||||
|
||||
return s.storeAccts(conf)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -189,9 +134,6 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -206,17 +148,28 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
|
||||
var accs []Account
|
||||
for _, k := range keys {
|
||||
accs = append(accs, Account{
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
ProjectID: conf.AccessAccounts[k].ProjectID,
|
||||
})
|
||||
}
|
||||
|
||||
return accs, nil
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
SigningRegion: s.region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
@@ -235,6 +188,11 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(s))
|
||||
}
|
||||
|
||||
if s.debug {
|
||||
opts = append(opts,
|
||||
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
|
||||
@@ -252,15 +210,15 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
|
||||
})
|
||||
if err != nil {
|
||||
// if the error is object not exists,
|
||||
// init empty accounts struct and return that
|
||||
// init empty accounts stuct and return that
|
||||
var nsk *types.NoSuchKey
|
||||
if errors.As(err, &nsk) {
|
||||
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
if apiErr.ErrorCode() == "NotFound" {
|
||||
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// IAMServiceSingle manages the single tenant (root-only) IAM service
|
||||
@@ -23,29 +23,26 @@ type IAMServiceSingle struct{}
|
||||
|
||||
var _ IAMService = &IAMServiceSingle{}
|
||||
|
||||
var ErrNotSupported = errors.New("method is not supported")
|
||||
|
||||
// CreateAccount not valid in single tenant mode
|
||||
func (IAMServiceSingle) CreateAccount(account Account) error {
|
||||
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
// GetUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
|
||||
return Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
}
|
||||
|
||||
// UpdateUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
|
||||
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
return Account{}, ErrNotSupported
|
||||
}
|
||||
|
||||
// DeleteUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) DeleteUserAccount(access string) error {
|
||||
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
// ListUserAccounts no accounts in single tenant mode
|
||||
func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
|
||||
return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
|
||||
return []Account{}, nil
|
||||
}
|
||||
|
||||
// Shutdown graceful termination of service
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
vault "github.com/hashicorp/vault-client-go"
|
||||
"github.com/hashicorp/vault-client-go/schema"
|
||||
)
|
||||
|
||||
type VaultIAMService struct {
|
||||
client *vault.Client
|
||||
reqOpts []vault.RequestOption
|
||||
secretStoragePath string
|
||||
rootAcc Account
|
||||
}
|
||||
|
||||
var _ IAMService = &VaultIAMService{}
|
||||
|
||||
func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
|
||||
opts := []vault.ClientOption{
|
||||
vault.WithAddress(endpoint),
|
||||
// set request timeout to 10 secs
|
||||
vault.WithRequestTimeout(10 * time.Second),
|
||||
}
|
||||
if serverCert != "" {
|
||||
tls := vault.TLSConfiguration{}
|
||||
|
||||
tls.ServerCertificate.FromBytes = []byte(serverCert)
|
||||
if clientCert != "" {
|
||||
if clientCertKey == "" {
|
||||
return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
|
||||
}
|
||||
|
||||
tls.ClientCertificate.FromBytes = []byte(clientCert)
|
||||
tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
|
||||
}
|
||||
|
||||
opts = append(opts, vault.WithTLS(tls))
|
||||
}
|
||||
|
||||
client, err := vault.New(opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init vault client: %w", err)
|
||||
}
|
||||
|
||||
reqOpts := []vault.RequestOption{}
|
||||
// if mount path is not specified, it defaults to "approle"
|
||||
if mountPath != "" {
|
||||
reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
|
||||
}
|
||||
|
||||
// Authentication
|
||||
switch {
|
||||
case rootToken != "":
|
||||
err := client.SetToken(rootToken)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("root token authentication failure: %w", err)
|
||||
}
|
||||
case roleID != "":
|
||||
if roleSecret == "" {
|
||||
return nil, fmt.Errorf("role id and role secret must both be specified")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
|
||||
RoleId: roleID,
|
||||
SecretId: roleSecret,
|
||||
}, reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("approle authentication failure: %w", err)
|
||||
}
|
||||
|
||||
if err := client.SetToken(resp.Auth.ClientToken); err != nil {
|
||||
return nil, fmt.Errorf("approle authentication set token failure: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
|
||||
}
|
||||
|
||||
return &VaultIAMService{
|
||||
client: client,
|
||||
reqOpts: reqOpts,
|
||||
secretStoragePath: secretStoragePath,
|
||||
rootAcc: rootAcc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) CreateAccount(account Account) error {
|
||||
if vt.rootAcc.Access == account.Access {
|
||||
return ErrUserExists
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
|
||||
Data: map[string]any{
|
||||
account.Access: account,
|
||||
},
|
||||
Options: map[string]interface{}{
|
||||
"cas": 0,
|
||||
},
|
||||
}, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "check-and-set") {
|
||||
return ErrUserExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
|
||||
if vt.rootAcc.Access == access {
|
||||
return vt.rootAcc, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
acc, err := parseVaultUserAccount(resp.Data.Data, access)
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
return acc, nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
|
||||
//TODO: We need something like a transaction here ?
|
||||
acc, err := vt.GetUserAccount(access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updateAcc(&acc, props)
|
||||
|
||||
err = vt.DeleteUserAccount(access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = vt.CreateAccount(acc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) DeleteUserAccount(access string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if vault.IsErrorStatus(err, 404) {
|
||||
return []Account{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accs := []Account{}
|
||||
|
||||
for _, acss := range resp.Data.Keys {
|
||||
acc, err := vt.GetUserAccount(acss)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accs = append(accs, acc)
|
||||
}
|
||||
|
||||
return accs, nil
|
||||
}
|
||||
|
||||
// the client doesn't have explicit shutdown, as it uses http.Client
|
||||
func (vt *VaultIAMService) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
|
||||
|
||||
func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
|
||||
usrAcc, ok := data[access].(map[string]interface{})
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
|
||||
acss, ok := usrAcc["access"].(string)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
secret, ok := usrAcc["secret"].(string)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
role, ok := usrAcc["role"].(string)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
userIdJson, ok := usrAcc["userID"].(json.Number)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
userId, err := userIdJson.Int64()
|
||||
if err != nil {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
groupIdJson, ok := usrAcc["groupID"].(json.Number)
|
||||
if !ok {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
groupId, err := groupIdJson.Int64()
|
||||
if err != nil {
|
||||
return acc, errInvalidUser
|
||||
}
|
||||
|
||||
return Account{
|
||||
Access: acss,
|
||||
Secret: secret,
|
||||
Role: Role(role),
|
||||
UserID: int(userId),
|
||||
GroupID: int(groupId),
|
||||
}, nil
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type BucketLockConfig struct {
|
||||
Enabled bool
|
||||
DefaultRetention *types.DefaultRetention
|
||||
CreatedAt *time.Time
|
||||
}
|
||||
|
||||
func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
|
||||
var lockConfig types.ObjectLockConfiguration
|
||||
if err := xml.Unmarshal(input, &lockConfig); err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
config := BucketLockConfig{
|
||||
Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
|
||||
}
|
||||
|
||||
if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
|
||||
retention := lockConfig.Rule.DefaultRetention
|
||||
|
||||
if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
if retention.Years != nil && retention.Days != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
if retention.Days != nil && *retention.Days <= 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
|
||||
}
|
||||
if retention.Years != nil && *retention.Years <= 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
|
||||
}
|
||||
|
||||
config.DefaultRetention = retention
|
||||
now := time.Now()
|
||||
config.CreatedAt = &now
|
||||
}
|
||||
|
||||
return json.Marshal(config)
|
||||
}
|
||||
|
||||
func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
|
||||
var config BucketLockConfig
|
||||
if err := json.Unmarshal(input, &config); err != nil {
|
||||
return nil, fmt.Errorf("parse object lock config: %w", err)
|
||||
}
|
||||
|
||||
result := &types.ObjectLockConfiguration{
|
||||
Rule: &types.ObjectLockRule{
|
||||
DefaultRetention: config.DefaultRetention,
|
||||
},
|
||||
}
|
||||
|
||||
if config.Enabled {
|
||||
result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
|
||||
var retention s3response.PutObjectRetentionInput
|
||||
if err := xml.Unmarshal(input, &retention); err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
if retention.RetainUntilDate.Before(time.Now()) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
|
||||
}
|
||||
switch retention.Mode {
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
default:
|
||||
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
return json.Marshal(retention)
|
||||
}
|
||||
|
||||
func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
|
||||
var retention types.ObjectLockRetention
|
||||
if err := json.Unmarshal(input, &retention); err != nil {
|
||||
return nil, fmt.Errorf("parse object lock retention: %w", err)
|
||||
}
|
||||
|
||||
return &retention, nil
|
||||
}
|
||||
|
||||
func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
|
||||
if status == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if *status {
|
||||
return &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
}
|
||||
}
|
||||
|
||||
return &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOff,
|
||||
}
|
||||
}
|
||||
|
||||
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
|
||||
data, err := be.GetObjectLockConfiguration(ctx, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var bucketLockConfig BucketLockConfig
|
||||
if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
|
||||
return fmt.Errorf("parse object lock config: %w", err)
|
||||
}
|
||||
|
||||
if !bucketLockConfig.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
checkDefaultRetention := false
|
||||
|
||||
if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
|
||||
expirationDate := *bucketLockConfig.CreatedAt
|
||||
if bucketLockConfig.DefaultRetention.Days != nil {
|
||||
expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
|
||||
}
|
||||
if bucketLockConfig.DefaultRetention.Years != nil {
|
||||
expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
|
||||
}
|
||||
|
||||
if expirationDate.After(time.Now()) {
|
||||
checkDefaultRetention = true
|
||||
}
|
||||
}
|
||||
|
||||
for _, obj := range objects {
|
||||
var key, versionId string
|
||||
if obj.Key != nil {
|
||||
key = *obj.Key
|
||||
}
|
||||
if obj.VersionId != nil {
|
||||
versionId = *obj.VersionId
|
||||
}
|
||||
checkRetention := true
|
||||
retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
|
||||
checkRetention = false
|
||||
}
|
||||
if err != nil && checkRetention {
|
||||
return err
|
||||
}
|
||||
|
||||
if checkRetention {
|
||||
retention, err := ParseObjectLockRetentionOutput(retentionData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if retention.Mode != "" && retention.RetainUntilDate != nil {
|
||||
if retention.RetainUntilDate.After(time.Now()) {
|
||||
switch retention.Mode {
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
if !bypass {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
} else {
|
||||
policy, err := be.GetBucketPolicy(ctx, bucket)
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checkLegalHold := true
|
||||
|
||||
status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
|
||||
if err != nil {
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
|
||||
checkLegalHold = false
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if checkLegalHold && *status {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
|
||||
if checkDefaultRetention {
|
||||
switch bucketLockConfig.DefaultRetention.Mode {
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
if !bypass {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
} else {
|
||||
policy, err := be.GetBucketPolicy(ctx, bucket)
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
|
||||
|
||||
actual := req.Header.Get("Authorization")
|
||||
if e, a := c.ExpSig, actual; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.OrigURI, req.URL.Path; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
|
||||
|
||||
actual := req.Header.Get("Authorization")
|
||||
if e, a := c.ExpSig, actual; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.OrigURI, req.URL.Path; e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,9 +18,9 @@ import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
"github.com/versity/versitygw/s3select"
|
||||
@@ -32,23 +32,20 @@ type Backend interface {
|
||||
Shutdown()
|
||||
|
||||
// bucket operations
|
||||
ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
|
||||
ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
|
||||
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
|
||||
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
|
||||
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
|
||||
PutBucketAcl(_ context.Context, bucket string, data []byte) error
|
||||
DeleteBucket(_ context.Context, bucket string) error
|
||||
PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
|
||||
GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
|
||||
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
|
||||
PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
|
||||
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
|
||||
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
|
||||
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
|
||||
DeleteBucketPolicy(_ context.Context, bucket string) error
|
||||
PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
|
||||
GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
|
||||
DeleteBucketOwnershipControls(_ context.Context, bucket string) error
|
||||
|
||||
// multipart operations
|
||||
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
|
||||
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
|
||||
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
|
||||
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
|
||||
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
|
||||
@@ -57,18 +54,18 @@ type Backend interface {
|
||||
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
|
||||
|
||||
// standard object operations
|
||||
PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
|
||||
PutObject(context.Context, *s3.PutObjectInput) (string, error)
|
||||
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
|
||||
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
|
||||
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
|
||||
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
|
||||
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
|
||||
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
|
||||
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
|
||||
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
|
||||
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
|
||||
DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
|
||||
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
|
||||
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
|
||||
DeleteObject(context.Context, *s3.DeleteObjectInput) error
|
||||
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
|
||||
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
|
||||
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
|
||||
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
|
||||
|
||||
// special case object operations
|
||||
RestoreObject(context.Context, *s3.RestoreObjectInput) error
|
||||
@@ -84,16 +81,8 @@ type Backend interface {
|
||||
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
|
||||
DeleteObjectTagging(_ context.Context, bucket, object string) error
|
||||
|
||||
// object lock operations
|
||||
PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
|
||||
GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
|
||||
PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
|
||||
GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
|
||||
PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
|
||||
GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
|
||||
|
||||
// non AWS actions
|
||||
ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
|
||||
ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
|
||||
ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
|
||||
}
|
||||
|
||||
@@ -108,7 +97,7 @@ func (BackendUnsupported) Shutdown() {}
|
||||
func (BackendUnsupported) String() string {
|
||||
return "Unsupported"
|
||||
}
|
||||
func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
@@ -123,14 +112,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
|
||||
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
|
||||
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
|
||||
func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
@@ -141,18 +130,9 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
|
||||
func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
|
||||
return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
@@ -173,33 +153,33 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu
|
||||
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
|
||||
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
|
||||
return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
|
||||
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
@@ -225,8 +205,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
|
||||
}
|
||||
}
|
||||
|
||||
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
|
||||
return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
|
||||
@@ -249,26 +229,7 @@ func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
|
||||
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
|
||||
|
||||
@@ -18,9 +18,7 @@ import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"io/fs"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -30,13 +28,9 @@ import (
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
const (
|
||||
// this is the media type for directories in AWS and Nextcloud
|
||||
DirContentType = "application/x-directory"
|
||||
DefaultContentType = "binary/octet-stream"
|
||||
|
||||
// this is the minimum allowed size for mp parts
|
||||
MinPartSize = 5 * 1024 * 1024
|
||||
var (
|
||||
// RFC3339TimeFormat RFC3339 time format
|
||||
RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
|
||||
)
|
||||
|
||||
func IsValidBucketName(name string) bool { return true }
|
||||
@@ -53,18 +47,8 @@ func (d ByObjectName) Len() int { return len(d) }
|
||||
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
||||
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
|
||||
|
||||
func GetPtrFromString(str string) *string {
|
||||
if str == "" {
|
||||
return nil
|
||||
}
|
||||
return &str
|
||||
}
|
||||
|
||||
func GetStringFromPtr(str *string) string {
|
||||
if str == nil {
|
||||
return ""
|
||||
}
|
||||
return *str
|
||||
func GetStringPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
func GetTimePtr(t time.Time) *time.Time {
|
||||
@@ -77,9 +61,9 @@ var (
|
||||
|
||||
// ParseRange parses input range header and returns startoffset, length, and
|
||||
// error. If no endoffset specified, then length is set to -1.
|
||||
func ParseRange(size int64, acceptRange string) (int64, int64, error) {
|
||||
func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
|
||||
if acceptRange == "" {
|
||||
return 0, size, nil
|
||||
return 0, fi.Size(), nil
|
||||
}
|
||||
|
||||
rangeKv := strings.Split(acceptRange, "=")
|
||||
@@ -115,38 +99,6 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
|
||||
return startOffset, endOffset - startOffset + 1, nil
|
||||
}
|
||||
|
||||
// ParseCopySource parses x-amz-copy-source header and returns source bucket,
|
||||
// source object, versionId, error respectively
|
||||
func ParseCopySource(copySourceHeader string) (string, string, string, error) {
|
||||
if copySourceHeader[0] == '/' {
|
||||
copySourceHeader = copySourceHeader[1:]
|
||||
}
|
||||
|
||||
var copySource, versionId string
|
||||
i := strings.LastIndex(copySourceHeader, "?versionId=")
|
||||
if i == -1 {
|
||||
copySource = copySourceHeader
|
||||
} else {
|
||||
copySource = copySourceHeader[:i]
|
||||
versionId = copySourceHeader[i+11:]
|
||||
}
|
||||
|
||||
srcBucket, srcObject, ok := strings.Cut(copySource, "/")
|
||||
if !ok {
|
||||
return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
|
||||
}
|
||||
|
||||
return srcBucket, srcObject, versionId, nil
|
||||
}
|
||||
|
||||
func CreateExceedingRangeErr(objSize int64) s3err.APIError {
|
||||
return s3err.APIError{
|
||||
Code: "InvalidArgument",
|
||||
Description: fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
|
||||
func GetMultipartMD5(parts []types.CompletedPart) string {
|
||||
var partsEtagBytes []byte
|
||||
for _, part := range parts {
|
||||
@@ -168,16 +120,3 @@ func md5String(data []byte) string {
|
||||
sum := md5.Sum(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
type FileSectionReadCloser struct {
|
||||
R io.Reader
|
||||
F *os.File
|
||||
}
|
||||
|
||||
func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
|
||||
return f.R.Read(p)
|
||||
}
|
||||
|
||||
func (f *FileSectionReadCloser) Close() error {
|
||||
return f.F.Close()
|
||||
}
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import "os"
|
||||
|
||||
// MetadataStorer defines the interface for managing metadata.
|
||||
// When object == "", the operation is on the bucket.
|
||||
type MetadataStorer interface {
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
// Returns the value of the attribute, or an error if the attribute does not exist.
|
||||
RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
// If attribute already exists, new attribute should replace existing.
|
||||
// Returns an error if the operation fails.
|
||||
StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
// Returns an error if the operation fails.
|
||||
DeleteAttribute(bucket, object, attribute string) error
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
// Returns list of attribute names, or an error if the operation fails.
|
||||
ListAttributes(bucket, object string) ([]string, error)
|
||||
|
||||
// DeleteAttributes removes all attributes for an object or a bucket.
|
||||
// Returns an error if the operation fails.
|
||||
DeleteAttributes(bucket, object string) error
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// NoMeta is a metadata storer that does not store metadata.
|
||||
// This can be useful for read only mounts where attempting to store metadata
|
||||
// would fail.
|
||||
type NoMeta struct{}
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
// always returns ErrNoSuchKey
|
||||
func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
// always returns nil without storing the attribute
|
||||
func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
// always returns nil without deleting the attribute
|
||||
func (NoMeta) DeleteAttribute(_, _, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
// always returns an empty list of attributes
|
||||
func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// DeleteAttributes removes all attributes for an object or a bucket.
|
||||
// always returns nil without deleting any attributes
|
||||
func (NoMeta) DeleteAttributes(bucket, object string) error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
// Copyright 2025 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// SideCar is a metadata storer that uses sidecar files to store metadata.
|
||||
type SideCar struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
const (
|
||||
sidecarmeta = "meta"
|
||||
)
|
||||
|
||||
// NewSideCar creates a new SideCar metadata storer.
|
||||
func NewSideCar(dir string) (SideCar, error) {
|
||||
fi, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return SideCar{}, fmt.Errorf("not a directory")
|
||||
}
|
||||
|
||||
return SideCar{dir: dir}, nil
|
||||
}
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
attr := filepath.Join(metadir, attribute)
|
||||
|
||||
value, err := os.ReadFile(attr)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read attribute: %v", err)
|
||||
}
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
err := os.MkdirAll(metadir, 0777)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create metadata directory: %v", err)
|
||||
}
|
||||
|
||||
attr := filepath.Join(metadir, attribute)
|
||||
err = os.WriteFile(attr, value, 0666)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write attribute: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
attr := filepath.Join(metadir, attribute)
|
||||
|
||||
err := os.Remove(attr)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return ErrNoSuchKey
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove attribute: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
|
||||
ents, err := os.ReadDir(metadir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return []string{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list attributes: %v", err)
|
||||
}
|
||||
|
||||
var attrs []string
|
||||
for _, ent := range ents {
|
||||
attrs = append(attrs, ent.Name())
|
||||
}
|
||||
|
||||
return attrs, nil
|
||||
}
|
||||
|
||||
// DeleteAttributes removes all attributes for an object or a bucket.
|
||||
func (s SideCar) DeleteAttributes(bucket, object string) error {
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
}
|
||||
|
||||
err := os.RemoveAll(metadir)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("failed to remove attributes: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package meta
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
)
|
||||
|
||||
const (
|
||||
xattrPrefix = "user."
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoSuchKey is returned when the key does not exist.
|
||||
ErrNoSuchKey = errors.New("no such key")
|
||||
)
|
||||
|
||||
type XattrMeta struct{}
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
|
||||
if f != nil {
|
||||
b, err := xattr.FGet(f, xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
|
||||
if f != nil {
|
||||
return xattr.FSet(f, xattrPrefix+attribute, value)
|
||||
}
|
||||
|
||||
return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
|
||||
}
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
|
||||
err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return ErrNoSuchKey
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteAttributes is not implemented for xattr since xattrs
|
||||
// are automatically removed when the file is deleted.
|
||||
func (x XattrMeta) DeleteAttributes(bucket, object string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAttributes lists all attributes for an object in a bucket.
|
||||
func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
|
||||
attrs, err := xattr.List(filepath.Join(bucket, object))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
attributes := make([]string, 0, len(attrs))
|
||||
for _, attr := range attrs {
|
||||
if !isUserAttr(attr) {
|
||||
continue
|
||||
}
|
||||
attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
|
||||
}
|
||||
return attributes, nil
|
||||
}
|
||||
|
||||
func isUserAttr(attr string) bool {
|
||||
return strings.HasPrefix(attr, xattrPrefix)
|
||||
}
|
||||
|
||||
// Test is a helper function to test if xattrs are supported.
|
||||
func (x XattrMeta) Test(path string) error {
|
||||
// check for platform support
|
||||
if !xattr.XATTR_SUPPORTED {
|
||||
return fmt.Errorf("xattrs are not supported on this platform")
|
||||
}
|
||||
|
||||
// check if the filesystem supports xattrs
|
||||
_, err := xattr.Get(path, "user.test")
|
||||
if errors.Is(err, syscall.ENOTSUP) {
|
||||
return fmt.Errorf("xattrs are not supported on this filesystem")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
// Copyright 2024 Versity Software
|
||||
|
||||
// MkdirAll borrowed from stdlib to add ability to set ownership
|
||||
// as directories are created
|
||||
@@ -15,6 +14,11 @@ import (
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultDirPerm fs.FileMode = 0755
|
||||
)
|
||||
|
||||
// MkdirAll is similar to os.MkdirAll but it will return
|
||||
// ErrObjectParentIsFile when appropriate
|
||||
// MkdirAll creates a directory named path,
|
||||
@@ -25,9 +29,9 @@ import (
|
||||
// Any newly created directory is set to provided uid/gid ownership.
|
||||
// If path is already a directory, MkdirAll does nothing
|
||||
// and returns nil.
|
||||
// Any directory created will be set to provided uid/gid ownership
|
||||
// Any directoy created will be set to provided uid/gid ownership
|
||||
// if doChown is true.
|
||||
func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
|
||||
func MkdirAll(path string, uid, gid int, doChown bool) error {
|
||||
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
|
||||
dir, err := os.Stat(path)
|
||||
if err == nil {
|
||||
@@ -50,14 +54,14 @@ func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) erro
|
||||
|
||||
if j > 1 {
|
||||
// Create parent.
|
||||
err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
|
||||
err = MkdirAll(path[:j-1], uid, gid, doChown)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Parent now exists; invoke Mkdir and use its result.
|
||||
err = os.Mkdir(path, dirPerm)
|
||||
err = os.Mkdir(path, defaultDirPerm)
|
||||
if err != nil {
|
||||
// Handle arguments like "foo/." by
|
||||
// double-checking that directory doesn't exist.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
24
backend/posix/with_enodata.go
Normal file
24
backend/posix/with_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !freebsd && !openbsd && !netbsd
|
||||
// +build !freebsd,!openbsd,!netbsd
|
||||
|
||||
package posix
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENODATA
|
||||
)
|
||||
@@ -43,7 +43,6 @@ type tmpfile struct {
|
||||
needsChown bool
|
||||
uid int
|
||||
gid int
|
||||
newDirPerm fs.FileMode
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -51,7 +50,7 @@ var (
|
||||
defaultFilePerm uint32 = 0644
|
||||
)
|
||||
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
|
||||
uid, gid, doChown := p.getChownIDs(acct)
|
||||
|
||||
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
|
||||
@@ -63,7 +62,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
|
||||
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
|
||||
if err != nil {
|
||||
// O_TMPFILE not supported, try fallback
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
@@ -82,7 +81,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
|
||||
gid: gid,
|
||||
}
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 && dofalloc {
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
@@ -109,11 +108,10 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
|
||||
needsChown: doChown,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
newDirPerm: p.newDirPerm,
|
||||
}
|
||||
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 && dofalloc {
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
@@ -136,9 +134,6 @@ func (tmp *tmpfile) falloc() error {
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// make sure this is cleaned up in all error cases
|
||||
defer tmp.f.Close()
|
||||
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
// with any other simultaneous uploads. The final operation is to move the
|
||||
@@ -153,7 +148,7 @@ func (tmp *tmpfile) link() error {
|
||||
|
||||
dir := filepath.Dir(objPath)
|
||||
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make parent dir: %w", err)
|
||||
}
|
||||
@@ -175,21 +170,11 @@ func (tmp *tmpfile) link() error {
|
||||
}
|
||||
defer dirf.Close()
|
||||
|
||||
for {
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
if errors.Is(err, syscall.EEXIST) {
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile (fd %q as %q): %w",
|
||||
filepath.Base(tmp.f.Name()), objPath, err)
|
||||
}
|
||||
break
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile (%q in %q): %w",
|
||||
filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
@@ -236,7 +221,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return tmp.f
|
||||
}
|
||||
|
||||
24
backend/posix/without_enodata.go
Normal file
24
backend/posix/without_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build freebsd || openbsd || netbsd
|
||||
// +build freebsd openbsd netbsd
|
||||
|
||||
package posix
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENOATTR
|
||||
)
|
||||
@@ -36,12 +36,12 @@ type tmpfile struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
|
||||
uid, gid, doChown := p.getChownIDs(acct)
|
||||
|
||||
// Create a temp file for upload while in progress (see link comments below).
|
||||
var err error
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
@@ -112,7 +112,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return tmp.f
|
||||
}
|
||||
|
||||
@@ -33,12 +33,6 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = &s.endpoint
|
||||
}), nil
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(cfg), nil
|
||||
}
|
||||
|
||||
@@ -56,6 +50,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(s))
|
||||
}
|
||||
|
||||
if s.disableChecksum {
|
||||
opts = append(opts,
|
||||
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
|
||||
@@ -68,3 +67,13 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
|
||||
|
||||
return config.LoadDefaultConfig(ctx, opts...)
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
SigningRegion: s.awsRegion,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
@@ -55,8 +54,6 @@ type S3Proxy struct {
|
||||
debug bool
|
||||
}
|
||||
|
||||
var _ backend.Backend = &S3Proxy{}
|
||||
|
||||
func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
|
||||
s := &S3Proxy{
|
||||
access: access,
|
||||
@@ -75,12 +72,8 @@ func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{
|
||||
ContinuationToken: &input.ContinuationToken,
|
||||
MaxBuckets: &input.MaxBuckets,
|
||||
Prefix: &input.Prefix,
|
||||
})
|
||||
func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
return s3response.ListAllMyBucketsResult{}, handleError(err)
|
||||
}
|
||||
@@ -100,8 +93,6 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, input s3response.ListBucketsI
|
||||
Buckets: s3response.ListAllMyBucketsList{
|
||||
Bucket: buckets,
|
||||
},
|
||||
ContinuationToken: backend.GetStringFromPtr(output.ContinuationToken),
|
||||
Prefix: backend.GetStringFromPtr(output.Prefix),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -118,8 +109,8 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
|
||||
|
||||
var tagSet []types.Tag
|
||||
tagSet = append(tagSet, types.Tag{
|
||||
Key: backend.GetPtrFromString(aclKey),
|
||||
Value: backend.GetPtrFromString(base64Encode(acl)),
|
||||
Key: backend.GetStringPtr(aclKey),
|
||||
Value: backend.GetStringPtr(base64Encode(acl)),
|
||||
})
|
||||
|
||||
_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
|
||||
@@ -131,100 +122,14 @@ func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
|
||||
_, err := s.client.DeleteBucket(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
|
||||
_, err := s.client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
|
||||
Bucket: &bucket,
|
||||
OwnershipControls: &types.OwnershipControls{
|
||||
Rules: []types.OwnershipControlsRule{
|
||||
{
|
||||
ObjectOwnership: ownership,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
var ownship types.ObjectOwnership
|
||||
resp, err := s.client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return ownship, handleError(err)
|
||||
}
|
||||
return resp.OwnershipControls.Rules[0].ObjectOwnership, nil
|
||||
}
|
||||
func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
|
||||
_, err := s.client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
|
||||
_, err := s.client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
|
||||
Bucket: &bucket,
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: status,
|
||||
},
|
||||
})
|
||||
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
out, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
|
||||
return s3response.GetBucketVersioningOutput{
|
||||
Status: &out.Status,
|
||||
MFADelete: &out.MFADelete,
|
||||
}, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
|
||||
out, err := s.client.ListObjectVersions(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.ListVersionsResult{}, handleError(err)
|
||||
}
|
||||
|
||||
return s3response.ListVersionsResult{
|
||||
CommonPrefixes: out.CommonPrefixes,
|
||||
DeleteMarkers: out.DeleteMarkers,
|
||||
Delimiter: out.Delimiter,
|
||||
EncodingType: out.EncodingType,
|
||||
IsTruncated: out.IsTruncated,
|
||||
KeyMarker: out.KeyMarker,
|
||||
MaxKeys: out.MaxKeys,
|
||||
Name: out.Name,
|
||||
NextKeyMarker: out.NextKeyMarker,
|
||||
NextVersionIdMarker: out.NextVersionIdMarker,
|
||||
Prefix: out.Prefix,
|
||||
VersionIdMarker: input.VersionIdMarker,
|
||||
Versions: out.Versions,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
|
||||
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
out, err := s.client.CreateMultipartUpload(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.InitiateMultipartUploadResult{}, handleError(err)
|
||||
}
|
||||
|
||||
return s3response.InitiateMultipartUploadResult{
|
||||
Bucket: *out.Bucket,
|
||||
Key: *out.Key,
|
||||
UploadId: *out.UploadId,
|
||||
}, nil
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
@@ -256,8 +161,8 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
|
||||
ID: *u.Owner.ID,
|
||||
DisplayName: *u.Owner.DisplayName,
|
||||
},
|
||||
StorageClass: u.StorageClass,
|
||||
Initiated: *u.Initiated,
|
||||
StorageClass: string(u.StorageClass),
|
||||
Initiated: u.Initiated.Format(backend.RFC3339TimeFormat),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -294,7 +199,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
|
||||
for _, p := range output.Parts {
|
||||
parts = append(parts, s3response.Part{
|
||||
PartNumber: int(*p.PartNumber),
|
||||
LastModified: *p.LastModified,
|
||||
LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
|
||||
ETag: *p.ETag,
|
||||
Size: *p.Size,
|
||||
})
|
||||
@@ -323,7 +228,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
|
||||
ID: *output.Owner.ID,
|
||||
DisplayName: *output.Owner.DisplayName,
|
||||
},
|
||||
StorageClass: output.StorageClass,
|
||||
StorageClass: string(output.StorageClass),
|
||||
PartNumberMarker: pnm,
|
||||
NextPartNumberMarker: npmn,
|
||||
MaxParts: int(*output.MaxParts),
|
||||
@@ -357,25 +262,17 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
|
||||
// streaming backend is not seekable,
|
||||
// use unsigned payload for streaming ops
|
||||
output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
|
||||
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
|
||||
))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, handleError(err)
|
||||
return "", handleError(err)
|
||||
}
|
||||
|
||||
var versionID string
|
||||
if output.VersionId != nil {
|
||||
versionID = *output.VersionId
|
||||
}
|
||||
|
||||
return s3response.PutObjectOutput{
|
||||
ETag: *output.ETag,
|
||||
VersionID: versionID,
|
||||
}, nil
|
||||
return *output.ETag, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
@@ -383,49 +280,24 @@ func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
|
||||
output, err := s.client.GetObject(ctx, input)
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
defer output.Body.Close()
|
||||
|
||||
_, err = io.Copy(w, output.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
|
||||
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
out, err := s.client.GetObjectAttributes(ctx, input)
|
||||
|
||||
parts := s3response.ObjectParts{}
|
||||
objParts := out.ObjectParts
|
||||
if objParts != nil {
|
||||
if objParts.PartNumberMarker != nil {
|
||||
partNumberMarker, err := strconv.Atoi(*objParts.PartNumberMarker)
|
||||
if err != nil {
|
||||
parts.PartNumberMarker = partNumberMarker
|
||||
}
|
||||
if objParts.NextPartNumberMarker != nil {
|
||||
nextPartNumberMarker, err := strconv.Atoi(*objParts.NextPartNumberMarker)
|
||||
if err != nil {
|
||||
parts.NextPartNumberMarker = nextPartNumberMarker
|
||||
}
|
||||
}
|
||||
if objParts.IsTruncated != nil {
|
||||
parts.IsTruncated = *objParts.IsTruncated
|
||||
}
|
||||
if objParts.MaxParts != nil {
|
||||
parts.MaxParts = int(*objParts.MaxParts)
|
||||
}
|
||||
parts.Parts = objParts.Parts
|
||||
}
|
||||
}
|
||||
|
||||
return s3response.GetObjectAttributesResponse{
|
||||
ETag: out.ETag,
|
||||
LastModified: out.LastModified,
|
||||
ObjectSize: out.ObjectSize,
|
||||
StorageClass: out.StorageClass,
|
||||
ObjectParts: &parts,
|
||||
}, handleError(err)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
@@ -433,52 +305,19 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
out, err := s.client.ListObjects(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.ListObjectsResult{}, handleError(err)
|
||||
}
|
||||
|
||||
contents := convertObjects(out.Contents)
|
||||
|
||||
return s3response.ListObjectsResult{
|
||||
CommonPrefixes: out.CommonPrefixes,
|
||||
Contents: contents,
|
||||
Delimiter: out.Delimiter,
|
||||
IsTruncated: out.IsTruncated,
|
||||
Marker: out.Marker,
|
||||
MaxKeys: out.MaxKeys,
|
||||
Name: out.Name,
|
||||
NextMarker: out.NextMarker,
|
||||
Prefix: out.Prefix,
|
||||
}, nil
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
|
||||
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
out, err := s.client.ListObjectsV2(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.ListObjectsV2Result{}, handleError(err)
|
||||
}
|
||||
|
||||
contents := convertObjects(out.Contents)
|
||||
|
||||
return s3response.ListObjectsV2Result{
|
||||
CommonPrefixes: out.CommonPrefixes,
|
||||
Contents: contents,
|
||||
Delimiter: out.Delimiter,
|
||||
IsTruncated: out.IsTruncated,
|
||||
ContinuationToken: out.ContinuationToken,
|
||||
MaxKeys: out.MaxKeys,
|
||||
Name: out.Name,
|
||||
NextContinuationToken: out.NextContinuationToken,
|
||||
Prefix: out.Prefix,
|
||||
KeyCount: out.KeyCount,
|
||||
}, nil
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
res, err := s.client.DeleteObject(ctx, input)
|
||||
return res, handleError(err)
|
||||
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
|
||||
_, err := s.client.DeleteObject(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
@@ -530,8 +369,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
|
||||
for i, tag := range tagout.TagSet {
|
||||
if *tag.Key == aclKey {
|
||||
tagout.TagSet[i] = types.Tag{
|
||||
Key: backend.GetPtrFromString(aclKey),
|
||||
Value: backend.GetPtrFromString(base64Encode(data)),
|
||||
Key: backend.GetStringPtr(aclKey),
|
||||
Value: backend.GetStringPtr(base64Encode(data)),
|
||||
}
|
||||
found = true
|
||||
break
|
||||
@@ -539,8 +378,8 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
|
||||
}
|
||||
if !found {
|
||||
tagout.TagSet = append(tagout.TagSet, types.Tag{
|
||||
Key: backend.GetPtrFromString(aclKey),
|
||||
Value: backend.GetPtrFromString(base64Encode(data)),
|
||||
Key: backend.GetStringPtr(aclKey),
|
||||
Value: backend.GetStringPtr(base64Encode(data)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -597,135 +436,8 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
|
||||
_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
|
||||
Bucket: &bucket,
|
||||
Policy: backend.GetPtrFromString(string(policy)),
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
|
||||
policy, err := s.client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
result := []byte{}
|
||||
if policy.Policy != nil {
|
||||
result = []byte(*policy.Policy)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
||||
_, err := s.client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
|
||||
cfg, err := auth.ParseBucketLockConfigurationOutput(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{
|
||||
Bucket: &bucket,
|
||||
ObjectLockConfiguration: cfg,
|
||||
})
|
||||
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
|
||||
resp, err := s.client.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
config := auth.BucketLockConfig{
|
||||
Enabled: resp.ObjectLockConfiguration.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
|
||||
DefaultRetention: resp.ObjectLockConfiguration.Rule.DefaultRetention,
|
||||
}
|
||||
|
||||
return json.Marshal(config)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
|
||||
ret, err := auth.ParseObjectLockRetentionOutput(retention)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
Retention: ret,
|
||||
BypassGovernanceRetention: &bypass,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
resp, err := s.client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
return json.Marshal(resp.Retention)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
|
||||
var st types.ObjectLockLegalHoldStatus
|
||||
if status {
|
||||
st = types.ObjectLockLegalHoldStatusOn
|
||||
} else {
|
||||
st = types.ObjectLockLegalHoldStatusOff
|
||||
}
|
||||
|
||||
_, err := s.client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: st,
|
||||
},
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
resp, err := s.client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
status := resp.LegalHold.Status == types.ObjectLockLegalHoldStatusOn
|
||||
return &status, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
|
||||
var acll auth.ACL
|
||||
if err := json.Unmarshal(acl, &acll); err != nil {
|
||||
return fmt.Errorf("unmarshal acl: %w", err)
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, acll.Owner), nil)
|
||||
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
@@ -755,7 +467,7 @@ func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []by
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return fmt.Errorf("%v", string(body))
|
||||
return fmt.Errorf(string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -831,21 +543,3 @@ func base64Decode(encoded string) ([]byte, error) {
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
|
||||
func convertObjects(objs []types.Object) []s3response.Object {
|
||||
result := make([]s3response.Object, len(objs))
|
||||
|
||||
for _, obj := range objs {
|
||||
result = append(result, s3response.Object{
|
||||
ETag: obj.ETag,
|
||||
Key: obj.Key,
|
||||
LastModified: obj.LastModified,
|
||||
Owner: obj.Owner,
|
||||
Size: obj.Size,
|
||||
RestoreStatus: obj.RestoreStatus,
|
||||
StorageClass: obj.StorageClass,
|
||||
})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -26,25 +26,21 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type ScoutfsOpts struct {
|
||||
ChownUID bool
|
||||
ChownGID bool
|
||||
ReadOnly bool
|
||||
GlacierMode bool
|
||||
BucketLinks bool
|
||||
NewDirPerm fs.FileMode
|
||||
}
|
||||
|
||||
type ScoutFS struct {
|
||||
@@ -52,9 +48,6 @@ type ScoutFS struct {
|
||||
rootfd *os.File
|
||||
rootdir string
|
||||
|
||||
// bucket/object metadata storage facility
|
||||
meta meta.MetadataStorer
|
||||
|
||||
// glaciermode enables the following behavior:
|
||||
// GET object: if file offline, return invalid object state
|
||||
// HEAD object: if file offline, set obj storage class to GLACIER
|
||||
@@ -71,13 +64,13 @@ type ScoutFS struct {
|
||||
chownuid bool
|
||||
chowngid bool
|
||||
|
||||
// read only mode prevents any backend modifications
|
||||
readonly bool
|
||||
|
||||
// euid/egid are the effective uid/gid of the running versitygw process
|
||||
// used to determine if chowning is needed
|
||||
euid int
|
||||
egid int
|
||||
|
||||
// newDirPerm is the permissions to use when creating new directories
|
||||
newDirPerm fs.FileMode
|
||||
}
|
||||
|
||||
var _ backend.Backend = &ScoutFS{}
|
||||
@@ -86,13 +79,8 @@ const (
|
||||
metaTmpDir = ".sgwtmp"
|
||||
metaTmpMultipartDir = metaTmpDir + "/multipart"
|
||||
tagHdr = "X-Amz-Tagging"
|
||||
metaHdr = "X-Amz-Meta"
|
||||
contentTypeHdr = "content-type"
|
||||
contentEncHdr = "content-encoding"
|
||||
emptyMD5 = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
etagkey = "etag"
|
||||
objectRetentionKey = "object-retention"
|
||||
objectLegalHoldKey = "object-legal-hold"
|
||||
etagkey = "user.etag"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -103,12 +91,11 @@ var (
|
||||
|
||||
const (
|
||||
// ScoutFS special xattr types
|
||||
|
||||
systemPrefix = "scoutfs.hide."
|
||||
onameAttr = systemPrefix + "objname"
|
||||
flagskey = systemPrefix + "sam_flags"
|
||||
stagecopykey = systemPrefix + "sam_stagereq"
|
||||
|
||||
fsBlocksize = 4096
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -160,6 +147,10 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
|
||||
// ioctl to not have to read and copy the part data to the final object. This
|
||||
// saves a read and write cycle for all mutlipart uploads.
|
||||
func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
if s.readonly {
|
||||
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
acct, ok := ctx.Value("account").(auth.Account)
|
||||
if !ok {
|
||||
acct = auth.Account{}
|
||||
@@ -196,20 +187,18 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
// check all parts ok
|
||||
last := len(parts) - 1
|
||||
partsize := int64(0)
|
||||
var totalsize int64
|
||||
for i, part := range parts {
|
||||
if part.PartNumber == nil || *part.PartNumber < 1 {
|
||||
for i, p := range parts {
|
||||
if p.PartNumber == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
|
||||
fullPartPath := filepath.Join(bucket, partObjPath)
|
||||
fi, err := os.Lstat(fullPartPath)
|
||||
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
|
||||
fi, err := os.Lstat(partPath)
|
||||
if err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
@@ -217,25 +206,23 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
|
||||
if i == 0 {
|
||||
partsize = fi.Size()
|
||||
}
|
||||
|
||||
// partsize must be a multiple of the filesystem blocksize
|
||||
// except for last part
|
||||
if i < last && partsize%fsBlocksize != 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
totalsize += fi.Size()
|
||||
// all parts except the last need to be the same size
|
||||
if i < last && partsize != fi.Size() {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
// non-last part sizes need to be multiples of 4k for move blocks
|
||||
// TODO: fallback to no move blocks if not 4k aligned?
|
||||
if i == 0 && i < last && fi.Size()%4096 != 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
b, err := s.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)
|
||||
b, err := xattr.Get(partPath, "user.etag")
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
if parts[i].ETag == nil || etag != *parts[i].ETag {
|
||||
if etag != *parts[i].ETag {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
}
|
||||
@@ -251,106 +238,59 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
|
||||
}
|
||||
defer f.cleanup()
|
||||
|
||||
for _, part := range parts {
|
||||
if part.PartNumber == nil || *part.PartNumber < 1 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
|
||||
fullPartPath := filepath.Join(bucket, partObjPath)
|
||||
pf, err := os.Open(fullPartPath)
|
||||
for _, p := range parts {
|
||||
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
|
||||
return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
|
||||
}
|
||||
|
||||
// scoutfs move data is a metadata only operation that moves the data
|
||||
// extent references from the source, appeding to the destination.
|
||||
// this needs to be 4k aligned.
|
||||
err = moveData(pf, f.File())
|
||||
err = moveData(pf, f.f)
|
||||
pf.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
|
||||
return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
upiddir := filepath.Join(objdir, uploadID)
|
||||
cType, _ := s.loadUserMetaData(bucket, upiddir, userMetaData)
|
||||
loadUserMetaData(upiddir, userMetaData)
|
||||
|
||||
objname := filepath.Join(bucket, object)
|
||||
dir := filepath.Dir(objname)
|
||||
if dir != "" {
|
||||
uid, gid, doChown := s.getChownIDs(acct)
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown, s.newDirPerm)
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
err = f.link()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("link object in namespace: %w", err)
|
||||
}
|
||||
|
||||
for k, v := range userMetaData {
|
||||
err = s.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
|
||||
err = xattr.Set(objname, "user."+k, []byte(v))
|
||||
if err != nil {
|
||||
// cleanup object if returning error
|
||||
os.Remove(objname)
|
||||
return nil, fmt.Errorf("set user attr %q: %w", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
// load and set tagging
|
||||
tagging, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
return nil, fmt.Errorf("get object tagging: %w", err)
|
||||
}
|
||||
if err == nil {
|
||||
err := s.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set object tagging: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// set content-type
|
||||
if cType != "" {
|
||||
err := s.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set object content type: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// load and set legal hold
|
||||
lHold, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey)
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
return nil, fmt.Errorf("get object legal hold: %w", err)
|
||||
}
|
||||
if err == nil {
|
||||
err := s.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set object legal hold: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// load and set retention
|
||||
ret, err := s.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey)
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
return nil, fmt.Errorf("get object retention: %w", err)
|
||||
}
|
||||
if err == nil {
|
||||
err := s.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set object retention: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate s3 compatible md5sum for complete multipart.
|
||||
s3MD5 := backend.GetMultipartMD5(parts)
|
||||
|
||||
err = s.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(s3MD5))
|
||||
err = xattr.Set(objname, "user.etag", []byte(s3MD5))
|
||||
if err != nil {
|
||||
// cleanup object if returning error
|
||||
os.Remove(objname)
|
||||
return nil, fmt.Errorf("set etag attr: %w", err)
|
||||
}
|
||||
|
||||
err = f.link()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("link object in namespace: %w", err)
|
||||
}
|
||||
|
||||
// cleanup tmp dirs
|
||||
os.RemoveAll(upiddir)
|
||||
// use Remove for objdir in case there are still other uploads
|
||||
@@ -378,107 +318,61 @@ func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte
|
||||
return sum, nil
|
||||
}
|
||||
|
||||
// fll out the user metadata map with the metadata for the object
|
||||
// and return the content type and encoding
|
||||
func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
|
||||
ents, err := s.meta.ListAttributes(bucket, object)
|
||||
func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
|
||||
ents, err := xattr.List(path)
|
||||
if err != nil || len(ents) == 0 {
|
||||
return "", ""
|
||||
return
|
||||
}
|
||||
for _, e := range ents {
|
||||
if !isValidMeta(e) {
|
||||
continue
|
||||
}
|
||||
b, err := s.meta.RetrieveAttribute(nil, bucket, object, e)
|
||||
b, err := xattr.Get(path, e)
|
||||
if err == errNoData {
|
||||
m[strings.TrimPrefix(e, "user.")] = ""
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if b == nil {
|
||||
m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = ""
|
||||
continue
|
||||
}
|
||||
m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
|
||||
m[strings.TrimPrefix(e, "user.")] = string(b)
|
||||
}
|
||||
|
||||
var contentType, contentEncoding string
|
||||
b, _ := s.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
|
||||
b, err := xattr.Get(path, "user.content-type")
|
||||
contentType = string(b)
|
||||
if err != nil {
|
||||
contentType = ""
|
||||
}
|
||||
if contentType != "" {
|
||||
m[contentTypeHdr] = contentType
|
||||
m["content-type"] = contentType
|
||||
}
|
||||
|
||||
b, _ = s.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
|
||||
b, err = xattr.Get(path, "user.content-encoding")
|
||||
contentEncoding = string(b)
|
||||
if err != nil {
|
||||
contentEncoding = ""
|
||||
}
|
||||
if contentEncoding != "" {
|
||||
m[contentEncHdr] = contentEncoding
|
||||
m["content-encoding"] = contentEncoding
|
||||
}
|
||||
|
||||
return contentType, contentEncoding
|
||||
return
|
||||
}
|
||||
|
||||
func isValidMeta(val string) bool {
|
||||
if strings.HasPrefix(val, metaHdr) {
|
||||
if strings.HasPrefix(val, "user.X-Amz-Meta") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(val, "Expires") {
|
||||
if strings.EqualFold(val, "user.Expires") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
|
||||
if input.PartNumber != nil {
|
||||
uploadId, sum, err := s.retrieveUploadId(bucket, object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read parts: %w", err)
|
||||
}
|
||||
|
||||
partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
|
||||
|
||||
part, err := os.Stat(filepath.Join(bucket, partPath))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat part: %w", err)
|
||||
}
|
||||
|
||||
b, err := s.meta.RetrieveAttribute(nil, bucket, partPath, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
partsCount := int32(len(ents))
|
||||
size := part.Size()
|
||||
|
||||
return &s3.HeadObjectOutput{
|
||||
LastModified: backend.GetTimePtr(part.ModTime()),
|
||||
ETag: &etag,
|
||||
PartsCount: &partsCount,
|
||||
ContentLength: &size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
@@ -488,30 +382,18 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
|
||||
}
|
||||
|
||||
objPath := filepath.Join(bucket, object)
|
||||
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat object: %w", err)
|
||||
}
|
||||
if strings.HasSuffix(object, "/") && !fi.IsDir() {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
|
||||
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
|
||||
|
||||
if fi.IsDir() {
|
||||
// this is the media type for directories in AWS and Nextcloud
|
||||
contentType = "application/x-directory"
|
||||
}
|
||||
|
||||
b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
|
||||
b, err := xattr.Get(objPath, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
@@ -550,55 +432,19 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
|
||||
|
||||
contentLength := fi.Size()
|
||||
|
||||
var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
|
||||
status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, *input.VersionId)
|
||||
if err == nil {
|
||||
if *status {
|
||||
objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
|
||||
} else {
|
||||
objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
|
||||
}
|
||||
}
|
||||
|
||||
var objectLockMode types.ObjectLockMode
|
||||
var objectLockRetainUntilDate *time.Time
|
||||
retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, *input.VersionId)
|
||||
if err == nil {
|
||||
var config types.ObjectLockRetention
|
||||
if err := json.Unmarshal(retention, &config); err == nil {
|
||||
objectLockMode = types.ObjectLockMode(config.Mode)
|
||||
objectLockRetainUntilDate = config.RetainUntilDate
|
||||
}
|
||||
}
|
||||
|
||||
return &s3.HeadObjectOutput{
|
||||
ContentLength: &contentLength,
|
||||
ContentType: &contentType,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
StorageClass: stclass,
|
||||
Restore: &requestOngoing,
|
||||
ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
|
||||
ObjectLockMode: objectLockMode,
|
||||
ObjectLockRetainUntilDate: objectLockRetainUntilDate,
|
||||
ContentLength: &contentLength,
|
||||
ContentType: &contentType,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
StorageClass: stclass,
|
||||
Restore: &requestOngoing,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
|
||||
sum := sha256.Sum256([]byte(object))
|
||||
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
entries, err := os.ReadDir(objdir)
|
||||
if err != nil || len(entries) == 0 {
|
||||
return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
return entries[0].Name(), sum, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
acceptRange := *input.Range
|
||||
@@ -612,23 +458,15 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
}
|
||||
|
||||
objPath := filepath.Join(bucket, object)
|
||||
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat object: %w", err)
|
||||
}
|
||||
|
||||
if strings.HasSuffix(object, "/") && !fi.IsDir() {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
|
||||
startOffset, length, err := backend.ParseRange(fi, acceptRange)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -675,14 +513,19 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open object: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
rdr := io.NewSectionReader(f, startOffset, length)
|
||||
_, err = io.Copy(writer, rdr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy data: %w", err)
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
|
||||
contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
|
||||
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
|
||||
|
||||
b, err := s.meta.RetrieveAttribute(nil, bucket, object, etagkey)
|
||||
b, err := xattr.Get(objPath, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
@@ -706,7 +549,6 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
TagCount: &tagCount,
|
||||
StorageClass: types.StorageClassStandard,
|
||||
ContentRange: &contentRange,
|
||||
Body: &backend.FileSectionReadCloser{R: rdr, F: f},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -731,9 +573,9 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
@@ -755,20 +597,20 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err)
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(bucket)
|
||||
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
|
||||
s.fileToObj(bucket), []string{metaTmpDir})
|
||||
if err != nil {
|
||||
return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
return nil, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
}
|
||||
|
||||
return s3response.ListObjectsResult{
|
||||
return &s3.ListObjectsOutput{
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
@@ -781,9 +623,9 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
|
||||
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
if input.Bucket == nil {
|
||||
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
@@ -805,20 +647,20 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err)
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(bucket)
|
||||
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys),
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
|
||||
s.fileToObj(bucket), []string{metaTmpDir})
|
||||
if err != nil {
|
||||
return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
return nil, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
}
|
||||
|
||||
return s3response.ListObjectsV2Result{
|
||||
return &s3.ListObjectsV2Output{
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
@@ -832,58 +674,53 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
|
||||
}
|
||||
|
||||
func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
|
||||
return func(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
return func(path string, d fs.DirEntry) (types.Object, error) {
|
||||
objPath := filepath.Join(bucket, path)
|
||||
if d.IsDir() {
|
||||
// directory object only happens if directory empty
|
||||
// check to see if this is a directory object by checking etag
|
||||
etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
|
||||
if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.Object{}, backend.ErrSkipObj
|
||||
etagBytes, err := xattr.Get(objPath, etagkey)
|
||||
if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
return types.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
}
|
||||
etag := string(etagBytes)
|
||||
|
||||
fi, err := d.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.Object{}, backend.ErrSkipObj
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
key := path + "/"
|
||||
mtime := fi.ModTime()
|
||||
|
||||
return s3response.Object{
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &key,
|
||||
LastModified: &mtime,
|
||||
StorageClass: types.ObjectStorageClassStandard,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// file object, get object info and fill out object data
|
||||
b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey)
|
||||
etagBytes, err := xattr.Get(objPath, etagkey)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.Object{}, backend.ErrSkipObj
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
return s3response.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
if err != nil && !isNoAttr(err) {
|
||||
return types.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
}
|
||||
// note: meta.ErrNoSuchKey will return etagBytes = []byte{}
|
||||
// so this will just set etag to "" if its not already set
|
||||
|
||||
etag := string(b)
|
||||
etag := string(etagBytes)
|
||||
|
||||
fi, err := d.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.Object{}, backend.ErrSkipObj
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
sc := types.ObjectStorageClassStandard
|
||||
@@ -892,10 +729,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
|
||||
// If so, we will return the InvalidObjectState error.
|
||||
st, err := statMore(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.Object{}, backend.ErrSkipObj
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("stat more: %w", err)
|
||||
return types.Object{}, fmt.Errorf("stat more: %w", err)
|
||||
}
|
||||
if st.Offline_blocks != 0 {
|
||||
sc = types.ObjectStorageClassGlacier
|
||||
@@ -903,12 +740,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
mtime := fi.ModTime()
|
||||
|
||||
return s3response.Object{
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: &mtime,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
StorageClass: sc,
|
||||
}, nil
|
||||
@@ -991,9 +827,15 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
|
||||
}
|
||||
|
||||
func isNoAttr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
xerr, ok := err.(*xattr.Error)
|
||||
if ok && xerr.Err == xattr.ENOATTR {
|
||||
return true
|
||||
}
|
||||
if err == errNoData {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -23,24 +23,21 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/versity/scoutfs-go"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
metastore := meta.XattrMeta{}
|
||||
|
||||
p, err := posix.New(rootdir, metastore, posix.PosixOpts{
|
||||
ChownUID: opts.ChownUID,
|
||||
ChownGID: opts.ChownGID,
|
||||
BucketLinks: opts.BucketLinks,
|
||||
NewDirPerm: opts.NewDirPerm,
|
||||
p, err := posix.New(rootdir, posix.PosixOpts{
|
||||
ChownUID: opts.ChownUID,
|
||||
ChownGID: opts.ChownGID,
|
||||
ReadOnly: opts.ReadOnly,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -52,14 +49,12 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
}
|
||||
|
||||
return &ScoutFS{
|
||||
Posix: p,
|
||||
rootfd: f,
|
||||
rootdir: rootdir,
|
||||
meta: metastore,
|
||||
chownuid: opts.ChownUID,
|
||||
chowngid: opts.ChownGID,
|
||||
glaciermode: opts.GlacierMode,
|
||||
newDirPerm: opts.NewDirPerm,
|
||||
Posix: p,
|
||||
rootfd: f,
|
||||
rootdir: rootdir,
|
||||
chownuid: opts.ChownUID,
|
||||
chowngid: opts.ChownGID,
|
||||
readonly: opts.ReadOnly,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -73,10 +68,10 @@ type tmpfile struct {
|
||||
needsChown bool
|
||||
uid int
|
||||
gid int
|
||||
newDirPerm fs.FileMode
|
||||
}
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultFilePerm uint32 = 0644
|
||||
)
|
||||
|
||||
@@ -104,7 +99,11 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
|
||||
needsChown: doChown,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
newDirPerm: s.newDirPerm,
|
||||
}
|
||||
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
if doChown {
|
||||
@@ -117,6 +116,14 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) falloc() error {
|
||||
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fallocate: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
@@ -132,7 +139,7 @@ func (tmp *tmpfile) link() error {
|
||||
|
||||
dir := filepath.Dir(objPath)
|
||||
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make parent dir: %w", err)
|
||||
}
|
||||
@@ -177,10 +184,6 @@ func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return tmp.f
|
||||
}
|
||||
|
||||
func moveData(from *os.File, to *os.File) error {
|
||||
return scoutfs.MoveData(from, to)
|
||||
}
|
||||
|
||||
@@ -28,7 +28,9 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
return nil, fmt.Errorf("scoutfs only available on linux")
|
||||
}
|
||||
|
||||
type tmpfile struct{}
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
var (
|
||||
errNotSupported = errors.New("not supported")
|
||||
@@ -40,6 +42,7 @@ func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile
|
||||
_ = s.chowngid
|
||||
_ = s.euid
|
||||
_ = s.egid
|
||||
_ = s.readonly
|
||||
return nil, errNotSupported
|
||||
}
|
||||
|
||||
@@ -54,10 +57,6 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) File() *os.File {
|
||||
return nil
|
||||
}
|
||||
|
||||
func moveData(_, _ *os.File) error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
24
backend/scoutfs/with_enodata.go
Normal file
24
backend/scoutfs/with_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !freebsd && !openbsd && !netbsd
|
||||
// +build !freebsd,!openbsd,!netbsd
|
||||
|
||||
package scoutfs
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENODATA
|
||||
)
|
||||
24
backend/scoutfs/without_enodata.go
Normal file
24
backend/scoutfs/without_enodata.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build freebsd || openbsd || netbsd
|
||||
// +build freebsd openbsd netbsd
|
||||
|
||||
package scoutfs
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENOATTR
|
||||
)
|
||||
329
backend/walk.go
329
backend/walk.go
@@ -15,34 +15,32 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type WalkResults struct {
|
||||
CommonPrefixes []types.CommonPrefix
|
||||
Objects []s3response.Object
|
||||
Objects []types.Object
|
||||
Truncated bool
|
||||
NextMarker string
|
||||
}
|
||||
|
||||
type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
|
||||
type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
|
||||
|
||||
var ErrSkipObj = errors.New("skip this object")
|
||||
|
||||
// Walk walks the supplied fs.FS and returns results compatible with list
|
||||
// objects responses
|
||||
func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
|
||||
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
|
||||
cpmap := make(map[string]struct{})
|
||||
var objects []s3response.Object
|
||||
var objects []types.Object
|
||||
|
||||
var pastMarker bool
|
||||
if marker == "" {
|
||||
@@ -53,21 +51,10 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
var newMarker string
|
||||
var truncated bool
|
||||
|
||||
root := "."
|
||||
if strings.Contains(prefix, "/") {
|
||||
idx := strings.LastIndex(prefix, "/")
|
||||
if idx > 0 {
|
||||
root = prefix[:idx]
|
||||
}
|
||||
}
|
||||
|
||||
err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
|
||||
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
// Ignore the root directory
|
||||
if path == "." {
|
||||
return nil
|
||||
@@ -84,9 +71,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
// After this point, return skipflag instead of nil
|
||||
// so we can skip a directory without an early return
|
||||
var skipflag error
|
||||
if d.IsDir() {
|
||||
// If prefix is defined and the directory does not match prefix,
|
||||
// do not descend into the directory because nothing will
|
||||
@@ -96,57 +80,44 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
// building to match. So only skip if path isn't a prefix of prefix
|
||||
// and prefix isn't a prefix of path.
|
||||
if prefix != "" &&
|
||||
!strings.HasPrefix(path+"/", prefix) &&
|
||||
!strings.HasPrefix(prefix, path+"/") {
|
||||
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
|
||||
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// Don't recurse into subdirectories which contain the delimiter
|
||||
// after reaching the prefix
|
||||
if delimiter != "" &&
|
||||
strings.HasPrefix(path+"/", prefix) &&
|
||||
strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
|
||||
skipflag = fs.SkipDir
|
||||
} else {
|
||||
// TODO: can we do better here rather than a second readdir
|
||||
// per directory?
|
||||
ents, err := fs.ReadDir(fileSystem, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("readdir %q: %w", path, err)
|
||||
}
|
||||
if len(ents) == 0 && delimiter == "" {
|
||||
dirobj, err := getObj(path+"/", d)
|
||||
if err == ErrSkipObj {
|
||||
return skipflag
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, dirobj)
|
||||
|
||||
return skipflag
|
||||
}
|
||||
|
||||
if len(ents) != 0 {
|
||||
return skipflag
|
||||
}
|
||||
// TODO: can we do better here rather than a second readdir
|
||||
// per directory?
|
||||
ents, err := fs.ReadDir(fileSystem, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("readdir %q: %w", path, err)
|
||||
}
|
||||
path += "/"
|
||||
if len(ents) == 0 {
|
||||
dirobj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, dirobj)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if !pastMarker {
|
||||
if path == marker {
|
||||
pastMarker = true
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
if path < marker {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// If object doesn't have prefix, don't include in results.
|
||||
if prefix != "" && !strings.HasPrefix(path, prefix) {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
|
||||
if delimiter == "" {
|
||||
@@ -154,7 +125,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
// prefix are included in results
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
@@ -165,7 +136,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
|
||||
// Since delimiter is specified, we only want results that
|
||||
@@ -194,7 +165,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
if !found {
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
@@ -203,38 +174,20 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
return skipflag
|
||||
return nil
|
||||
}
|
||||
|
||||
// Common prefixes are a set, so should not have duplicates.
|
||||
// These are abstractly a "directory", so need to include the
|
||||
// delimiter at the end when we add to the map.
|
||||
cprefNoDelim := prefix + before
|
||||
cpref := prefix + before + delimiter
|
||||
if cpref == marker {
|
||||
pastMarker = true
|
||||
return skipflag
|
||||
}
|
||||
|
||||
if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
|
||||
// skip common prefixes that are before the marker
|
||||
return skipflag
|
||||
}
|
||||
|
||||
cpmap[cpref] = struct{}{}
|
||||
// delimiter at the end.
|
||||
cpmap[prefix+before+delimiter] = struct{}{}
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
newMarker = cpref
|
||||
truncated = true
|
||||
return fs.SkipAll
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return skipflag
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
// suppress file not found caused by user's prefix
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
return WalkResults{}, nil
|
||||
}
|
||||
return WalkResults{}, err
|
||||
}
|
||||
|
||||
@@ -267,213 +220,3 @@ func contains(a string, strs []string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type WalkVersioningResults struct {
|
||||
CommonPrefixes []types.CommonPrefix
|
||||
ObjectVersions []types.ObjectVersion
|
||||
DelMarkers []types.DeleteMarkerEntry
|
||||
Truncated bool
|
||||
NextMarker string
|
||||
NextVersionIdMarker string
|
||||
}
|
||||
|
||||
type ObjVersionFuncResult struct {
|
||||
ObjectVersions []types.ObjectVersion
|
||||
DelMarkers []types.DeleteMarkerEntry
|
||||
NextVersionIdMarker string
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
|
||||
|
||||
// WalkVersions walks the supplied fs.FS and returns results compatible with
|
||||
// ListObjectVersions action response
|
||||
func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
|
||||
cpmap := make(map[string]struct{})
|
||||
var objects []types.ObjectVersion
|
||||
var delMarkers []types.DeleteMarkerEntry
|
||||
|
||||
var pastMarker bool
|
||||
if keyMarker == "" {
|
||||
pastMarker = true
|
||||
}
|
||||
var nextMarker string
|
||||
var nextVersionIdMarker string
|
||||
var truncated bool
|
||||
|
||||
pastVersionIdMarker := versionIdMarker == ""
|
||||
|
||||
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
// Ignore the root directory
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
if contains(d.Name(), skipdirs) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
if !pastMarker {
|
||||
if path == keyMarker {
|
||||
pastMarker = true
|
||||
}
|
||||
if path < keyMarker {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
// If prefix is defined and the directory does not match prefix,
|
||||
// do not descend into the directory because nothing will
|
||||
// match this prefix. Make sure to append the / at the end of
|
||||
// directories since this is implied as a directory path name.
|
||||
// If path is a prefix of prefix, then path could still be
|
||||
// building to match. So only skip if path isn't a prefix of prefix
|
||||
// and prefix isn't a prefix of path.
|
||||
if prefix != "" &&
|
||||
!strings.HasPrefix(path+"/", prefix) &&
|
||||
!strings.HasPrefix(prefix, path+"/") {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// Don't recurse into subdirectories when listing with delimiter.
|
||||
if delimiter == "/" &&
|
||||
prefix != path+"/" &&
|
||||
strings.HasPrefix(path+"/", prefix) {
|
||||
cpmap[path+"/"] = struct{}{}
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, res.ObjectVersions...)
|
||||
delMarkers = append(delMarkers, res.DelMarkers...)
|
||||
if res.Truncated {
|
||||
truncated = true
|
||||
nextMarker = path
|
||||
nextVersionIdMarker = res.NextVersionIdMarker
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// If object doesn't have prefix, don't include in results.
|
||||
if prefix != "" && !strings.HasPrefix(path, prefix) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if delimiter == "" {
|
||||
// If no delimiter specified, then all files with matching
|
||||
// prefix are included in results
|
||||
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, res.ObjectVersions...)
|
||||
delMarkers = append(delMarkers, res.DelMarkers...)
|
||||
if res.Truncated {
|
||||
truncated = true
|
||||
nextMarker = path
|
||||
nextVersionIdMarker = res.NextVersionIdMarker
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Since delimiter is specified, we only want results that
|
||||
// do not contain the delimiter beyond the prefix. If the
|
||||
// delimiter exists past the prefix, then the substring
|
||||
// between the prefix and delimiter is part of common prefixes.
|
||||
//
|
||||
// For example:
|
||||
// prefix = A/
|
||||
// delimiter = /
|
||||
// and objects:
|
||||
// A/file
|
||||
// A/B/file
|
||||
// B/C
|
||||
// would return:
|
||||
// objects: A/file
|
||||
// common prefix: A/B/
|
||||
//
|
||||
// Note: No objects are included past the common prefix since
|
||||
// these are all rolled up into the common prefix.
|
||||
// Note: The delimiter can be anything, so we have to operate on
|
||||
// the full path without any assumptions on posix directory hierarchy
|
||||
// here. Usually the delimiter will be "/", but thats not required.
|
||||
suffix := strings.TrimPrefix(path, prefix)
|
||||
before, _, found := strings.Cut(suffix, delimiter)
|
||||
if !found {
|
||||
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, res.ObjectVersions...)
|
||||
delMarkers = append(delMarkers, res.DelMarkers...)
|
||||
|
||||
if res.Truncated {
|
||||
truncated = true
|
||||
nextMarker = path
|
||||
nextVersionIdMarker = res.NextVersionIdMarker
|
||||
return fs.SkipAll
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Common prefixes are a set, so should not have duplicates.
|
||||
// These are abstractly a "directory", so need to include the
|
||||
// delimiter at the end.
|
||||
cpmap[prefix+before+delimiter] = struct{}{}
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
nextMarker = path
|
||||
truncated = true
|
||||
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return WalkVersioningResults{}, err
|
||||
}
|
||||
|
||||
var commonPrefixStrings []string
|
||||
for k := range cpmap {
|
||||
commonPrefixStrings = append(commonPrefixStrings, k)
|
||||
}
|
||||
sort.Strings(commonPrefixStrings)
|
||||
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
|
||||
for _, cp := range commonPrefixStrings {
|
||||
pfx := cp
|
||||
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
|
||||
Prefix: &pfx,
|
||||
})
|
||||
}
|
||||
|
||||
return WalkVersioningResults{
|
||||
CommonPrefixes: commonPrefixes,
|
||||
ObjectVersions: objects,
|
||||
DelMarkers: delMarkers,
|
||||
Truncated: truncated,
|
||||
NextMarker: nextMarker,
|
||||
NextVersionIdMarker: nextVersionIdMarker,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -15,50 +15,36 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type walkTest struct {
|
||||
fsys fs.FS
|
||||
getobj backend.GetObjFunc
|
||||
cases []testcase
|
||||
fsys fs.FS
|
||||
expected backend.WalkResults
|
||||
getobj backend.GetObjFunc
|
||||
}
|
||||
|
||||
type testcase struct {
|
||||
name string
|
||||
prefix string
|
||||
delimiter string
|
||||
marker string
|
||||
maxObjs int32
|
||||
expected backend.WalkResults
|
||||
}
|
||||
|
||||
func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
func getObj(path string, d fs.DirEntry) (types.Object, error) {
|
||||
if d.IsDir() {
|
||||
etag := getMD5(path)
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
mtime := fi.ModTime()
|
||||
|
||||
return s3response.Object{
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: &mtime,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -66,16 +52,15 @@ func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
mtime := fi.ModTime()
|
||||
|
||||
return s3response.Object{
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: &mtime,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
}, nil
|
||||
}
|
||||
@@ -97,154 +82,50 @@ func TestWalk(t *testing.T) {
|
||||
"photos/2006/February/sample3.jpg": {},
|
||||
"photos/2006/February/sample4.jpg": {},
|
||||
},
|
||||
getobj: getObj,
|
||||
cases: []testcase{
|
||||
{
|
||||
name: "aws example",
|
||||
delimiter: "/",
|
||||
maxObjs: 1000,
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photos/"),
|
||||
}},
|
||||
Objects: []s3response.Object{{
|
||||
Key: backend.GetPtrFromString("sample.jpg"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("photos/"),
|
||||
}},
|
||||
Objects: []types.Object{{
|
||||
Key: backend.GetStringPtr("sample.jpg"),
|
||||
}},
|
||||
},
|
||||
getobj: getObj,
|
||||
},
|
||||
{
|
||||
// test case single dir/single file
|
||||
fsys: fstest.MapFS{
|
||||
"test/file": {},
|
||||
},
|
||||
getobj: getObj,
|
||||
cases: []testcase{
|
||||
{
|
||||
name: "single dir single file",
|
||||
delimiter: "/",
|
||||
maxObjs: 1000,
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("test/"),
|
||||
}},
|
||||
Objects: []s3response.Object{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// non-standard delimiter
|
||||
fsys: fstest.MapFS{
|
||||
"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
|
||||
"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
|
||||
"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("test/"),
|
||||
}},
|
||||
Objects: []types.Object{},
|
||||
},
|
||||
getobj: getObj,
|
||||
cases: []testcase{
|
||||
{
|
||||
name: "different delimiter 1",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 2",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 3",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 4",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 5",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|6/Januar|",
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different delimiter 6",
|
||||
delimiter: "|",
|
||||
maxObjs: 1000,
|
||||
prefix: "photo|s/200|6/Januar|y/sampl|",
|
||||
expected: backend.WalkResults{
|
||||
Objects: []s3response.Object{
|
||||
{
|
||||
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
|
||||
},
|
||||
{
|
||||
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
|
||||
},
|
||||
{
|
||||
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
for _, tc := range tt.cases {
|
||||
res, err := backend.Walk(context.Background(),
|
||||
tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
|
||||
tt.getobj, []string{})
|
||||
if err != nil {
|
||||
t.Errorf("tc.name: walk: %v", err)
|
||||
}
|
||||
|
||||
compareResults(tc.name, res, tc.expected, t)
|
||||
res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
|
||||
if err != nil {
|
||||
t.Fatalf("walk: %v", err)
|
||||
}
|
||||
|
||||
compareResults(res, tt.expected, t)
|
||||
}
|
||||
}
|
||||
|
||||
func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
|
||||
func compareResults(got, wanted backend.WalkResults, t *testing.T) {
|
||||
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
|
||||
t.Errorf("%v: unexpected common prefix, got %v wanted %v",
|
||||
name,
|
||||
t.Errorf("unexpected common prefix, got %v wanted %v",
|
||||
printCommonPrefixes(got.CommonPrefixes),
|
||||
printCommonPrefixes(wanted.CommonPrefixes))
|
||||
}
|
||||
|
||||
if !compareObjects(got.Objects, wanted.Objects) {
|
||||
t.Errorf("%v: unexpected object, got %v wanted %v",
|
||||
name,
|
||||
t.Errorf("unexpected object, got %v wanted %v",
|
||||
printObjects(got.Objects),
|
||||
printObjects(wanted.Objects))
|
||||
}
|
||||
@@ -287,7 +168,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
|
||||
return res + "]"
|
||||
}
|
||||
|
||||
func compareObjects(a, b []s3response.Object) bool {
|
||||
func compareObjects(a, b []types.Object) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
@@ -303,7 +184,7 @@ func compareObjects(a, b []s3response.Object) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func containsObject(c s3response.Object, list []s3response.Object) bool {
|
||||
func containsObject(c types.Object, list []types.Object) bool {
|
||||
for _, cp := range list {
|
||||
if *c.Key == *cp.Key {
|
||||
return true
|
||||
@@ -312,67 +193,14 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func printObjects(list []s3response.Object) string {
|
||||
func printObjects(list []types.Object) string {
|
||||
res := "["
|
||||
for _, cp := range list {
|
||||
var key string
|
||||
if cp.Key == nil {
|
||||
key = "<nil>"
|
||||
} else {
|
||||
key = *cp.Key
|
||||
}
|
||||
if res == "[" {
|
||||
res = res + key
|
||||
res = res + *cp.Key
|
||||
} else {
|
||||
res = res + ", " + key
|
||||
res = res + ", " + *cp.Key
|
||||
}
|
||||
}
|
||||
return res + "]"
|
||||
}
|
||||
|
||||
type slowFS struct {
|
||||
fstest.MapFS
|
||||
}
|
||||
|
||||
const (
|
||||
readDirPause = 100 * time.Millisecond
|
||||
|
||||
// walkTimeOut should be less than the tree traversal time
|
||||
// which is the readdirPause time * the number of directories
|
||||
walkTimeOut = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||
time.Sleep(readDirPause)
|
||||
return s.MapFS.ReadDir(name)
|
||||
}
|
||||
|
||||
func TestWalkStop(t *testing.T) {
|
||||
s := &slowFS{MapFS: fstest.MapFS{
|
||||
"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
|
||||
}}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
|
||||
defer cancel()
|
||||
|
||||
var err error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, err = backend.Walk(ctx, s, "", "/", "", 1000,
|
||||
func(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
return s3response.Object{}, nil
|
||||
}, []string{})
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-time.After(1 * time.Second):
|
||||
t.Fatalf("walk is not terminated in time")
|
||||
case <-ctx.Done():
|
||||
}
|
||||
wg.Wait()
|
||||
if err != ctx.Err() {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,9 +17,8 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -29,7 +28,6 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
@@ -38,9 +36,7 @@ import (
|
||||
var (
|
||||
adminAccess string
|
||||
adminSecret string
|
||||
adminRegion string
|
||||
adminEndpoint string
|
||||
allowInsecure bool
|
||||
)
|
||||
|
||||
func adminCommand() *cli.Command {
|
||||
@@ -82,33 +78,10 @@ func adminCommand() *cli.Command {
|
||||
Usage: "groupID for the new user",
|
||||
Aliases: []string{"gi"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "update-user",
|
||||
Usage: "Updates a user account",
|
||||
Action: updateUser,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "user access key id to be updated",
|
||||
Required: true,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "secret access key for the new user",
|
||||
Aliases: []string{"s"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "user-id",
|
||||
Usage: "userID for the new user",
|
||||
Aliases: []string{"ui"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "group-id",
|
||||
Usage: "groupID for the new user",
|
||||
Aliases: []string{"gi"},
|
||||
Name: "project-id",
|
||||
Usage: "projectID for the new user",
|
||||
Aliases: []string{"pi"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -173,14 +146,6 @@ func adminCommand() *cli.Command {
|
||||
Required: true,
|
||||
Destination: &adminSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "region",
|
||||
Usage: "admin s3 region string",
|
||||
EnvVars: []string{"ADMIN_REGION"},
|
||||
Value: "us-east-1",
|
||||
Destination: &adminRegion,
|
||||
Aliases: []string{"r"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "endpoint-url",
|
||||
Usage: "admin apis endpoint url",
|
||||
@@ -189,80 +154,65 @@ func adminCommand() *cli.Command {
|
||||
Required: true,
|
||||
Destination: &adminEndpoint,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "allow-insecure",
|
||||
Usage: "disable tls certificate verification for the admin endpoint",
|
||||
EnvVars: []string{"ADMIN_ALLOW_INSECURE"},
|
||||
Aliases: []string{"ai"},
|
||||
Destination: &allowInsecure,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func initHTTPClient() *http.Client {
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
|
||||
}
|
||||
return &http.Client{Transport: tr}
|
||||
}
|
||||
|
||||
func createUser(ctx *cli.Context) error {
|
||||
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
|
||||
userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
|
||||
userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
|
||||
if access == "" || secret == "" {
|
||||
return fmt.Errorf("invalid input parameters for the new user access/secret keys")
|
||||
return fmt.Errorf("invalid input parameters for the new user")
|
||||
}
|
||||
if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
|
||||
return fmt.Errorf("invalid input parameter for role: %v", role)
|
||||
}
|
||||
|
||||
acc := auth.Account{
|
||||
Access: access,
|
||||
Secret: secret,
|
||||
Role: auth.Role(role),
|
||||
UserID: userID,
|
||||
GroupID: groupID,
|
||||
Access: access,
|
||||
Secret: secret,
|
||||
Role: auth.Role(role),
|
||||
UserID: userID,
|
||||
GroupID: groupID,
|
||||
ProjectID: projectID,
|
||||
}
|
||||
|
||||
accxml, err := xml.Marshal(acc)
|
||||
accJson, err := json.Marshal(acc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse user data: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256(accxml)
|
||||
hashedPayload := sha256.Sum256(accJson)
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -270,7 +220,7 @@ func createUser(ctx *cli.Context) error {
|
||||
func deleteUser(ctx *cli.Context) error {
|
||||
access := ctx.String("access")
|
||||
if access == "" {
|
||||
return fmt.Errorf("invalid input parameter for the user access key")
|
||||
return fmt.Errorf("invalid input parameter for the new user")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
|
||||
@@ -285,82 +235,25 @@ func deleteUser(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateUser(ctx *cli.Context) error {
|
||||
access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
|
||||
props := auth.MutableProps{}
|
||||
if ctx.IsSet("secret") {
|
||||
props.Secret = &secret
|
||||
}
|
||||
if ctx.IsSet("user-id") {
|
||||
props.UserID = &userId
|
||||
}
|
||||
if ctx.IsSet("group-id") {
|
||||
props.GroupID = &groupId
|
||||
}
|
||||
|
||||
propsxml, err := xml.Marshal(props)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse user attributes: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256(propsxml)
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -378,34 +271,34 @@ func listUsers(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
var accs auth.ListUserAccountsResult
|
||||
if err := xml.Unmarshal(body, &accs); err != nil {
|
||||
var accs []auth.Account
|
||||
if err := json.Unmarshal(body, &accs); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printAcctTable(accs.Accounts)
|
||||
printAcctTable(accs)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -422,10 +315,10 @@ const (
|
||||
func printAcctTable(accs []auth.Account) {
|
||||
w := new(tabwriter.Writer)
|
||||
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
|
||||
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
|
||||
fmt.Fprintln(w, "-------\t----\t------\t-------")
|
||||
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
|
||||
fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
|
||||
for _, acc := range accs {
|
||||
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
|
||||
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
w.Flush()
|
||||
@@ -445,27 +338,25 @@ func changeBucketOwner(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -495,45 +386,34 @@ func listBuckets(ctx *cli.Context) error {
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return parseApiError(body)
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
var result s3response.ListBucketsResult
|
||||
if err := xml.Unmarshal(body, &result); err != nil {
|
||||
var buckets []s3response.Bucket
|
||||
if err := json.Unmarshal(body, &buckets); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printBuckets(result.Buckets)
|
||||
printBuckets(buckets)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseApiError(body []byte) error {
|
||||
var apiErr smithy.GenericAPIError
|
||||
err := xml.Unmarshal(body, &apiErr)
|
||||
if err != nil {
|
||||
apiErr.Code = "InternalServerError"
|
||||
apiErr.Message = err.Error()
|
||||
}
|
||||
|
||||
return &apiErr
|
||||
}
|
||||
|
||||
@@ -7,9 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
"github.com/versity/versitygw/tests/integration"
|
||||
)
|
||||
@@ -58,9 +56,7 @@ func initPosix(ctx context.Context) {
|
||||
log.Fatalf("make temp directory: %v", err)
|
||||
}
|
||||
|
||||
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
|
||||
NewDirPerm: 0755,
|
||||
})
|
||||
be, err := posix.New(tempdir, posix.PosixOpts{})
|
||||
if err != nil {
|
||||
log.Fatalf("init posix: %v", err)
|
||||
}
|
||||
@@ -78,9 +74,6 @@ func initPosix(ctx context.Context) {
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// wait for server to start
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
|
||||
@@ -19,17 +19,14 @@ import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
@@ -37,46 +34,33 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
port, admPort string
|
||||
rootUserAccess string
|
||||
rootUserSecret string
|
||||
region string
|
||||
admCertFile, admKeyFile string
|
||||
certFile, keyFile string
|
||||
kafkaURL, kafkaTopic, kafkaKey string
|
||||
natsURL, natsTopic string
|
||||
eventWebhookURL string
|
||||
eventConfigFilePath string
|
||||
logWebhookURL, accessLog string
|
||||
adminLogFile string
|
||||
healthPath string
|
||||
debug bool
|
||||
pprof string
|
||||
quiet bool
|
||||
readonly bool
|
||||
iamDir string
|
||||
ldapURL, ldapBindDN, ldapPassword string
|
||||
ldapQueryBase, ldapObjClasses string
|
||||
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
|
||||
ldapUserIdAtr, ldapGroupIdAtr string
|
||||
vaultEndpointURL, vaultSecretStoragePath string
|
||||
vaultMountPath, vaultRootToken string
|
||||
vaultRoleId, vaultRoleSecret string
|
||||
vaultServerCert, vaultClientCert string
|
||||
vaultClientCertKey string
|
||||
s3IamAccess, s3IamSecret string
|
||||
s3IamRegion, s3IamBucket string
|
||||
s3IamEndpoint string
|
||||
s3IamSslNoVerify, s3IamDebug bool
|
||||
iamCacheDisable bool
|
||||
iamCacheTTL int
|
||||
iamCachePrune int
|
||||
metricsService string
|
||||
statsdServers string
|
||||
dogstatsServers string
|
||||
ipaHost, ipaVaultName string
|
||||
ipaUser, ipaPassword string
|
||||
ipaInsecure, ipaDebug bool
|
||||
port, admPort string
|
||||
rootUserAccess string
|
||||
rootUserSecret string
|
||||
region string
|
||||
admCertFile, admKeyFile string
|
||||
certFile, keyFile string
|
||||
kafkaURL, kafkaTopic, kafkaKey string
|
||||
natsURL, natsTopic string
|
||||
eventWebhookURL string
|
||||
eventConfigFilePath string
|
||||
logWebhookURL string
|
||||
accessLog string
|
||||
healthPath string
|
||||
debug bool
|
||||
pprof string
|
||||
quiet bool
|
||||
iamDir string
|
||||
ldapURL, ldapBindDN, ldapPassword string
|
||||
ldapQueryBase, ldapObjClasses string
|
||||
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
|
||||
s3IamAccess, s3IamSecret string
|
||||
s3IamRegion, s3IamBucket string
|
||||
s3IamEndpoint string
|
||||
s3IamSslNoVerify, s3IamDebug bool
|
||||
iamCacheDisable bool
|
||||
iamCacheTTL int
|
||||
iamCachePrune int
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -117,13 +101,10 @@ func main() {
|
||||
|
||||
func initApp() *cli.App {
|
||||
return &cli.App{
|
||||
Usage: "Versity S3 Gateway",
|
||||
Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
|
||||
to access the supported backend storage as if it was a native S3 service.
|
||||
VersityGW is an open-source project licensed under the Apache 2.0 License. The
|
||||
source code is hosted on GitHub at https://github.com/versity/versitygw, and
|
||||
documentation can be found in the GitHub wiki.`,
|
||||
Copyright: "Copyright (c) 2023-2024 Versity Software",
|
||||
Name: "versitygw",
|
||||
Usage: "Start S3 gateway service with specified backend storage.",
|
||||
Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
|
||||
to access the supported backend storage as if it was a native S3 service.`,
|
||||
Action: func(ctx *cli.Context) error {
|
||||
return ctx.App.Command("help").Run(ctx)
|
||||
},
|
||||
@@ -209,7 +190,6 @@ func initFlags() []cli.Flag {
|
||||
&cli.BoolFlag{
|
||||
Name: "debug",
|
||||
Usage: "enable debug output",
|
||||
Value: false,
|
||||
EnvVars: []string{"VGW_DEBUG"},
|
||||
Destination: &debug,
|
||||
},
|
||||
@@ -232,12 +212,6 @@ func initFlags() []cli.Flag {
|
||||
EnvVars: []string{"LOGFILE", "VGW_ACCESS_LOG"},
|
||||
Destination: &accessLog,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "admin-access-log",
|
||||
Usage: "enable admin server access logging to specified file",
|
||||
EnvVars: []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
|
||||
Destination: &adminLogFile,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "log-webhook-url",
|
||||
Usage: "webhook url to send the audit logs",
|
||||
@@ -347,72 +321,6 @@ func initFlags() []cli.Flag {
|
||||
EnvVars: []string{"VGW_IAM_LDAP_ROLE_ATR"},
|
||||
Destination: &ldapRoleAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-user-id-atr",
|
||||
Usage: "ldap server user id attribute name",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_USER_ID_ATR"},
|
||||
Destination: &ldapUserIdAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-group-id-atr",
|
||||
Usage: "ldap server user group id attribute name",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
|
||||
Destination: &ldapGroupIdAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-endpoint-url",
|
||||
Usage: "vault server url",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
|
||||
Destination: &vaultEndpointURL,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-secret-storage-path",
|
||||
Usage: "vault server secret storage path",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
|
||||
Destination: &vaultSecretStoragePath,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-mount-path",
|
||||
Usage: "vault server mount path",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_MOUNT_PATH"},
|
||||
Destination: &vaultMountPath,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-root-token",
|
||||
Usage: "vault server root token",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
|
||||
Destination: &vaultRootToken,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-role-id",
|
||||
Usage: "vault server user role id",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ROLE_ID"},
|
||||
Destination: &vaultRoleId,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-role-secret",
|
||||
Usage: "vault server user role secret",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_ROLE_SECRET"},
|
||||
Destination: &vaultRoleSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-server_cert",
|
||||
Usage: "vault server TLS certificate",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_SERVER_CERT"},
|
||||
Destination: &vaultServerCert,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-client_cert",
|
||||
Usage: "vault client TLS certificate",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT"},
|
||||
Destination: &vaultClientCert,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-vault-client_cert_key",
|
||||
Usage: "vault client TLS certificate key",
|
||||
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
|
||||
Destination: &vaultClientCertKey,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-access",
|
||||
Usage: "s3 IAM access key",
|
||||
@@ -483,69 +391,6 @@ func initFlags() []cli.Flag {
|
||||
EnvVars: []string{"VGW_HEALTH"},
|
||||
Destination: &healthPath,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "readonly",
|
||||
Usage: "allow only read operations across all the gateway",
|
||||
EnvVars: []string{"VGW_READ_ONLY"},
|
||||
Destination: &readonly,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-service-name",
|
||||
Usage: "service name tag for metrics, hostname if blank",
|
||||
EnvVars: []string{"VGW_METRICS_SERVICE_NAME"},
|
||||
Aliases: []string{"msn"},
|
||||
Destination: &metricsService,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-statsd-servers",
|
||||
Usage: "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
|
||||
EnvVars: []string{"VGW_METRICS_STATSD_SERVERS"},
|
||||
Aliases: []string{"mss"},
|
||||
Destination: &statsdServers,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-dogstatsd-servers",
|
||||
Usage: "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
|
||||
EnvVars: []string{"VGW_METRICS_DOGSTATS_SERVERS"},
|
||||
Aliases: []string{"mds"},
|
||||
Destination: &dogstatsServers,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-host",
|
||||
Usage: "FreeIPA server url e.g. https://ipa.example.test",
|
||||
EnvVars: []string{"VGW_IPA_HOST"},
|
||||
Destination: &ipaHost,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-vault-name",
|
||||
Usage: "A name of the user vault containing their secret",
|
||||
EnvVars: []string{"VGW_IPA_VAULT_NAME"},
|
||||
Destination: &ipaVaultName,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-user",
|
||||
Usage: "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
|
||||
EnvVars: []string{"VGW_IPA_USER"},
|
||||
Destination: &ipaUser,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "ipa-password",
|
||||
Usage: "Password of the user used to connect to FreeIPA.",
|
||||
EnvVars: []string{"VGW_IPA_PASSWORD"},
|
||||
Destination: &ipaPassword,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "ipa-insecure",
|
||||
Usage: "Verify TLS certificate of FreeIPA server. Default is 'true'.",
|
||||
EnvVars: []string{"VGW_IPA_INSECURE"},
|
||||
Destination: &ipaInsecure,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "ipa-debug",
|
||||
Usage: "FreeIPA IAM debug output",
|
||||
EnvVars: []string{"VGW_IPA_DEBUG"},
|
||||
Destination: &ipaDebug,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -563,12 +408,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
StreamRequestBody: true,
|
||||
DisableKeepalive: true,
|
||||
Network: fiber.NetworkTCP,
|
||||
DisableStartupMessage: true,
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
StreamRequestBody: true,
|
||||
DisableKeepalive: true,
|
||||
})
|
||||
|
||||
var opts []s3api.Option
|
||||
@@ -599,15 +442,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
if healthPath != "" {
|
||||
opts = append(opts, s3api.WithHealth(healthPath))
|
||||
}
|
||||
if readonly {
|
||||
opts = append(opts, s3api.WithReadOnly())
|
||||
}
|
||||
|
||||
admApp := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
Network: fiber.NetworkTCP,
|
||||
DisableStartupMessage: true,
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
})
|
||||
|
||||
var admOpts []s3api.AdminOpt
|
||||
@@ -628,70 +466,38 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
}
|
||||
|
||||
iam, err := auth.New(&auth.Opts{
|
||||
RootAccount: auth.Account{
|
||||
Access: rootUserAccess,
|
||||
Secret: rootUserSecret,
|
||||
Role: auth.RoleAdmin,
|
||||
},
|
||||
Dir: iamDir,
|
||||
LDAPServerURL: ldapURL,
|
||||
LDAPBindDN: ldapBindDN,
|
||||
LDAPPassword: ldapPassword,
|
||||
LDAPQueryBase: ldapQueryBase,
|
||||
LDAPObjClasses: ldapObjClasses,
|
||||
LDAPAccessAtr: ldapAccessAtr,
|
||||
LDAPSecretAtr: ldapSecAtr,
|
||||
LDAPRoleAtr: ldapRoleAtr,
|
||||
LDAPUserIdAtr: ldapUserIdAtr,
|
||||
LDAPGroupIdAtr: ldapGroupIdAtr,
|
||||
VaultEndpointURL: vaultEndpointURL,
|
||||
VaultSecretStoragePath: vaultSecretStoragePath,
|
||||
VaultMountPath: vaultMountPath,
|
||||
VaultRootToken: vaultRootToken,
|
||||
VaultRoleId: vaultRoleId,
|
||||
VaultRoleSecret: vaultRoleSecret,
|
||||
VaultServerCert: vaultServerCert,
|
||||
VaultClientCert: vaultClientCert,
|
||||
VaultClientCertKey: vaultClientCertKey,
|
||||
S3Access: s3IamAccess,
|
||||
S3Secret: s3IamSecret,
|
||||
S3Region: s3IamRegion,
|
||||
S3Bucket: s3IamBucket,
|
||||
S3Endpoint: s3IamEndpoint,
|
||||
S3DisableSSlVerfiy: s3IamSslNoVerify,
|
||||
S3Debug: s3IamDebug,
|
||||
CacheDisable: iamCacheDisable,
|
||||
CacheTTL: iamCacheTTL,
|
||||
CachePrune: iamCachePrune,
|
||||
IpaHost: ipaHost,
|
||||
IpaVaultName: ipaVaultName,
|
||||
IpaUser: ipaUser,
|
||||
IpaPassword: ipaPassword,
|
||||
IpaInsecure: ipaInsecure,
|
||||
IpaDebug: ipaDebug,
|
||||
Dir: iamDir,
|
||||
LDAPServerURL: ldapURL,
|
||||
LDAPBindDN: ldapBindDN,
|
||||
LDAPPassword: ldapPassword,
|
||||
LDAPQueryBase: ldapQueryBase,
|
||||
LDAPObjClasses: ldapObjClasses,
|
||||
LDAPAccessAtr: ldapAccessAtr,
|
||||
LDAPSecretAtr: ldapSecAtr,
|
||||
LDAPRoleAtr: ldapRoleAtr,
|
||||
S3Access: s3IamAccess,
|
||||
S3Secret: s3IamSecret,
|
||||
S3Region: s3IamRegion,
|
||||
S3Bucket: s3IamBucket,
|
||||
S3Endpoint: s3IamEndpoint,
|
||||
S3DisableSSlVerfiy: s3IamSslNoVerify,
|
||||
S3Debug: s3IamDebug,
|
||||
CacheDisable: iamCacheDisable,
|
||||
CacheTTL: iamCacheTTL,
|
||||
CachePrune: iamCachePrune,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup iam: %w", err)
|
||||
}
|
||||
|
||||
loggers, err := s3log.InitLogger(&s3log.LogConfig{
|
||||
LogFile: accessLog,
|
||||
WebhookURL: logWebhookURL,
|
||||
AdminLogFile: adminLogFile,
|
||||
logger, err := s3log.InitLogger(&s3log.LogConfig{
|
||||
LogFile: accessLog,
|
||||
WebhookURL: logWebhookURL,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup logger: %w", err)
|
||||
}
|
||||
|
||||
metricsManager, err := metrics.NewManager(ctx, metrics.Config{
|
||||
ServiceName: metricsService,
|
||||
StatsdServers: statsdServers,
|
||||
DogStatsdServers: dogstatsServers,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("init metrics manager: %w", err)
|
||||
}
|
||||
|
||||
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
|
||||
KafkaURL: kafkaURL,
|
||||
KafkaTopic: kafkaTopic,
|
||||
@@ -708,16 +514,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
|
||||
Access: rootUserAccess,
|
||||
Secret: rootUserSecret,
|
||||
}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
|
||||
}, port, region, iam, logger, evSender, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init gateway: %v", err)
|
||||
}
|
||||
|
||||
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
|
||||
|
||||
if !quiet {
|
||||
printBanner(port, admPort, certFile != "", admCertFile != "")
|
||||
}
|
||||
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
|
||||
|
||||
c := make(chan error, 2)
|
||||
go func() { c <- srv.Serve() }()
|
||||
@@ -734,17 +536,10 @@ Loop:
|
||||
case err = <-c:
|
||||
break Loop
|
||||
case <-sigHup:
|
||||
if loggers.S3Logger != nil {
|
||||
err = loggers.S3Logger.HangUp()
|
||||
if logger != nil {
|
||||
err = logger.HangUp()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("HUP s3 logger: %w", err)
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
if loggers.AdminLogger != nil {
|
||||
err = loggers.AdminLogger.HangUp()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("HUP admin logger: %w", err)
|
||||
err = fmt.Errorf("HUP logger: %w", err)
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
@@ -762,22 +557,13 @@ Loop:
|
||||
fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
|
||||
}
|
||||
|
||||
if loggers.S3Logger != nil {
|
||||
err := loggers.S3Logger.Shutdown()
|
||||
if logger != nil {
|
||||
err := logger.Shutdown()
|
||||
if err != nil {
|
||||
if saveErr == nil {
|
||||
saveErr = err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
|
||||
}
|
||||
}
|
||||
if loggers.AdminLogger != nil {
|
||||
err := loggers.AdminLogger.Shutdown()
|
||||
if err != nil {
|
||||
if saveErr == nil {
|
||||
saveErr = err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -791,183 +577,5 @@ Loop:
|
||||
}
|
||||
}
|
||||
|
||||
if metricsManager != nil {
|
||||
metricsManager.Close()
|
||||
}
|
||||
|
||||
return saveErr
|
||||
}
|
||||
|
||||
func printBanner(port, admPort string, ssl, admSsl bool) {
|
||||
interfaces, err := getMatchingIPs(port)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var admInterfaces []string
|
||||
if admPort != "" {
|
||||
admInterfaces, err = getMatchingIPs(admPort)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
title := "VersityGW"
|
||||
version := fmt.Sprintf("Version %v, Build %v", Version, Build)
|
||||
urls := []string{}
|
||||
|
||||
hst, prt, err := net.SplitHostPort(port)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, ip := range interfaces {
|
||||
url := fmt.Sprintf("http://%s:%s", ip, prt)
|
||||
if ssl {
|
||||
url = fmt.Sprintf("https://%s:%s", ip, prt)
|
||||
}
|
||||
urls = append(urls, url)
|
||||
}
|
||||
|
||||
if hst == "" {
|
||||
hst = "0.0.0.0"
|
||||
}
|
||||
|
||||
boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
|
||||
|
||||
lines := []string{
|
||||
centerText(title),
|
||||
centerText(version),
|
||||
centerText(boundHost),
|
||||
centerText(""),
|
||||
}
|
||||
|
||||
if len(admInterfaces) > 0 {
|
||||
lines = append(lines,
|
||||
leftText("S3 service listening on:"),
|
||||
)
|
||||
} else {
|
||||
lines = append(lines,
|
||||
leftText("Admin/S3 service listening on:"),
|
||||
)
|
||||
}
|
||||
|
||||
for _, url := range urls {
|
||||
lines = append(lines, leftText(" "+url))
|
||||
}
|
||||
|
||||
if len(admInterfaces) > 0 {
|
||||
lines = append(lines,
|
||||
centerText(""),
|
||||
leftText("Admin service listening on:"),
|
||||
)
|
||||
|
||||
_, prt, err := net.SplitHostPort(admPort)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, ip := range admInterfaces {
|
||||
url := fmt.Sprintf("http://%s:%s", ip, prt)
|
||||
if admSsl {
|
||||
url = fmt.Sprintf("https://%s:%s", ip, prt)
|
||||
}
|
||||
lines = append(lines, leftText(" "+url))
|
||||
}
|
||||
}
|
||||
|
||||
// Print the top border
|
||||
fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
|
||||
|
||||
// Print each line
|
||||
for _, line := range lines {
|
||||
fmt.Printf("│%-*s│\n", columnWidth-2, line)
|
||||
}
|
||||
|
||||
// Print the bottom border
|
||||
fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
|
||||
}
|
||||
|
||||
// getMatchingIPs returns all IP addresses for local system interfaces that
|
||||
// match the input address specification.
|
||||
func getMatchingIPs(spec string) ([]string, error) {
|
||||
// Split the input spec into IP and port
|
||||
host, _, err := net.SplitHostPort(spec)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse address/port: %v", err)
|
||||
}
|
||||
|
||||
// Handle cases where IP is omitted (e.g., ":1234")
|
||||
if host == "" {
|
||||
host = "0.0.0.0"
|
||||
}
|
||||
|
||||
ipaddr, err := net.ResolveIPAddr("ip", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parsedInputIP := ipaddr.IP
|
||||
|
||||
var result []string
|
||||
|
||||
// Get all network interfaces
|
||||
interfaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, iface := range interfaces {
|
||||
// Get all addresses associated with the interface
|
||||
addrs, err := iface.Addrs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, addr := range addrs {
|
||||
// Parse the address to get the IP part
|
||||
ipAddr, _, err := net.ParseCIDR(addr.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ipAddr.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
if ipAddr.IsInterfaceLocalMulticast() {
|
||||
continue
|
||||
}
|
||||
if ipAddr.IsLinkLocalMulticast() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if the IP matches the input specification
|
||||
if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
|
||||
result = append(result, ipAddr.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const columnWidth = 70
|
||||
|
||||
func centerText(text string) string {
|
||||
padding := (columnWidth - 2 - len(text)) / 2
|
||||
if padding < 0 {
|
||||
padding = 0
|
||||
}
|
||||
return strings.Repeat(" ", padding) + text
|
||||
}
|
||||
|
||||
func leftText(text string) string {
|
||||
if len(text) > columnWidth-2 {
|
||||
return text
|
||||
}
|
||||
return text + strings.Repeat(" ", columnWidth-2-len(text))
|
||||
}
|
||||
|
||||
@@ -16,21 +16,14 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"math"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
var (
|
||||
chownuid, chowngid bool
|
||||
bucketlinks bool
|
||||
versioningDir string
|
||||
dirPerms uint
|
||||
sidecar string
|
||||
nometa bool
|
||||
readonly bool
|
||||
)
|
||||
|
||||
func posixCommand() *cli.Command {
|
||||
@@ -62,36 +55,10 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
|
||||
Destination: &chowngid,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "bucketlinks",
|
||||
Usage: "allow symlinked directories at bucket level to be treated as buckets",
|
||||
EnvVars: []string{"VGW_BUCKET_LINKS"},
|
||||
Destination: &bucketlinks,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "versioning-dir",
|
||||
Usage: "the directory path to enable bucket versioning",
|
||||
EnvVars: []string{"VGW_VERSIONING_DIR"},
|
||||
Destination: &versioningDir,
|
||||
},
|
||||
&cli.UintFlag{
|
||||
Name: "dir-perms",
|
||||
Usage: "default directory permissions for new directories",
|
||||
EnvVars: []string{"VGW_DIR_PERMS"},
|
||||
Destination: &dirPerms,
|
||||
DefaultText: "0755",
|
||||
Value: 0755,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "sidecar",
|
||||
Usage: "use provided sidecar directory to store metadata",
|
||||
EnvVars: []string{"VGW_META_SIDECAR"},
|
||||
Destination: &sidecar,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "nometa",
|
||||
Usage: "disable metadata storage",
|
||||
EnvVars: []string{"VGW_META_NONE"},
|
||||
Destination: &nometa,
|
||||
Name: "readonly",
|
||||
Usage: "allow only read operations to backend",
|
||||
EnvVars: []string{"VGW_READ_ONLY"},
|
||||
Destination: &readonly,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -102,46 +69,13 @@ func runPosix(ctx *cli.Context) error {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
gwroot := (ctx.Args().Get(0))
|
||||
|
||||
if dirPerms > math.MaxUint32 {
|
||||
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
|
||||
}
|
||||
|
||||
if nometa && sidecar != "" {
|
||||
return fmt.Errorf("cannot use both nometa and sidecar metadata")
|
||||
}
|
||||
|
||||
opts := posix.PosixOpts{
|
||||
ChownUID: chownuid,
|
||||
ChownGID: chowngid,
|
||||
BucketLinks: bucketlinks,
|
||||
VersioningDir: versioningDir,
|
||||
NewDirPerm: fs.FileMode(dirPerms),
|
||||
}
|
||||
|
||||
var ms meta.MetadataStorer
|
||||
switch {
|
||||
case sidecar != "":
|
||||
sc, err := meta.NewSideCar(sidecar)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init sidecar metadata: %w", err)
|
||||
}
|
||||
ms = sc
|
||||
opts.SideCarDir = sidecar
|
||||
case nometa:
|
||||
ms = meta.NoMeta{}
|
||||
default:
|
||||
ms = meta.XattrMeta{}
|
||||
err := meta.XattrMeta{}.Test(gwroot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("xattr check failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
be, err := posix.New(gwroot, ms, opts)
|
||||
be, err := posix.New(ctx.Args().Get(0), posix.PosixOpts{
|
||||
ChownUID: chownuid,
|
||||
ChownGID: chowngid,
|
||||
ReadOnly: readonly,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init posix backend: %w", err)
|
||||
return fmt.Errorf("init posix: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx.Context, be)
|
||||
|
||||
@@ -16,8 +16,6 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"math"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/scoutfs"
|
||||
@@ -25,6 +23,10 @@ import (
|
||||
|
||||
var (
|
||||
glacier bool
|
||||
|
||||
// defined in posix.go:
|
||||
// chownuid, chowngid bool
|
||||
// readonly bool
|
||||
)
|
||||
|
||||
func scoutfsCommand() *cli.Command {
|
||||
@@ -66,18 +68,10 @@ move interfaces as well as support for tiered filesystems.`,
|
||||
Destination: &chowngid,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "bucketlinks",
|
||||
Usage: "allow symlinked directories at bucket level to be treated as buckets",
|
||||
EnvVars: []string{"VGW_BUCKET_LINKS"},
|
||||
Destination: &bucketlinks,
|
||||
},
|
||||
&cli.UintFlag{
|
||||
Name: "dir-perms",
|
||||
Usage: "default directory permissions for new directories",
|
||||
EnvVars: []string{"VGW_DIR_PERMS"},
|
||||
Destination: &dirPerms,
|
||||
DefaultText: "0755",
|
||||
Value: 0755,
|
||||
Name: "readonly",
|
||||
Usage: "allow only read operations to backend",
|
||||
EnvVars: []string{"VGW_READ_ONLY"},
|
||||
Destination: &readonly,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -88,16 +82,11 @@ func runScoutfs(ctx *cli.Context) error {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
if dirPerms > math.MaxUint32 {
|
||||
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
|
||||
}
|
||||
|
||||
var opts scoutfs.ScoutfsOpts
|
||||
opts.GlacierMode = glacier
|
||||
opts.ChownUID = chownuid
|
||||
opts.ChownGID = chowngid
|
||||
opts.BucketLinks = bucketlinks
|
||||
opts.NewDirPerm = fs.FileMode(dirPerms)
|
||||
opts.ReadOnly = readonly
|
||||
|
||||
be, err := scoutfs.New(ctx.Args().Get(0), opts)
|
||||
if err != nil {
|
||||
|
||||
@@ -22,23 +22,20 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
awsID string
|
||||
awsSecret string
|
||||
endpoint string
|
||||
prefix string
|
||||
dstBucket string
|
||||
partSize int64
|
||||
objSize int64
|
||||
concurrency int
|
||||
files int
|
||||
totalReqs int
|
||||
upload bool
|
||||
download bool
|
||||
pathStyle bool
|
||||
checksumDisable bool
|
||||
versioningEnabled bool
|
||||
azureTests bool
|
||||
tlsStatus bool
|
||||
awsID string
|
||||
awsSecret string
|
||||
endpoint string
|
||||
prefix string
|
||||
dstBucket string
|
||||
partSize int64
|
||||
objSize int64
|
||||
concurrency int
|
||||
files int
|
||||
totalReqs int
|
||||
upload bool
|
||||
download bool
|
||||
pathStyle bool
|
||||
checksumDisable bool
|
||||
)
|
||||
|
||||
func testCommand() *cli.Command {
|
||||
@@ -80,12 +77,6 @@ func initTestFlags() []cli.Flag {
|
||||
Aliases: []string{"d"},
|
||||
Destination: &debug,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "allow-insecure",
|
||||
Usage: "skip tls verification",
|
||||
Aliases: []string{"ai"},
|
||||
Destination: &tlsStatus,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,33 +87,11 @@ func initTestCommands() []*cli.Command {
|
||||
Usage: "Tests the full flow of gateway.",
|
||||
Description: `Runs all the available tests to test the full flow of the gateway.`,
|
||||
Action: getAction(integration.TestFullFlow),
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "versioning-enabled",
|
||||
Usage: "Test the bucket object versioning, if the versioning is enabled",
|
||||
Destination: &versioningEnabled,
|
||||
Aliases: []string{"vs"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "azure-test-mode",
|
||||
Usage: "Skips tests that are not supported by Azure",
|
||||
Destination: &azureTests,
|
||||
Aliases: []string{"azure"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "posix",
|
||||
Usage: "Tests posix specific features",
|
||||
Action: getAction(integration.TestPosix),
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "versioning-enabled",
|
||||
Usage: "Test posix when versioning is enabled",
|
||||
Destination: &versioningEnabled,
|
||||
Aliases: []string{"vs"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "iam",
|
||||
@@ -218,7 +187,6 @@ func initTestCommands() []*cli.Command {
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
integration.WithPartSize(partSize),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
@@ -279,7 +247,6 @@ func initTestCommands() []*cli.Command {
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
@@ -305,17 +272,10 @@ func getAction(tf testFunc) func(*cli.Context) error {
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if versioningEnabled {
|
||||
opts = append(opts, integration.WithVersioningEnabled())
|
||||
}
|
||||
if azureTests {
|
||||
opts = append(opts, integration.WithAzureMode())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
tf(s)
|
||||
@@ -343,27 +303,15 @@ func extractIntTests() (commands []*cli.Command) {
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithTLSStatus(tlsStatus),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if versioningEnabled {
|
||||
opts = append(opts, integration.WithVersioningEnabled())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
err := testFunc(s)
|
||||
return err
|
||||
},
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "versioning-enabled",
|
||||
Usage: "Test the bucket object versioning, if the versioning is enabled",
|
||||
Destination: &versioningEnabled,
|
||||
Aliases: []string{"vs"},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
return
|
||||
|
||||
@@ -54,24 +54,22 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
|
||||
}
|
||||
|
||||
config := s3event.EventFilter{
|
||||
s3event.EventObjectCreated: true,
|
||||
s3event.EventObjectCreatedPut: true,
|
||||
s3event.EventObjectCreatedPost: true,
|
||||
s3event.EventObjectCreatedCopy: true,
|
||||
s3event.EventCompleteMultipartUpload: true,
|
||||
s3event.EventObjectRemoved: true,
|
||||
s3event.EventObjectRemovedDelete: true,
|
||||
s3event.EventObjectRemovedDeleteObjects: true,
|
||||
s3event.EventObjectTagging: true,
|
||||
s3event.EventObjectTaggingPut: true,
|
||||
s3event.EventObjectTaggingDelete: true,
|
||||
s3event.EventObjectAclPut: true,
|
||||
s3event.EventObjectRestore: true,
|
||||
s3event.EventObjectRestorePost: true,
|
||||
s3event.EventObjectRestoreCompleted: true,
|
||||
s3event.EventObjectCreated: true,
|
||||
s3event.EventObjectCreatedPut: true,
|
||||
s3event.EventObjectCreatedPost: true,
|
||||
s3event.EventObjectCreatedCopy: true,
|
||||
s3event.EventCompleteMultipartUpload: true,
|
||||
s3event.EventObjectDeleted: true,
|
||||
s3event.EventObjectTagging: true,
|
||||
s3event.EventObjectTaggingPut: true,
|
||||
s3event.EventObjectTaggingDelete: true,
|
||||
s3event.EventObjectAclPut: true,
|
||||
s3event.EventObjectRestore: true,
|
||||
s3event.EventObjectRestorePost: true,
|
||||
s3event.EventObjectRestoreCompleted: true,
|
||||
}
|
||||
|
||||
configBytes, err := json.MarshalIndent(config, "", " ")
|
||||
configBytes, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse event config: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
version: "3"
|
||||
services:
|
||||
posix:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: tests/Dockerfile.dev
|
||||
dockerfile: ./Dockerfile.dev
|
||||
args:
|
||||
- IAM_DIR=${IAM_DIR}
|
||||
- SETUP_DIR=${SETUP_DIR}
|
||||
@@ -14,7 +15,7 @@ services:
|
||||
proxy:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: tests/Dockerfile.dev
|
||||
dockerfile: ./Dockerfile.dev
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
@@ -30,13 +31,13 @@ services:
|
||||
hostname: azurite
|
||||
command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
|
||||
volumes:
|
||||
- ./tests/certs:/tests/certs
|
||||
- ./certs:/certs
|
||||
azuritegw:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: tests/Dockerfile.dev
|
||||
dockerfile: ./Dockerfile.dev
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
- 7070:7070
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
|
||||
@@ -1,19 +0,0 @@
|
||||
# Versity Gateway Dashboard
|
||||
|
||||
This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana.
|
||||
|
||||
The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
|
||||
|
||||
This implementation uses the influxql query language.
|
||||
|
||||
## Usage
|
||||
|
||||
From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
|
||||
|
||||
To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`.
|
||||
|
||||
The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
|
||||
|
||||
The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with username: `admin` and password: `admin`.
|
||||
|
||||
To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.
|
||||
@@ -1,4 +0,0 @@
|
||||
export AWS_SECRET_ACCESS_KEY=password
|
||||
export AWS_ACCESS_KEY_ID=user
|
||||
export AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
export AWS_REGION=us-east-1
|
||||
@@ -1,64 +0,0 @@
|
||||
services:
|
||||
telegraf:
|
||||
image: telegraf
|
||||
container_name: telegraf
|
||||
restart: always
|
||||
volumes:
|
||||
- ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
|
||||
depends_on:
|
||||
- influxdb
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '8125:8125/udp'
|
||||
|
||||
influxdb:
|
||||
image: influxdb
|
||||
container_name: influxdb
|
||||
restart: always
|
||||
environment:
|
||||
- DOCKER_INFLUXDB_INIT_MODE=setup
|
||||
- DOCKER_INFLUXDB_INIT_USERNAME=admin
|
||||
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
|
||||
- DOCKER_INFLUXDB_INIT_ORG=myorg
|
||||
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
|
||||
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
|
||||
ports:
|
||||
- '8086:8086'
|
||||
volumes:
|
||||
- influxdb_data:/var/lib/influxdb
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana
|
||||
container_name: grafana-server
|
||||
restart: always
|
||||
depends_on:
|
||||
- influxdb
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
- GF_INSTALL_PLUGINS=
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./metrics-exploration/grafana_data:/var/lib/grafana
|
||||
- ./metrics-exploration/provisioning:/etc/grafana/provisioning
|
||||
|
||||
versitygw:
|
||||
image: versity/versitygw:latest
|
||||
container_name: versitygw
|
||||
ports:
|
||||
- "7070:7070"
|
||||
environment:
|
||||
- ROOT_ACCESS_KEY=user
|
||||
- ROOT_SECRET_KEY=password
|
||||
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
|
||||
depends_on:
|
||||
- telegraf
|
||||
command: >
|
||||
posix /tmp/vgw
|
||||
|
||||
volumes:
|
||||
influxdb_data: {}
|
||||
@@ -1,64 +0,0 @@
|
||||
services:
|
||||
telegraf:
|
||||
image: telegraf
|
||||
container_name: telegraf
|
||||
restart: always
|
||||
volumes:
|
||||
- ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
|
||||
depends_on:
|
||||
- influxdb
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '8125:8125/udp'
|
||||
|
||||
influxdb:
|
||||
image: influxdb
|
||||
container_name: influxdb
|
||||
restart: always
|
||||
environment:
|
||||
- DOCKER_INFLUXDB_INIT_MODE=setup
|
||||
- DOCKER_INFLUXDB_INIT_USERNAME=admin
|
||||
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
|
||||
- DOCKER_INFLUXDB_INIT_ORG=myorg
|
||||
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
|
||||
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
|
||||
ports:
|
||||
- '8086:8086'
|
||||
volumes:
|
||||
- influxdb_data:/var/lib/influxdb
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana
|
||||
container_name: grafana-server
|
||||
restart: always
|
||||
depends_on:
|
||||
- influxdb
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
- GF_INSTALL_PLUGINS=
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./grafana_data:/var/lib/grafana
|
||||
- ./provisioning:/etc/grafana/provisioning
|
||||
|
||||
versitygw:
|
||||
image: versity/versitygw:latest
|
||||
container_name: versitygw
|
||||
ports:
|
||||
- "7070:7070"
|
||||
environment:
|
||||
- ROOT_ACCESS_KEY=user
|
||||
- ROOT_SECRET_KEY=password
|
||||
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
|
||||
depends_on:
|
||||
- telegraf
|
||||
command: >
|
||||
posix /tmp/vgw
|
||||
|
||||
volumes:
|
||||
influxdb_data: {}
|
||||
@@ -1,25 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
# <string> an unique provider name. Required
|
||||
- name: 'influxql'
|
||||
# <int> Org id. Default to 1
|
||||
orgId: 1
|
||||
# <string> name of the dashboard folder.
|
||||
folder: 'influxql'
|
||||
# <string> folder UID. will be automatically generated if not specified
|
||||
folderUid: ''
|
||||
# <string> provider type. Default to 'file'
|
||||
type: file
|
||||
# <bool> disable dashboard deletion
|
||||
disableDeletion: false
|
||||
# <int> how often Grafana will scan for changed dashboards
|
||||
updateIntervalSeconds: 10
|
||||
# <bool> allow updating provisioned dashboards from the UI
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
# <string, required> path to dashboard files on disk. Required when using the 'file' type
|
||||
path: /etc/grafana/provisioning/dashboards/influxql
|
||||
# <bool> use folder names from filesystem to create folders in Grafana
|
||||
foldersFromFilesStructure: true
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,13 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: influxdb
|
||||
type: influxdb
|
||||
isDefault: true
|
||||
access: proxy
|
||||
url: http://influxdb:8086
|
||||
jsonData:
|
||||
dbName: 'metrics'
|
||||
httpHeaderName1: 'Authorization'
|
||||
secureJsonData:
|
||||
httpHeaderValue1: 'Token my-super-secret-auth-token'
|
||||
@@ -1,34 +0,0 @@
|
||||
[global_tags]
|
||||
|
||||
[agent]
|
||||
debug = true
|
||||
quiet = false
|
||||
interval = "60s"
|
||||
round_interval = true
|
||||
metric_batch_size = 1000
|
||||
metric_buffer_limit = 10000
|
||||
collection_jitter = "0s"
|
||||
flush_interval = "10s"
|
||||
flush_jitter = "0s"
|
||||
precision = ""
|
||||
hostname = "versitygw"
|
||||
omit_hostname = false
|
||||
|
||||
[[outputs.file]]
|
||||
files = ["stdout"]
|
||||
|
||||
[[outputs.influxdb_v2]]
|
||||
urls = ["http://influxdb:8086"]
|
||||
timeout = "5s"
|
||||
token = "my-super-secret-auth-token"
|
||||
organization = "myorg"
|
||||
bucket = "metrics"
|
||||
|
||||
[[inputs.statsd]]
|
||||
protocol = "udp4"
|
||||
service_address = ":8125"
|
||||
percentiles = [90]
|
||||
metric_separator = "_"
|
||||
datadog_extensions = false
|
||||
allowed_pending_messages = 10000
|
||||
percentile_limit = 1000
|
||||
@@ -1,6 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
. ./aws_env_setup.sh
|
||||
|
||||
aws s3 mb s3://test
|
||||
aws s3 cp docker-compose.yml s3://test/test.yaml
|
||||
@@ -153,14 +153,6 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
# specified, all configured bucket events will be sent to the webhook.
|
||||
#VGW_EVENT_WEBHOOK_URL=
|
||||
|
||||
# Bucket events can be filtered for any of the above event types. The
|
||||
# VGW_EVENT_FILTER option specifies a config file that contains the event
|
||||
# filter rules. The event filter rules are used to determine which events are
|
||||
# sent to the configured event services. Run:
|
||||
# versitygw utils gen-event-filter-config --path .
|
||||
# to generate a default rules file "event_config.json" in the current directory.
|
||||
#VGW_EVENT_FILTER=
|
||||
|
||||
#######################
|
||||
# Debug / Diagnostics #
|
||||
#######################
|
||||
@@ -190,40 +182,7 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
# as a dedicated IAM service.
|
||||
#VGW_IAM_DIR=
|
||||
|
||||
# The Vault options will enable the Vault IAM service with accounts stored in
|
||||
# the HashiCorp Vault service. The Vault URL is the address and port of the
|
||||
# Vault server with the format <IP/host>:<port>. A root taken can be used for
|
||||
# testing, but it is recommended to use the role based authentication in
|
||||
# production. The Vault server certificate, client certificate, and client
|
||||
# certificate key are optional, and will default to not verifying the server
|
||||
# certificate and not using client certificates. The Vault server certificate
|
||||
# is used to verify the Vault server, and the client certificate and key are
|
||||
# used to authenticate the gateway to the Vault server. See wiki documentation
|
||||
# for an example of using Vault in dev mode with the gateway.
|
||||
#VGW_IAM_VAULT_ENDPOINT_URL=
|
||||
#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
|
||||
#VGW_IAM_VAULT_MOUNT_PATH=
|
||||
#VGW_IAM_VAULT_ROOT_TOKEN=
|
||||
#VGW_IAM_VAULT_ROLE_ID=
|
||||
#VGW_IAM_VAULT_ROLE_SECRET=
|
||||
#VGW_IAM_VAULT_SERVER_CERT=
|
||||
#VGW_IAM_VAULT_CLIENT_CERT=
|
||||
#VGW_IAM_VAULT_CLIENT_CERT_KEY=
|
||||
|
||||
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
|
||||
# stores the account information JSON encoded in an S3 object. This should use
|
||||
# a bucket that is not accessible to general users when using s3 backend to
|
||||
# prevent access to account credentials. This IAM service is added for
|
||||
# convenience, but is not considered as secure or scalable as a dedicated IAM
|
||||
# service.
|
||||
#VGW_S3_IAM_ACCESS_KEY=
|
||||
#VGW_S3_IAM_SECRET_KEY=
|
||||
#VGW_S3_IAM_REGION=
|
||||
#VGW_S3_IAM_ENDPOINT=
|
||||
#VGW_S3_IAM_BUCKET=
|
||||
#VGW_S3_IAM_NO_VERIFY=
|
||||
|
||||
# The LDAP options will enable the LDAP IAM service with accounts stored in an
|
||||
# The ldap options will enable the LDAP IAM service with accounts stored in an
|
||||
# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
|
||||
# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
|
||||
# secret credentials and role respectively. The other options are used to
|
||||
@@ -237,8 +196,19 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_IAM_LDAP_ACCESS_ATR=
|
||||
#VGW_IAM_LDAP_SECRET_ATR=
|
||||
#VGW_IAM_LDAP_ROLE_ATR=
|
||||
#VGW_IAM_LDAP_USER_ID_ATR=
|
||||
#VGW_IAM_LDAP_GROUP_ID_ATR=
|
||||
|
||||
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
|
||||
# stores the account information JSON encoded in an S3 object. This should use
|
||||
# a bucket that is not accessible to general users when using s3 backend to
|
||||
# prevent access to account credentials. This IAM service is added for
|
||||
# convenience, but is not considered as secure or scalable as a dedicated IAM
|
||||
# service.
|
||||
#VGW_S3_IAM_ACCESS_KEY=
|
||||
#VGW_S3_IAM_SECRET_KEY=
|
||||
#VGW_S3_IAM_REGION=
|
||||
#VGW_S3_IAM_ENDPOINT=
|
||||
#VGW_S3_IAM_BUCKET=
|
||||
#VGW_S3_IAM_NO_VERIFY=
|
||||
|
||||
###############
|
||||
# IAM caching #
|
||||
@@ -258,29 +228,6 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_IAM_CACHE_TTL=120
|
||||
#VGW_IAM_CACHE_PRUNE=3600
|
||||
|
||||
###########
|
||||
# Metrics #
|
||||
###########
|
||||
|
||||
# The metrics service name is a tag that is added to all metrics to help
|
||||
# identify the source of the metrics. This is especially useful when multiple
|
||||
# gateways are running. The default is the hostname of the system.
|
||||
#VGW_METRICS_SERVICE_NAME=$HOSTNAME
|
||||
|
||||
# The metrics service will send metrics to the configured statsd servers. The
|
||||
# servers are specified as a comma separated list of host:port pairs. The
|
||||
# default is to not send metrics to any statsd servers. The gateway uses
|
||||
# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
|
||||
#VGW_METRICS_STATSD_SERVERS=
|
||||
|
||||
# The metrics service will send metrics to the configured dogstatsd servers.
|
||||
# The servers are specified as a comma separated list of host:port pairs. The
|
||||
# default is to not send metrics to any dogstatsd servers. Generally
|
||||
# DataDog recommends installing a local agent to collect metrics and forward
|
||||
# them to the DataDog service. In this case the option value would be the
|
||||
# local agent address: 127.0.0.1:8125.
|
||||
#VGW_METRICS_DOGSTATS_SERVERS=
|
||||
|
||||
######################################
|
||||
# VersityGW Backend Specific Options #
|
||||
######################################
|
||||
@@ -307,15 +254,11 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_CHOWN_UID=false
|
||||
#VGW_CHOWN_GID=false
|
||||
|
||||
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
|
||||
# to directories at the top level gateway directory as buckets.
|
||||
#VGW_BUCKET_LINKS=false
|
||||
|
||||
# The default permissions mode when creating new directories is 0755. Use
|
||||
# VGW_DIR_PERMS option to set a different mode for any new directory that the
|
||||
# gateway creates. This applies to buckets created through the gateway as well
|
||||
# as any parent directories automatically created with object uploads.
|
||||
#VGW_DIR_PERMS=0755
|
||||
# The VGW_READ_ONLY option will disable all write operations to the backend
|
||||
# filesystem. This is useful for creating a read-only gateway for clients.
|
||||
# This will prevent any PUT, POST, DELETE, and multipart upload operations
|
||||
# for objects and buckets as well as preventing updating metadata for objects.
|
||||
#VGW_READ_ONLY=false
|
||||
|
||||
###########
|
||||
# scoutfs #
|
||||
@@ -348,15 +291,11 @@ ROOT_SECRET_ACCESS_KEY=
|
||||
#VGW_CHOWN_UID=false
|
||||
#VGW_CHOWN_GID=false
|
||||
|
||||
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
|
||||
# to directories at the top level gateway directory as buckets.
|
||||
#VGW_BUCKET_LINKS=false
|
||||
|
||||
# The default permissions mode when creating new directories is 0755. Use
|
||||
# VGW_DIR_PERMS option to set a different mode for any new directory that the
|
||||
# gateway creates. This applies to buckets created through the gateway as well
|
||||
# as any parent directories automatically created with object uploads.
|
||||
#VGW_DIR_PERMS=0755
|
||||
# The VGW_READ_ONLY option will disable all write operations to the backend
|
||||
# filesystem. This is useful for creating a read-only gateway for clients.
|
||||
# This will prevent any PUT, POST, DELETE, and multipart upload operations
|
||||
# for objects and buckets as well as preventing updating metadata for objects.
|
||||
#VGW_READ_ONLY=false
|
||||
|
||||
######
|
||||
# s3 #
|
||||
|
||||
102
go.mod
102
go.mod
@@ -1,82 +1,70 @@
|
||||
module github.com/versity/versitygw
|
||||
|
||||
go 1.21.0
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
|
||||
github.com/DataDog/datadog-go/v5 v5.6.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.0
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2
|
||||
github.com/aws/smithy-go v1.22.2
|
||||
github.com/go-ldap/ldap/v3 v3.4.10
|
||||
github.com/gofiber/fiber/v2 v2.52.6
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
|
||||
github.com/aws/smithy-go v1.20.2
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/gofiber/fiber/v2 v2.52.4
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/vault-client-go v0.4.3
|
||||
github.com/nats-io/nats.go v1.38.0
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/pkg/xattr v0.4.10
|
||||
github.com/nats-io/nats.go v1.34.1
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.47
|
||||
github.com/smira/go-statsd v1.3.4
|
||||
github.com/urfave/cli/v2 v2.27.5
|
||||
github.com/valyala/fasthttp v1.58.0
|
||||
github.com/urfave/cli/v2 v2.27.1
|
||||
github.com/valyala/fasthttp v1.52.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
|
||||
golang.org/x/sync v0.10.0
|
||||
golang.org/x/sys v0.29.0
|
||||
golang.org/x/sys v0.19.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.9 // indirect
|
||||
github.com/nats-io/nkeys v0.4.7 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
golang.org/x/crypto v0.32.0 // indirect
|
||||
golang.org/x/net v0.34.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
golang.org/x/time v0.9.0 // indirect
|
||||
golang.org/x/crypto v0.22.0 // indirect
|
||||
golang.org/x/net v0.24.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.4
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.57
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
|
||||
github.com/klauspost/compress v1.17.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
|
||||
)
|
||||
|
||||
307
go.sum
307
go.sum
@@ -1,186 +1,129 @@
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 h1:1mvYtZfWQAnwNah/C+Z+Jb9rQH95LPE2vlmMuWAHJk8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1/go.mod h1:75I/mXtme1JyWFtz8GocPHVFyH421IBoZErnO16dd0k=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1 h1:Bk5uOhSAenHyR5P61D/NzeQCv+4fEVV8mOkJ82NqpWw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1/go.mod h1:QZ4pw3or1WPmRBxf0cHd1tknzrT54WPBOQoGutCPvSU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 h1:H5xDQaE3XowWfhZRUpnfC+rGZMEVoSiji+b+/HFAPU4=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
|
||||
github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
|
||||
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.0 h1:b1wM5CcE65Ujwn565qcwgtOTT1aT4ADOHHgglKjG7fk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.0/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.4 h1:ObNqKsDYFGr2WxnoXKOhCvTlf3HhwtoGgc+KmZ4H5yg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.4/go.mod h1:j2/AF7j/qxVmsNIChw1tWfsVKOayJoGRDjg1Tgq7NPk=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.57 h1:kFQDsbdBAR3GZsB8xA+51ptEnq9TIj3tS4MuP5b+TcQ=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.57/go.mod h1:2kerxPUUbTagAr/kkaHiqvj/bcYHzi2qiJS/ZinllU0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 h1:7lOW8NUwE9UZekS1DYoiPdVAqZ6A+LheHWb+mHbNOq8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27/go.mod h1:w1BASFIPOPUae7AgaH4SbjNbfdkxuggLyGfNFTn8ITY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57 h1:4hFrvTb32jty/LpKdIwWhMgqITPxNo9l1X1hjUyVCZ4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57/go.mod h1:n6n8rfggAVPgDVldL1zk9QUzIWImRb6OWI8t9CfDImM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 h1:lWm9ucLSRFiI4dQQafLrEOmEDGry3Swrz0BIRdiHJqQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31/go.mod h1:Huu6GG0YTfbPphQkDSo4dEGmQRTKb9k9G7RdtyQWxuI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 h1:ACxDklUKKXb48+eg5ROZXi1vDgfMyfIA/WyvqHcHI0o=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31/go.mod h1:yadnfsDwqXeVaohbGc/RaD287PuyRw2wugkh5ZL2J6k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 h1:8IwBjuLdqIO1dGB+dZ9zJEl8wzY3bVYxcs0Xyu/Lsc0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31/go.mod h1:8tMBcuVjL4kP/ECEIWTCWtwV2kj6+ouEKl4cqR4iWLw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 h1:siiQ+jummya9OLPDEyHVb2dLW4aOMe22FGDd0sAfuSw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5/go.mod h1:iHVx2J9pWzITdP5MJY6qWfG34TfD9EA+Qi3eV6qQCXw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 h1:O+8vD2rGjfihBewr5bT+QUfYUHIxCVgG61LHoT59shM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12/go.mod h1:usVdWJaosa66NMvmCrr08NcWDBRv4E6+YFG2pUdw1Lk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 h1:tkVNm99nkJnFo1H9IIQb5QkCiPcvCDn3Pos+IeTbGRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12/go.mod h1:dIVlquSPUMqEJtx2/W17SM2SuESRaVEhEV9alcMqxjw=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2 h1:dyC+iA2+Yc7iDMDh0R4eT6fi8TgBduc+BOWCy6Br0/o=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2/go.mod h1:FHSHmyEUkzRbaFFqqm6bkLAOQHgqhsLmfCahvCBMiyA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 h1:c5WJ3iHz7rLIgArznb3JCSQT3uUMiz9DLZhIX+1G8ok=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.14/go.mod h1:+JJQTxB6N4niArC14YNtxcQtwEqzS3o9Z32n7q33Rfs=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 h1:f1L/JtUkVODD+k1+IiSJUUv8A++2qVr+Xvb3xWXETMU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13/go.mod h1:tvqlFoja8/s0o+UruA1Nrezo/df0PzdunMDDurUfg6U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 h1:fqg6c1KVrc3SYWma/egWue5rKI4G2+M4wMQN2JosNAA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.12/go.mod h1:7Yn+p66q/jt38qMoVfNvjbm3D89mGBnkwDcijgtih8w=
|
||||
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
|
||||
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
|
||||
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
|
||||
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
|
||||
github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
|
||||
github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
|
||||
github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
|
||||
github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
|
||||
github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
|
||||
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
|
||||
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
|
||||
github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
|
||||
github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
|
||||
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
|
||||
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/nats-io/nats.go v1.38.0 h1:A7P+g7Wjp4/NWqDOOP/K6hfhr54DvdDQUznt5JFg9XA=
|
||||
github.com/nats-io/nats.go v1.38.0/go.mod h1:IGUM++TwokGnXPs82/wCuiHS02/aKrdYUQkU8If6yjw=
|
||||
github.com/nats-io/nkeys v0.4.9 h1:qe9Faq2Gxwi6RZnZMXfmGMZkg3afLLOtrU+gDZJ35b0=
|
||||
github.com/nats-io/nkeys v0.4.9/go.mod h1:jcMqs+FLG+W5YO36OX6wFIFcmpdAns+w1Wm6D3I/evE=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
|
||||
github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
|
||||
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
|
||||
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
|
||||
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
|
||||
github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
|
||||
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
|
||||
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
|
||||
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
|
||||
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
|
||||
github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
|
||||
github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
|
||||
github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.58.0 h1:GGB2dWxSbEprU9j0iMJHgdKYJVDyjrOwF9RE59PbRuE=
|
||||
github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw=
|
||||
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
|
||||
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
|
||||
@@ -191,86 +134,49 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
|
||||
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
|
||||
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
|
||||
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
|
||||
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
|
||||
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
@@ -278,24 +184,17 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
|
||||
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,269 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
type Action struct {
|
||||
Name string
|
||||
Service string
|
||||
}
|
||||
|
||||
var (
|
||||
ActionMap map[string]Action
|
||||
)
|
||||
|
||||
var (
|
||||
ActionUndetected = "ActionUnDetected"
|
||||
ActionAbortMultipartUpload = "s3_AbortMultipartUpload"
|
||||
ActionCompleteMultipartUpload = "s3_CompleteMultipartUpload"
|
||||
ActionCopyObject = "s3_CopyObject"
|
||||
ActionCreateBucket = "s3_CreateBucket"
|
||||
ActionCreateMultipartUpload = "s3_CreateMultipartUpload"
|
||||
ActionDeleteBucket = "s3_DeleteBucket"
|
||||
ActionDeleteBucketPolicy = "s3_DeleteBucketPolicy"
|
||||
ActionDeleteBucketTagging = "s3_DeleteBucketTagging"
|
||||
ActionDeleteObject = "s3_DeleteObject"
|
||||
ActionDeleteObjectTagging = "s3_DeleteObjectTagging"
|
||||
ActionDeleteObjects = "s3_DeleteObjects"
|
||||
ActionGetBucketAcl = "s3_GetBucketAcl"
|
||||
ActionGetBucketPolicy = "s3_GetBucketPolicy"
|
||||
ActionGetBucketTagging = "s3_GetBucketTagging"
|
||||
ActionGetBucketVersioning = "s3_GetBucketVersioning"
|
||||
ActionGetObject = "s3_GetObject"
|
||||
ActionGetObjectAcl = "s3_GetObjectAcl"
|
||||
ActionGetObjectAttributes = "s3_GetObjectAttributes"
|
||||
ActionGetObjectLegalHold = "s3_GetObjectLegalHold"
|
||||
ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
|
||||
ActionGetObjectRetention = "s3_GetObjectRetention"
|
||||
ActionGetObjectTagging = "s3_GetObjectTagging"
|
||||
ActionHeadBucket = "s3_HeadBucket"
|
||||
ActionHeadObject = "s3_HeadObject"
|
||||
ActionListAllMyBuckets = "s3_ListAllMyBuckets"
|
||||
ActionListMultipartUploads = "s3_ListMultipartUploads"
|
||||
ActionListObjectVersions = "s3_ListObjectVersions"
|
||||
ActionListObjects = "s3_ListObjects"
|
||||
ActionListObjectsV2 = "s3_ListObjectsV2"
|
||||
ActionListParts = "s3_ListParts"
|
||||
ActionPutBucketAcl = "s3_PutBucketAcl"
|
||||
ActionPutBucketPolicy = "s3_PutBucketPolicy"
|
||||
ActionPutBucketTagging = "s3_PutBucketTagging"
|
||||
ActionPutBucketVersioning = "s3_PutBucketVersioning"
|
||||
ActionPutObject = "s3_PutObject"
|
||||
ActionPutObjectAcl = "s3_PutObjectAcl"
|
||||
ActionPutObjectLegalHold = "s3_PutObjectLegalHold"
|
||||
ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
|
||||
ActionPutObjectRetention = "s3_PutObjectRetention"
|
||||
ActionPutObjectTagging = "s3_PutObjectTagging"
|
||||
ActionRestoreObject = "s3_RestoreObject"
|
||||
ActionSelectObjectContent = "s3_SelectObjectContent"
|
||||
ActionUploadPart = "s3_UploadPart"
|
||||
ActionUploadPartCopy = "s3_UploadPartCopy"
|
||||
ActionPutBucketOwnershipControls = "s3_PutBucketOwnershipControls"
|
||||
ActionGetBucketOwnershipControls = "s3_GetBucketOwnershipControls"
|
||||
ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
|
||||
|
||||
// Admin actions
|
||||
ActionAdminCreateUser = "admin_CreateUser"
|
||||
ActionAdminUpdateUser = "admin_UpdateUser"
|
||||
ActionAdminDeleteUser = "admin_DeleteUser"
|
||||
ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
|
||||
ActionAdminListUsers = "admin_ListUsers"
|
||||
ActionAdminListBuckets = "admin_ListBuckets"
|
||||
)
|
||||
|
||||
func init() {
|
||||
ActionMap = make(map[string]Action)
|
||||
|
||||
ActionMap[ActionUndetected] = Action{
|
||||
Name: "ActionUnDetected",
|
||||
Service: "unknown",
|
||||
}
|
||||
|
||||
ActionMap[ActionAbortMultipartUpload] = Action{
|
||||
Name: "AbortMultipartUpload",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCompleteMultipartUpload] = Action{
|
||||
Name: "CompleteMultipartUpload",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCopyObject] = Action{
|
||||
Name: "CopyObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCreateBucket] = Action{
|
||||
Name: "CreateBucket",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionCreateMultipartUpload] = Action{
|
||||
Name: "CreateMultipartUpload",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteBucket] = Action{
|
||||
Name: "DeleteBucket",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteBucketPolicy] = Action{
|
||||
Name: "DeleteBucketPolicy",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteBucketTagging] = Action{
|
||||
Name: "DeleteBucketTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteObject] = Action{
|
||||
Name: "DeleteObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteObjectTagging] = Action{
|
||||
Name: "DeleteObjectTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionDeleteObjects] = Action{
|
||||
Name: "DeleteObjects",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketAcl] = Action{
|
||||
Name: "GetBucketAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketPolicy] = Action{
|
||||
Name: "GetBucketPolicy",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketTagging] = Action{
|
||||
Name: "GetBucketTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetBucketVersioning] = Action{
|
||||
Name: "GetBucketVersioning",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObject] = Action{
|
||||
Name: "GetObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectAcl] = Action{
|
||||
Name: "GetObjectAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectAttributes] = Action{
|
||||
Name: "GetObjectAttributes",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectLegalHold] = Action{
|
||||
Name: "GetObjectLegalHold",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectLockConfiguration] = Action{
|
||||
Name: "GetObjectLockConfiguration",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectRetention] = Action{
|
||||
Name: "GetObjectRetention",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionGetObjectTagging] = Action{
|
||||
Name: "GetObjectTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionHeadBucket] = Action{
|
||||
Name: "HeadBucket",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionHeadObject] = Action{
|
||||
Name: "HeadObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListAllMyBuckets] = Action{
|
||||
Name: "ListAllMyBuckets",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListMultipartUploads] = Action{
|
||||
Name: "ListMultipartUploads",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListObjectVersions] = Action{
|
||||
Name: "ListObjectVersions",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListObjects] = Action{
|
||||
Name: "ListObjects",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListObjectsV2] = Action{
|
||||
Name: "ListObjectsV2",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionListParts] = Action{
|
||||
Name: "ListParts",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketAcl] = Action{
|
||||
Name: "PutBucketAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketPolicy] = Action{
|
||||
Name: "PutBucketPolicy",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketTagging] = Action{
|
||||
Name: "PutBucketTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutBucketVersioning] = Action{
|
||||
Name: "PutBucketVersioning",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObject] = Action{
|
||||
Name: "PutObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectAcl] = Action{
|
||||
Name: "PutObjectAcl",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectLegalHold] = Action{
|
||||
Name: "PutObjectLegalHold",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectLockConfiguration] = Action{
|
||||
Name: "PutObjectLockConfiguration",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectRetention] = Action{
|
||||
Name: "PutObjectRetention",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionPutObjectTagging] = Action{
|
||||
Name: "PutObjectTagging",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionRestoreObject] = Action{
|
||||
Name: "RestoreObject",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionSelectObjectContent] = Action{
|
||||
Name: "SelectObjectContent",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionUploadPart] = Action{
|
||||
Name: "UploadPart",
|
||||
Service: "s3",
|
||||
}
|
||||
ActionMap[ActionUploadPartCopy] = Action{
|
||||
Name: "UploadPartCopy",
|
||||
Service: "s3",
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
dogstats "github.com/DataDog/datadog-go/v5/statsd"
|
||||
)
|
||||
|
||||
// vgwDogStatsd metrics type
|
||||
type vgwDogStatsd struct {
|
||||
c *dogstats.Client
|
||||
}
|
||||
|
||||
var (
|
||||
rateSampleAlways = 1.0
|
||||
)
|
||||
|
||||
// newDogStatsd takes a server address and returns a statsd merics
|
||||
func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
|
||||
c, err := dogstats.New(server,
|
||||
dogstats.WithMaxMessagesPerPayload(1000),
|
||||
dogstats.WithNamespace("versitygw"),
|
||||
dogstats.WithTags([]string{
|
||||
"service:" + service,
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &vgwDogStatsd{c: c}, nil
|
||||
}
|
||||
|
||||
// Close closes statsd connections
|
||||
func (s *vgwDogStatsd) Close() {
|
||||
s.c.Close()
|
||||
}
|
||||
|
||||
func (t Tag) ddString() string {
|
||||
if t.Value == "" {
|
||||
return t.Key
|
||||
}
|
||||
return fmt.Sprintf("%v:%v", t.Key, t.Value)
|
||||
}
|
||||
|
||||
// Add adds value to key
|
||||
func (s *vgwDogStatsd) Add(key string, value int64, tags ...Tag) {
|
||||
stags := make([]string, len(tags))
|
||||
for i, t := range tags {
|
||||
stags[i] = t.ddString()
|
||||
}
|
||||
s.c.Count(key, value, stags, rateSampleAlways)
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
var (
|
||||
// max size of data items to buffer before dropping
|
||||
// new incoming data items
|
||||
dataItemCount = 100000
|
||||
)
|
||||
|
||||
// Tag is added metadata for metrics
|
||||
type Tag struct {
|
||||
// Key is tag name
|
||||
Key string
|
||||
// Value is tag data
|
||||
Value string
|
||||
}
|
||||
|
||||
// Manager is a manager of metrics plugins
|
||||
type Manager struct {
|
||||
wg sync.WaitGroup
|
||||
ctx context.Context
|
||||
|
||||
config Config
|
||||
|
||||
publishers []publisher
|
||||
addDataChan chan datapoint
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ServiceName string
|
||||
StatsdServers string
|
||||
DogStatsdServers string
|
||||
}
|
||||
|
||||
// NewManager initializes metrics plugins and returns a new metrics manager
|
||||
func NewManager(ctx context.Context, conf Config) (*Manager, error) {
|
||||
if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if conf.ServiceName == "" {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
||||
}
|
||||
conf.ServiceName = hostname
|
||||
}
|
||||
|
||||
addDataChan := make(chan datapoint, dataItemCount)
|
||||
|
||||
mgr := &Manager{
|
||||
addDataChan: addDataChan,
|
||||
ctx: ctx,
|
||||
config: conf,
|
||||
}
|
||||
|
||||
// setup statsd endpoints
|
||||
if len(conf.StatsdServers) > 0 {
|
||||
statsdServers := strings.Split(conf.StatsdServers, ",")
|
||||
|
||||
for _, server := range statsdServers {
|
||||
statsd, err := newStatsd(server, conf.ServiceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mgr.publishers = append(mgr.publishers, statsd)
|
||||
}
|
||||
}
|
||||
|
||||
// setup dogstatsd endpoints
|
||||
if len(conf.DogStatsdServers) > 0 {
|
||||
dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
|
||||
|
||||
for _, server := range dogStatsdServers {
|
||||
dogStatsd, err := newDogStatsd(server, conf.ServiceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mgr.publishers = append(mgr.publishers, dogStatsd)
|
||||
}
|
||||
}
|
||||
|
||||
mgr.wg.Add(1)
|
||||
go mgr.addForwarder(addDataChan)
|
||||
|
||||
return mgr, nil
|
||||
}
|
||||
|
||||
func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) {
|
||||
// In case of Authentication failures, url parsing ...
|
||||
if action == "" {
|
||||
action = ActionUndetected
|
||||
}
|
||||
|
||||
a := ActionMap[action]
|
||||
reqTags := []Tag{
|
||||
{Key: "method", Value: ctx.Method()},
|
||||
{Key: "api", Value: a.Service},
|
||||
{Key: "action", Value: a.Name},
|
||||
}
|
||||
|
||||
reqStatus := status
|
||||
|
||||
if err != nil {
|
||||
var apierr s3err.APIError
|
||||
if errors.As(err, &apierr) {
|
||||
reqStatus = apierr.HTTPStatusCode
|
||||
} else {
|
||||
reqStatus = http.StatusInternalServerError
|
||||
}
|
||||
}
|
||||
if reqStatus == 0 {
|
||||
reqStatus = http.StatusOK
|
||||
}
|
||||
|
||||
reqTags = append(reqTags, Tag{
|
||||
Key: "status",
|
||||
Value: fmt.Sprintf("%v", reqStatus),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
m.increment("failed_count", reqTags...)
|
||||
} else {
|
||||
m.increment("success_count", reqTags...)
|
||||
}
|
||||
|
||||
switch action {
|
||||
case ActionPutObject:
|
||||
m.add("bytes_written", count, reqTags...)
|
||||
m.increment("object_created_count", reqTags...)
|
||||
case ActionCompleteMultipartUpload:
|
||||
m.increment("object_created_count", reqTags...)
|
||||
case ActionUploadPart:
|
||||
m.add("bytes_written", count, reqTags...)
|
||||
case ActionGetObject:
|
||||
m.add("bytes_read", count, reqTags...)
|
||||
case ActionDeleteObject:
|
||||
m.increment("object_removed_count", reqTags...)
|
||||
case ActionDeleteObjects:
|
||||
m.add("object_removed_count", count, reqTags...)
|
||||
}
|
||||
}
|
||||
|
||||
// increment increments the key by one
|
||||
func (m *Manager) increment(key string, tags ...Tag) {
|
||||
m.add(key, 1, tags...)
|
||||
}
|
||||
|
||||
// add adds value to key
|
||||
func (m *Manager) add(key string, value int64, tags ...Tag) {
|
||||
if m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
d := datapoint{
|
||||
key: key,
|
||||
value: value,
|
||||
tags: tags,
|
||||
}
|
||||
|
||||
select {
|
||||
case m.addDataChan <- d:
|
||||
default:
|
||||
// channel full, drop the updates
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes metrics channels, waits for data to complete, closes all plugins
|
||||
func (m *Manager) Close() {
|
||||
// drain the datapoint channels
|
||||
close(m.addDataChan)
|
||||
m.wg.Wait()
|
||||
|
||||
// close all publishers
|
||||
for _, p := range m.publishers {
|
||||
p.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// publisher is the interface for interacting with the metrics plugins
|
||||
type publisher interface {
|
||||
Add(key string, value int64, tags ...Tag)
|
||||
Close()
|
||||
}
|
||||
|
||||
func (m *Manager) addForwarder(addChan <-chan datapoint) {
|
||||
for data := range addChan {
|
||||
for _, s := range m.publishers {
|
||||
s.Add(data.key, data.value, data.tags...)
|
||||
}
|
||||
}
|
||||
m.wg.Done()
|
||||
}
|
||||
|
||||
type datapoint struct {
|
||||
key string
|
||||
value int64
|
||||
tags []Tag
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/smira/go-statsd"
|
||||
)
|
||||
|
||||
// vgwStatsd metrics type
|
||||
type vgwStatsd struct {
|
||||
c *statsd.Client
|
||||
}
|
||||
|
||||
// newStatsd takes a server address and returns a statsd merics
|
||||
// Supply service name to be used as a tag to identify the spcific
|
||||
// gateway instance, this may typically be the gateway hostname
|
||||
func newStatsd(server string, service string) (*vgwStatsd, error) {
|
||||
c := statsd.NewClient(
|
||||
server,
|
||||
statsd.MetricPrefix("versitygw."),
|
||||
statsd.TagStyle(statsd.TagFormatInfluxDB),
|
||||
statsd.DefaultTags(statsd.StringTag("service", service)),
|
||||
)
|
||||
return &vgwStatsd{c: c}, nil
|
||||
}
|
||||
|
||||
// Close closes statsd connections
|
||||
func (s *vgwStatsd) Close() {
|
||||
s.c.Close()
|
||||
}
|
||||
|
||||
// Add adds value to key
|
||||
func (s *vgwStatsd) Add(key string, value int64, tags ...Tag) {
|
||||
stags := make([]statsd.Tag, len(tags))
|
||||
for i, t := range tags {
|
||||
stags[i] = statsd.StringTag(t.Key, t.Value)
|
||||
}
|
||||
s.c.Incr(key, value, stags...)
|
||||
}
|
||||
122
runtests.sh
122
runtests.sh
@@ -5,27 +5,15 @@ rm -rf /tmp/gw
|
||||
mkdir /tmp/gw
|
||||
rm -rf /tmp/covdata
|
||||
mkdir /tmp/covdata
|
||||
rm -rf /tmp/versioing.covdata
|
||||
mkdir /tmp/versioning.covdata
|
||||
rm -rf /tmp/versioningdir
|
||||
mkdir /tmp/versioningdir
|
||||
|
||||
# setup tls certificate and key
|
||||
ECHO "Generating TLS certificate and key in the cert.pem and key.pem files"
|
||||
|
||||
openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
|
||||
openssl req -new -x509 -key key.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
|
||||
|
||||
ECHO "Running the sdk test over http"
|
||||
# run server in background not versioning-enabled
|
||||
# port: 7070(default)
|
||||
# run server in background
|
||||
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
|
||||
GW_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if gateway process is still running
|
||||
# check if server is still running
|
||||
if ! kill -0 $GW_PID; then
|
||||
echo "server no longer running"
|
||||
exit 1
|
||||
@@ -51,110 +39,8 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_PID
|
||||
|
||||
ECHO "Running the sdk test over https"
|
||||
|
||||
# run server in background with TLS certificate
|
||||
# port: 7071(default)
|
||||
GOCOVERDIR=/tmp/https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7071 -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
|
||||
GW_HTTPS_PID=$!
|
||||
|
||||
sleep 1
|
||||
|
||||
# check if https gateway process is still running
|
||||
if ! kill -0 $GW_HTTPS_PID; then
|
||||
echo "server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
# full flow tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow; then
|
||||
echo "full flow tests failed"
|
||||
kill $GW_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
# posix tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 posix; then
|
||||
echo "posix tests failed"
|
||||
kill $GW_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
# iam tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 iam; then
|
||||
echo "iam tests failed"
|
||||
kill $GW_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kill $GW_HTTPS_PID
|
||||
|
||||
|
||||
ECHO "Running the sdk test over http against the versioning-enabled gateway"
|
||||
# run server in background versioning-enabled
|
||||
# port: 7072
|
||||
GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7072 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
|
||||
GW_VS_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if versioning-enabled gateway process is still running
|
||||
if ! kill -0 $GW_VS_PID; then
|
||||
echo "versioning-enabled server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
# full flow tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs; then
|
||||
echo "versioning-enabled full-flow tests failed"
|
||||
kill $GW_VS_PID
|
||||
exit 1
|
||||
fi
|
||||
# posix tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 posix -vs; then
|
||||
echo "versiongin-enabled posix tests failed"
|
||||
kill $GW_VS_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_VS_PID
|
||||
|
||||
ECHO "Running the sdk test over https against the versioning-enabled gateway"
|
||||
# run server in background versioning-enabled
|
||||
# port: 7073
|
||||
GOCOVERDIR=/tmp/versioning.https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7073 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
|
||||
GW_VS_HTTPS_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if versioning-enabled gateway process is still running
|
||||
if ! kill -0 $GW_VS_HTTPS_PID; then
|
||||
echo "versioning-enabled server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
# full flow tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs; then
|
||||
echo "versioning-enabled full-flow tests failed"
|
||||
kill $GW_VS_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
# posix tests
|
||||
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 posix -vs; then
|
||||
echo "versiongin-enabled posix tests failed"
|
||||
kill $GW_VS_HTTPS_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_VS_HTTPS_PID
|
||||
|
||||
exit 0
|
||||
|
||||
# if the above binary was built with -cover enabled (make testbin),
|
||||
|
||||
@@ -19,13 +19,12 @@ import (
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
type S3AdminRouter struct{}
|
||||
|
||||
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
|
||||
controller := controllers.NewAdminController(iam, be, logger)
|
||||
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
|
||||
controller := controllers.NewAdminController(iam, be)
|
||||
|
||||
// CreateUser admin api
|
||||
app.Patch("/create-user", controller.CreateUser)
|
||||
@@ -33,9 +32,6 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
|
||||
// DeleteUsers admin api
|
||||
app.Patch("/delete-user", controller.DeleteUser)
|
||||
|
||||
// UpdateUser admin api
|
||||
app.Patch("/update-user", controller.UpdateUser)
|
||||
|
||||
// ListUsers admin api
|
||||
app.Patch("/list-users", controller.ListUsers)
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
type S3AdminServer struct {
|
||||
@@ -33,7 +32,7 @@ type S3AdminServer struct {
|
||||
cert *tls.Certificate
|
||||
}
|
||||
|
||||
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
|
||||
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
|
||||
server := &S3AdminServer{
|
||||
app: app,
|
||||
backend: be,
|
||||
@@ -47,16 +46,14 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
|
||||
|
||||
// Logging middlewares
|
||||
app.Use(logger.New())
|
||||
app.Use(middlewares.DecodeURL(l, nil))
|
||||
app.Use(middlewares.DecodeURL(nil))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
|
||||
app.Use(middlewares.VerifyMD5Body(nil))
|
||||
app.Use(middlewares.AclParser(be, nil))
|
||||
|
||||
// Admin role checker
|
||||
app.Use(middlewares.IsAdmin(l))
|
||||
|
||||
server.router.Init(app, be, iam, l)
|
||||
server.router.Init(app, be, iam)
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
@@ -16,188 +16,107 @@ package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type AdminController struct {
|
||||
iam auth.IAMService
|
||||
be backend.Backend
|
||||
l s3log.AuditLogger
|
||||
}
|
||||
|
||||
func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
|
||||
return AdminController{iam: iam, be: be, l: l}
|
||||
func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
|
||||
return AdminController{iam: iam, be: be}
|
||||
}
|
||||
|
||||
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
var usr auth.Account
|
||||
err := xml.Unmarshal(ctx.Body(), &usr)
|
||||
err := json.Unmarshal(ctx.Body(), &usr)
|
||||
if err != nil {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
})
|
||||
return fmt.Errorf("failed to parse request body: %w", err)
|
||||
}
|
||||
|
||||
if !usr.Role.IsValid() {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
})
|
||||
if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
|
||||
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
|
||||
}
|
||||
|
||||
err = c.iam.CreateAccount(usr)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "user already exists") {
|
||||
err = s3err.GetAPIError(s3err.ErrAdminUserExists)
|
||||
}
|
||||
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
})
|
||||
return fmt.Errorf("failed to create a user: %w", err)
|
||||
}
|
||||
|
||||
return SendResponse(ctx, nil,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminCreateUser,
|
||||
Status: http.StatusCreated,
|
||||
})
|
||||
}
|
||||
|
||||
func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
|
||||
access := ctx.Query("access")
|
||||
if access == "" {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
}
|
||||
|
||||
var props auth.MutableProps
|
||||
if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
}
|
||||
|
||||
err := c.iam.UpdateUserAccount(access, props)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "user not found") {
|
||||
err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
|
||||
}
|
||||
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
}
|
||||
|
||||
return SendResponse(ctx, nil,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminUpdateUser,
|
||||
})
|
||||
return ctx.SendString("The user has been created successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
|
||||
access := ctx.Query("access")
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
|
||||
err := c.iam.DeleteUserAccount(access)
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminDeleteUser,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.SendString("The user has been deleted successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
accs, err := c.iam.ListUserAccounts()
|
||||
return SendXMLResponse(ctx,
|
||||
auth.ListUserAccountsResult{
|
||||
Accounts: accs,
|
||||
}, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminListUsers,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.JSON(accs)
|
||||
}
|
||||
|
||||
func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
owner := ctx.Query("owner")
|
||||
bucket := ctx.Query("bucket")
|
||||
|
||||
accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
|
||||
if err != nil {
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return err
|
||||
}
|
||||
if len(accs) > 0 {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return fmt.Errorf("user specified as the new bucket owner does not exist")
|
||||
}
|
||||
|
||||
acl := auth.ACL{
|
||||
Owner: owner,
|
||||
Grantees: []auth.Grantee{
|
||||
{
|
||||
Permission: auth.PermissionFullControl,
|
||||
Access: owner,
|
||||
Type: types.TypeCanonicalUser,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
aclParsed, err := json.Marshal(acl)
|
||||
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
|
||||
if err != nil {
|
||||
return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
|
||||
return SendResponse(ctx, err,
|
||||
&MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminChangeBucketOwner,
|
||||
})
|
||||
return ctx.Status(201).SendString("Bucket owner has been updated successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
|
||||
buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
|
||||
return SendXMLResponse(ctx,
|
||||
s3response.ListBucketsResult{
|
||||
Buckets: buckets,
|
||||
}, err, &MetaOpts{
|
||||
Logger: c.l,
|
||||
Action: metrics.ActionAdminListBuckets,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.JSON(buckets)
|
||||
}
|
||||
|
||||
@@ -15,11 +15,12 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
@@ -42,26 +43,33 @@ func TestAdminController_CreateUser(t *testing.T) {
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
succUser := `
|
||||
<Account>
|
||||
<Access>access</Access>
|
||||
<Secret>secret</Secret>
|
||||
<Role>admin</Role>
|
||||
<UserID>0</UserID>
|
||||
<GroupID>0</GroupID>
|
||||
</Account>
|
||||
`
|
||||
invuser := `
|
||||
<Account>
|
||||
<Access>access</Access>
|
||||
<Secret>secret</Secret>
|
||||
<Role>invalid_role</Role>
|
||||
<UserID>0</UserID>
|
||||
<GroupID>0</GroupID>
|
||||
</Account>
|
||||
`
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
usr := auth.Account{
|
||||
Access: "access",
|
||||
Secret: "secret",
|
||||
Role: "invalid role",
|
||||
}
|
||||
|
||||
user, _ := json.Marshal(&usr)
|
||||
|
||||
usr.Role = "admin"
|
||||
|
||||
succUsr, _ := json.Marshal(&usr)
|
||||
|
||||
appErr.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -71,31 +79,31 @@ func TestAdminController_CreateUser(t *testing.T) {
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-create-user-malformed-body",
|
||||
name: "Admin-create-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-user-role",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-requester-role",
|
||||
app: appErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-requester-role",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 201,
|
||||
statusCode: 500,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -111,100 +119,6 @@ func TestAdminController_CreateUser(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminController_UpdateUser(t *testing.T) {
|
||||
type args struct {
|
||||
req *http.Request
|
||||
}
|
||||
|
||||
adminController := AdminController{
|
||||
iam: &IAMServiceMock{
|
||||
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Patch("/update-user", adminController.UpdateUser)
|
||||
|
||||
adminControllerErr := AdminController{
|
||||
iam: &IAMServiceMock{
|
||||
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
|
||||
return auth.ErrNoSuchUser
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
appNotFound := fiber.New()
|
||||
|
||||
appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
|
||||
|
||||
succUser := `
|
||||
<Account>
|
||||
<Secret>secret</Secret>
|
||||
<UserID>0</UserID>
|
||||
<GroupID>0</GroupID>
|
||||
</Account>
|
||||
`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
args args
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-update-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-update-user-missing-access",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
},
|
||||
{
|
||||
name: "Admin-update-user-invalid-request-body",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Admin-update-user-not-found",
|
||||
app: appNotFound,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
if resp.StatusCode != tt.statusCode {
|
||||
t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminController_DeleteUser(t *testing.T) {
|
||||
type args struct {
|
||||
req *http.Request
|
||||
@@ -220,8 +134,22 @@ func TestAdminController_DeleteUser(t *testing.T) {
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appErr.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -238,6 +166,15 @@ func TestAdminController_DeleteUser(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-delete-user-invalid-requester-role",
|
||||
app: appErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
@@ -274,9 +211,30 @@ func TestAdminController_ListUsers(t *testing.T) {
|
||||
}
|
||||
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appErr.Patch("/list-users", adminControllerErr.ListUsers)
|
||||
|
||||
appRoleErr := fiber.New()
|
||||
|
||||
appRoleErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appRoleErr.Patch("/list-users", adminController.ListUsers)
|
||||
|
||||
appSucc := fiber.New()
|
||||
|
||||
appSucc.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appSucc.Patch("/list-users", adminController.ListUsers)
|
||||
|
||||
tests := []struct {
|
||||
@@ -286,6 +244,15 @@ func TestAdminController_ListUsers(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-list-users-access-denied",
|
||||
app: appRoleErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Admin-list-users-iam-error",
|
||||
app: appErr,
|
||||
@@ -324,7 +291,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
}
|
||||
adminController := AdminController{
|
||||
be: &BackendMock{
|
||||
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
|
||||
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
@@ -352,12 +319,39 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
|
||||
|
||||
appRoleErr := fiber.New()
|
||||
|
||||
appRoleErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
|
||||
|
||||
appIamErr := fiber.New()
|
||||
|
||||
appIamErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
|
||||
|
||||
appIamNoSuchUser := fiber.New()
|
||||
|
||||
appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
|
||||
|
||||
tests := []struct {
|
||||
@@ -367,6 +361,15 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Change-bucket-owner-access-denied",
|
||||
app: appRoleErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Change-bucket-owner-check-account-server-error",
|
||||
app: appIamErr,
|
||||
@@ -383,7 +386,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Change-bucket-owner-success",
|
||||
@@ -392,7 +395,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
|
||||
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
statusCode: 201,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -421,8 +424,23 @@ func TestAdminController_ListBuckets(t *testing.T) {
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/list-buckets", adminController.ListBuckets)
|
||||
|
||||
appRoleErr := fiber.New()
|
||||
|
||||
appRoleErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -430,6 +448,15 @@ func TestAdminController_ListBuckets(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "List-buckets-incorrect-role",
|
||||
app: appRoleErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "List-buckets-success",
|
||||
app: app,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
@@ -76,7 +77,7 @@ func TestNew(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false)
|
||||
got := New(tt.args.be, tt.args.iam, nil, nil, false)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("New() = %v, want %v", got, tt.want)
|
||||
}
|
||||
@@ -92,7 +93,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, nil
|
||||
},
|
||||
},
|
||||
@@ -109,7 +110,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
|
||||
appErr := fiber.New()
|
||||
s3ApiControllerErr := S3ApiController{
|
||||
be: &BackendMock{
|
||||
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
|
||||
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
|
||||
},
|
||||
},
|
||||
@@ -187,10 +188,10 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
|
||||
return &s3.GetObjectAclOutput{}, nil
|
||||
},
|
||||
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
|
||||
return s3response.GetObjectAttributesResponse{}, nil
|
||||
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
return &s3.GetObjectAttributesOutput{}, nil
|
||||
},
|
||||
GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
|
||||
return &s3.GetObjectOutput{
|
||||
Metadata: map[string]string{"hello": "world"},
|
||||
ContentType: getPtr("application/xml"),
|
||||
@@ -204,19 +205,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
|
||||
return map[string]string{"hello": "world"}, nil
|
||||
},
|
||||
GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
result, err := json.Marshal(types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeCompliance,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
},
|
||||
GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
result := true
|
||||
return &result, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
@@ -248,24 +236,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-get-object-retention-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?retention", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-get-object-legal-hold-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?legal-hold", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-actions-invalid-max-parts-string",
|
||||
app: app,
|
||||
@@ -359,11 +329,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
req *http.Request
|
||||
}
|
||||
|
||||
objectLockResult, err := json.Marshal(auth.BucketLockConfig{})
|
||||
if err != nil {
|
||||
t.Errorf("failed to parse object lock result %v", err)
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
@@ -373,30 +338,24 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
return s3response.ListMultipartUploadsResult{}, nil
|
||||
},
|
||||
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
|
||||
return s3response.ListObjectsV2Result{}, nil
|
||||
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return &s3.ListObjectsV2Output{}, nil
|
||||
},
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
return s3response.ListObjectsResult{}, nil
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return &s3.ListObjectsOutput{}, nil
|
||||
},
|
||||
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
return map[string]string{}, nil
|
||||
},
|
||||
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
return s3response.GetBucketVersioningOutput{}, nil
|
||||
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
|
||||
return &s3.GetBucketVersioningOutput{}, nil
|
||||
},
|
||||
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
|
||||
return s3response.ListVersionsResult{}, nil
|
||||
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
|
||||
return &s3.ListObjectVersionsOutput{}, nil
|
||||
},
|
||||
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return []byte{}, nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return objectLockResult, nil
|
||||
},
|
||||
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
return types.ObjectOwnershipBucketOwnerEnforced, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -410,14 +369,14 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
|
||||
app.Get("/:bucket", s3ApiController.ListActions)
|
||||
|
||||
// Error case
|
||||
//Error case
|
||||
s3ApiControllerError := S3ApiController{
|
||||
be: &BackendMock{
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return acldata, nil
|
||||
},
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
|
||||
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
@@ -450,15 +409,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-ownership-control-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-tagging-success",
|
||||
app: app,
|
||||
@@ -468,15 +418,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-object-lock-configuration-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket?object-lock", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-acl-success",
|
||||
app: app,
|
||||
@@ -573,7 +514,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
app := fiber.New()
|
||||
|
||||
// Mock valid acl
|
||||
acl := auth.ACL{Owner: "valid access"}
|
||||
acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
|
||||
acldata, err := json.Marshal(acl)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to parse the params: %v", err.Error())
|
||||
@@ -604,6 +545,14 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
</AccessControlPolicy>
|
||||
`
|
||||
|
||||
succBody := `
|
||||
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Owner>
|
||||
<ID>valid access</ID>
|
||||
</Owner>
|
||||
</AccessControlPolicy>
|
||||
`
|
||||
|
||||
tagBody := `
|
||||
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<TagSet>
|
||||
@@ -635,34 +584,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
}
|
||||
`
|
||||
|
||||
objectLockBody := `
|
||||
<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<ObjectLockEnabled>Enabled</ObjectLockEnabled>
|
||||
<Rule>
|
||||
<DefaultRetention>
|
||||
<Mode>GOVERNANCE</Mode>
|
||||
<Years>2</Years>
|
||||
</DefaultRetention>
|
||||
</Rule>
|
||||
</ObjectLockConfiguration>
|
||||
`
|
||||
|
||||
ownershipBody := `
|
||||
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Rule>
|
||||
<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
|
||||
</Rule>
|
||||
</OwnershipControls>
|
||||
`
|
||||
|
||||
invalidOwnershipBody := `
|
||||
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Rule>
|
||||
<ObjectOwnership>invalid_value</ObjectOwnership>
|
||||
</Rule>
|
||||
</OwnershipControls>
|
||||
`
|
||||
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
@@ -677,21 +598,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
|
||||
return nil
|
||||
},
|
||||
PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
|
||||
PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
|
||||
return nil
|
||||
},
|
||||
PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
|
||||
return nil
|
||||
},
|
||||
PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
|
||||
return nil
|
||||
},
|
||||
PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
|
||||
return nil
|
||||
},
|
||||
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
|
||||
return types.ObjectOwnershipBucketOwnerPreferred, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
// Mock ctx.Locals
|
||||
@@ -715,18 +627,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
|
||||
// PutBucketAcl incorrect bucket owner case
|
||||
incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
|
||||
incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
|
||||
|
||||
// PutBucketAcl acl success
|
||||
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
|
||||
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
|
||||
aclSuccReq.Header.Set("X-Amz-Acl", "private")
|
||||
|
||||
// Invalid acl body case
|
||||
errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
|
||||
errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
|
||||
|
||||
invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
|
||||
invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
@@ -750,42 +660,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-ownership-controls-invalid-ownership",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-ownership-controls-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Put-object-lock-configuration-invalid-body",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Put-object-lock-configuration-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", strings.NewReader(objectLockBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
@@ -858,7 +732,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
req: incorrectBucketOwner,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
statusCode: 403,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-acl-success",
|
||||
@@ -870,16 +744,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Create-bucket-invalid-acl-ownership-combination",
|
||||
app: app,
|
||||
args: args{
|
||||
req: invAclOwnershipReq,
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Create-bucket-invalid-bucket-name",
|
||||
name: "Put-bucket-invalid-bucket-name",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/aa", nil),
|
||||
@@ -888,7 +753,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Create-bucket-success",
|
||||
name: "Put-bucket-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
|
||||
@@ -941,19 +806,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
</Tagging>
|
||||
`
|
||||
|
||||
//retentionBody := `
|
||||
//<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
// <Mode>GOVERNANCE</Mode>
|
||||
// <RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
|
||||
//</Retention>
|
||||
//`
|
||||
|
||||
legalHoldBody := `
|
||||
<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>ON</Status>
|
||||
</LegalHold>
|
||||
`
|
||||
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
@@ -968,8 +820,8 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
CopyObjectResult: &types.CopyObjectResult{},
|
||||
}, nil
|
||||
},
|
||||
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
return s3response.PutObjectOutput{}, nil
|
||||
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
|
||||
return "ETag", nil
|
||||
},
|
||||
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
|
||||
return "hello", nil
|
||||
@@ -980,15 +832,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
return s3response.CopyObjectResult{}, nil
|
||||
},
|
||||
PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
|
||||
return nil
|
||||
},
|
||||
PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
|
||||
return nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
@@ -1067,42 +910,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "put-object-retention-invalid-request",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
//{
|
||||
// name: "put-object-retention-success",
|
||||
// app: app,
|
||||
// args: args{
|
||||
// req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
|
||||
// },
|
||||
// wantErr: false,
|
||||
// statusCode: 200,
|
||||
//},
|
||||
{
|
||||
name: "put-legal-hold-invalid-request",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "put-legal-hold-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", strings.NewReader(legalHoldBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Put-object-acl-invalid-acl",
|
||||
app: app,
|
||||
@@ -1217,18 +1024,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
DeleteBucketFunc: func(_ context.Context, bucket string) error {
|
||||
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
|
||||
return nil
|
||||
},
|
||||
DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
return nil
|
||||
},
|
||||
DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
return nil
|
||||
},
|
||||
DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1267,23 +1068,6 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
{
|
||||
name: "Delete-bucket-ownership-controls-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
}, {
|
||||
name: "Delete-bucket-policy-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
@@ -1312,9 +1096,6 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
|
||||
DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
return s3response.DeleteResult{}, nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1383,8 +1164,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return acldata, nil
|
||||
},
|
||||
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
|
||||
return nil
|
||||
},
|
||||
AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
|
||||
return nil
|
||||
@@ -1392,9 +1173,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
|
||||
DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error {
|
||||
return nil
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1414,11 +1192,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return acldata, nil
|
||||
},
|
||||
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
|
||||
return s3err.GetAPIError(7)
|
||||
},
|
||||
}}
|
||||
|
||||
@@ -1510,7 +1285,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
|
||||
ctx.Locals("isRoot", true)
|
||||
ctx.Locals("isDebug", false)
|
||||
ctx.Locals("parsedAcl", auth.ACL{})
|
||||
ctx.Locals("region", "us-east-1")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
@@ -1534,7 +1308,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
|
||||
ctx.Locals("isRoot", true)
|
||||
ctx.Locals("isDebug", false)
|
||||
ctx.Locals("parsedAcl", auth.ACL{})
|
||||
ctx.Locals("region", "us-east-1")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
@@ -1628,7 +1401,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
|
||||
return acldata, nil
|
||||
},
|
||||
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
return nil, s3err.GetAPIError(42)
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1697,8 +1470,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
return &s3.CompleteMultipartUploadOutput{}, nil
|
||||
},
|
||||
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
|
||||
return s3response.InitiateMultipartUploadResult{}, nil
|
||||
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return &s3.CreateMultipartUploadOutput{}, nil
|
||||
},
|
||||
SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
return func(w *bufio.Writer) {}
|
||||
@@ -1713,19 +1486,6 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
</SelectObjectContentRequest>
|
||||
`
|
||||
|
||||
completMpBody := `
|
||||
<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Part>
|
||||
<ETag>etag</ETag>
|
||||
<PartNumber>1</PartNumber>
|
||||
</Part>
|
||||
</CompleteMultipartUpload>
|
||||
`
|
||||
|
||||
completMpEmptyBody := `
|
||||
<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></CompleteMultipartUpload>
|
||||
`
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("account", auth.Account{Access: "valid access"})
|
||||
ctx.Locals("isRoot", true)
|
||||
@@ -1746,11 +1506,20 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
name: "Restore-object-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Restore-object-error",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Select-object-content-invalid-body",
|
||||
app: app,
|
||||
@@ -1778,20 +1547,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Complete-multipart-upload-empty-parts",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpEmptyBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Complete-multipart-upload-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpBody)),
|
||||
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(`<root><key>body</key></root>`)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
|
||||
@@ -33,9 +33,6 @@ var _ auth.IAMService = &IAMServiceMock{}
|
||||
// ShutdownFunc: func() error {
|
||||
// panic("mock out the Shutdown method")
|
||||
// },
|
||||
// UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
|
||||
// panic("mock out the UpdateUserAccount method")
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// // use mockedIAMService in code that requires auth.IAMService
|
||||
@@ -58,9 +55,6 @@ type IAMServiceMock struct {
|
||||
// ShutdownFunc mocks the Shutdown method.
|
||||
ShutdownFunc func() error
|
||||
|
||||
// UpdateUserAccountFunc mocks the UpdateUserAccount method.
|
||||
UpdateUserAccountFunc func(access string, props auth.MutableProps) error
|
||||
|
||||
// calls tracks calls to the methods.
|
||||
calls struct {
|
||||
// CreateAccount holds details about calls to the CreateAccount method.
|
||||
@@ -84,20 +78,12 @@ type IAMServiceMock struct {
|
||||
// Shutdown holds details about calls to the Shutdown method.
|
||||
Shutdown []struct {
|
||||
}
|
||||
// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
|
||||
UpdateUserAccount []struct {
|
||||
// Access is the access argument value.
|
||||
Access string
|
||||
// Props is the props argument value.
|
||||
Props auth.MutableProps
|
||||
}
|
||||
}
|
||||
lockCreateAccount sync.RWMutex
|
||||
lockDeleteUserAccount sync.RWMutex
|
||||
lockGetUserAccount sync.RWMutex
|
||||
lockListUserAccounts sync.RWMutex
|
||||
lockShutdown sync.RWMutex
|
||||
lockUpdateUserAccount sync.RWMutex
|
||||
}
|
||||
|
||||
// CreateAccount calls CreateAccountFunc.
|
||||
@@ -249,39 +235,3 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
|
||||
mock.lockShutdown.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
// UpdateUserAccount calls UpdateUserAccountFunc.
|
||||
func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
|
||||
if mock.UpdateUserAccountFunc == nil {
|
||||
panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
Access string
|
||||
Props auth.MutableProps
|
||||
}{
|
||||
Access: access,
|
||||
Props: props,
|
||||
}
|
||||
mock.lockUpdateUserAccount.Lock()
|
||||
mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
|
||||
mock.lockUpdateUserAccount.Unlock()
|
||||
return mock.UpdateUserAccountFunc(access, props)
|
||||
}
|
||||
|
||||
// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedIAMService.UpdateUserAccountCalls())
|
||||
func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
|
||||
Access string
|
||||
Props auth.MutableProps
|
||||
} {
|
||||
var calls []struct {
|
||||
Access string
|
||||
Props auth.MutableProps
|
||||
}
|
||||
mock.lockUpdateUserAccount.RLock()
|
||||
calls = mock.calls.UpdateUserAccount
|
||||
mock.lockUpdateUserAccount.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
@@ -32,7 +31,7 @@ var (
|
||||
singlePath = regexp.MustCompile(`^/[^/]+/?$`)
|
||||
)
|
||||
|
||||
func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
|
||||
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
|
||||
path := ctx.Path()
|
||||
@@ -49,19 +48,10 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
|
||||
!ctx.Request().URI().QueryArgs().Has("acl") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("tagging") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("versioning") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("policy") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("object-lock") &&
|
||||
!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
|
||||
!ctx.Request().URI().QueryArgs().Has("policy") {
|
||||
if err := auth.MayCreateBucket(acct, isRoot); err != nil {
|
||||
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
|
||||
}
|
||||
if readonly {
|
||||
return controllers.SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrAccessDenied),
|
||||
&controllers.MetaOpts{
|
||||
Logger: logger,
|
||||
Action: "CreateBucket",
|
||||
})
|
||||
}
|
||||
return ctx.Next()
|
||||
}
|
||||
data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != auth.RoleAdmin {
|
||||
path := ctx.Path()
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
|
||||
&controllers.MetaOpts{
|
||||
Logger: logger,
|
||||
Action: detectAction(path),
|
||||
})
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func detectAction(path string) (action string) {
|
||||
if strings.Contains(path, "create-user") {
|
||||
action = metrics.ActionAdminCreateUser
|
||||
} else if strings.Contains(path, "update-user") {
|
||||
action = metrics.ActionAdminUpdateUser
|
||||
} else if strings.Contains(path, "delete-user") {
|
||||
action = metrics.ActionAdminDeleteUser
|
||||
} else if strings.Contains(path, "list-user") {
|
||||
action = metrics.ActionAdminListUsers
|
||||
} else if strings.Contains(path, "list-buckets") {
|
||||
action = metrics.ActionAdminListBuckets
|
||||
} else if strings.Contains(path, "change-bucket-owner") {
|
||||
action = metrics.ActionAdminChangeBucketOwner
|
||||
}
|
||||
return action
|
||||
}
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
@@ -41,7 +40,7 @@ type RootUserConfig struct {
|
||||
Secret string
|
||||
}
|
||||
|
||||
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
|
||||
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
@@ -55,16 +54,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
ctx.Locals("startTime", time.Now())
|
||||
authorization := ctx.Get("Authorization")
|
||||
if authorization == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
|
||||
}
|
||||
|
||||
authData, err := utils.ParseAuthorization(authorization)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
if authData.Algorithm != "AWS4-HMAC-SHA256" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
|
||||
}
|
||||
|
||||
if authData.Region != region {
|
||||
@@ -72,40 +71,40 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}, logger, mm)
|
||||
}, logger)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
// Check X-Amz-Date header
|
||||
date := ctx.Get("X-Amz-Date")
|
||||
if date == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
|
||||
}
|
||||
|
||||
// Parse the date and check the date validity
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
|
||||
}
|
||||
|
||||
if date[:8] != authData.Date {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
|
||||
}
|
||||
|
||||
// Validate the dates difference
|
||||
err = utils.ValidateDate(tdate)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
@@ -126,7 +125,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
|
||||
// Compare the calculated hash with the hash provided
|
||||
if hashPayload != hexPayload {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,13 +134,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
|
||||
}
|
||||
}
|
||||
|
||||
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
@@ -165,6 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
|
||||
return a.iam.GetUserAccount(access)
|
||||
}
|
||||
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
@@ -20,14 +20,13 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
// ProcessChunkedBody initializes the chunked upload stream if the
|
||||
// request appears to be a chunked upload
|
||||
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
|
||||
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
|
||||
if decodedLength == "" {
|
||||
@@ -37,7 +36,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
|
||||
|
||||
authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
@@ -47,12 +46,12 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
var err error
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
var cr io.Reader
|
||||
var cr *utils.ChunkReader
|
||||
cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
|
||||
return cr
|
||||
})
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
@@ -20,13 +20,12 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
@@ -39,16 +38,16 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
|
||||
|
||||
authData, err := utils.ParsePresignedURIParts(ctx)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
@@ -62,7 +61,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
|
||||
|
||||
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
|
||||
@@ -18,19 +18,19 @@ import (
|
||||
"net/url"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
|
||||
func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
|
||||
reqURL := ctx.Request().URI().String()
|
||||
decoded, err := url.Parse(reqURL)
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
ctx.Path(unescp)
|
||||
ctx.Path(decoded.Path)
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,9 +18,7 @@ import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
@@ -29,29 +27,26 @@ type S3ApiRouter struct {
|
||||
WithAdmSrv bool
|
||||
}
|
||||
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
|
||||
s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
|
||||
s3ApiController := controllers.New(be, iam, logger, evs, debug)
|
||||
|
||||
if sa.WithAdmSrv {
|
||||
adminController := controllers.NewAdminController(iam, be, aLogger)
|
||||
adminController := controllers.NewAdminController(iam, be)
|
||||
|
||||
// CreateUser admin api
|
||||
app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
|
||||
app.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
// DeleteUsers admin api
|
||||
app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
|
||||
|
||||
// UpdateUser admin api
|
||||
app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
|
||||
app.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
// ListUsers admin api
|
||||
app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
|
||||
app.Patch("/list-users", adminController.ListUsers)
|
||||
|
||||
// ChangeBucketOwner admin api
|
||||
app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
|
||||
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
|
||||
|
||||
// ListBucketsAndOwners admin api
|
||||
app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
|
||||
app.Patch("/list-buckets", adminController.ListBuckets)
|
||||
}
|
||||
|
||||
// ListBuckets action
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false)
|
||||
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,36 +22,23 @@ import (
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
type S3ApiServer struct {
|
||||
app *fiber.App
|
||||
backend backend.Backend
|
||||
router *S3ApiRouter
|
||||
port string
|
||||
cert *tls.Certificate
|
||||
quiet bool
|
||||
debug bool
|
||||
readonly bool
|
||||
health string
|
||||
app *fiber.App
|
||||
backend backend.Backend
|
||||
router *S3ApiRouter
|
||||
port string
|
||||
cert *tls.Certificate
|
||||
quiet bool
|
||||
debug bool
|
||||
health string
|
||||
}
|
||||
|
||||
func New(
|
||||
app *fiber.App,
|
||||
be backend.Backend,
|
||||
root middlewares.RootUserConfig,
|
||||
port, region string,
|
||||
iam auth.IAMService,
|
||||
l s3log.AuditLogger,
|
||||
adminLogger s3log.AuditLogger,
|
||||
evs s3event.S3EventSender,
|
||||
mm *metrics.Manager,
|
||||
opts ...Option,
|
||||
) (*S3ApiServer, error) {
|
||||
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
|
||||
server := &S3ApiServer{
|
||||
app: app,
|
||||
backend: be,
|
||||
@@ -65,9 +52,7 @@ func New(
|
||||
|
||||
// Logging middlewares
|
||||
if !server.quiet {
|
||||
app.Use(logger.New(logger.Config{
|
||||
Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
|
||||
}))
|
||||
app.Use(logger.New())
|
||||
}
|
||||
// Set up health endpoint if specified
|
||||
if server.health != "" {
|
||||
@@ -75,17 +60,17 @@ func New(
|
||||
return ctx.SendStatus(http.StatusOK)
|
||||
})
|
||||
}
|
||||
app.Use(middlewares.DecodeURL(l, mm))
|
||||
app.Use(middlewares.DecodeURL(l))
|
||||
app.Use(middlewares.RequestLogger(server.debug))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
|
||||
app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
app.Use(middlewares.AclParser(be, l, server.readonly))
|
||||
app.Use(middlewares.AclParser(be, l))
|
||||
|
||||
server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly)
|
||||
server.router.Init(app, be, iam, l, evs, server.debug)
|
||||
|
||||
return server, nil
|
||||
}
|
||||
@@ -118,10 +103,6 @@ func WithHealth(health string) Option {
|
||||
return func(s *S3ApiServer) { s.health = health }
|
||||
}
|
||||
|
||||
func WithReadOnly() Option {
|
||||
return func(s *S3ApiServer) { s.readonly = true }
|
||||
}
|
||||
|
||||
func (sa *S3ApiServer) Serve() (err error) {
|
||||
if sa.cert != nil {
|
||||
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
|
||||
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil, nil)
|
||||
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
|
||||
@@ -260,3 +260,19 @@ func removeSpace(str string) string {
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
var (
|
||||
specialValues = map[string]bool{
|
||||
"UNSIGNED-PAYLOAD": true,
|
||||
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
|
||||
}
|
||||
)
|
||||
|
||||
// IsSpecialPayload checks for streaming/unsigned authorization types
|
||||
func IsSpecialPayload(str string) bool {
|
||||
return specialValues[str]
|
||||
}
|
||||
|
||||
@@ -1,17 +1,3 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
|
||||
@@ -15,85 +15,255 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type payloadType string
|
||||
// chunked uploads described in:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
|
||||
|
||||
const (
|
||||
payloadTypeUnsigned payloadType = "UNSIGNED-PAYLOAD"
|
||||
payloadTypeStreamingUnsignedTrailer payloadType = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
|
||||
payloadTypeStreamingSigned payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
payloadTypeStreamingSignedTrailer payloadType = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
|
||||
payloadTypeStreamingEcdsa payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
|
||||
payloadTypeStreamingEcdsaTrailer payloadType = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
|
||||
chunkHdrStr = ";chunk-signature="
|
||||
chunkHdrDelim = "\r\n"
|
||||
zeroLenSig = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
awsV4 = "AWS4"
|
||||
awsS3Service = "s3"
|
||||
awsV4Request = "aws4_request"
|
||||
streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
|
||||
)
|
||||
|
||||
// ChunkReader reads from chunked upload request body, and returns
|
||||
// object data stream
|
||||
type ChunkReader struct {
|
||||
r io.Reader
|
||||
signingKey []byte
|
||||
prevSig string
|
||||
parsedSig string
|
||||
currentChunkSize int64
|
||||
chunkDataLeft int64
|
||||
trailerExpected int
|
||||
stash []byte
|
||||
chunkHash hash.Hash
|
||||
strToSignPrefix string
|
||||
skipcheck bool
|
||||
}
|
||||
|
||||
// NewChunkReader reads from request body io.Reader and parses out the
|
||||
// chunk metadata in stream. The headers are validated for proper signatures.
|
||||
// Reading from the chunk reader will read only the object data stream
|
||||
// without the chunk headers/trailers.
|
||||
func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (*ChunkReader, error) {
|
||||
return &ChunkReader{
|
||||
r: r,
|
||||
signingKey: getSigningKey(secret, region, date),
|
||||
// the authdata.Signature is validated in the auth-reader,
|
||||
// so we can use that here without any other checks
|
||||
prevSig: authdata.Signature,
|
||||
chunkHash: sha256.New(),
|
||||
strToSignPrefix: getStringToSignPrefix(date, region),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Read satisfies the io.Reader for this type
|
||||
func (cr *ChunkReader) Read(p []byte) (int, error) {
|
||||
n, err := cr.r.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if cr.chunkDataLeft < int64(n) {
|
||||
chunkSize := cr.chunkDataLeft
|
||||
if chunkSize > 0 {
|
||||
cr.chunkHash.Write(p[:chunkSize])
|
||||
}
|
||||
n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
|
||||
n += int(chunkSize)
|
||||
return n, err
|
||||
}
|
||||
|
||||
cr.chunkDataLeft -= int64(n)
|
||||
cr.chunkHash.Write(p[:n])
|
||||
return n, err
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
|
||||
// This part is the same for all chunks,
|
||||
// only the previous signature and hash of current chunk changes
|
||||
func getStringToSignPrefix(date time.Time, region string) string {
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/%s",
|
||||
date.Format("20060102"),
|
||||
region,
|
||||
awsS3Service,
|
||||
awsV4Request)
|
||||
|
||||
return fmt.Sprintf("%s\n%s\n%s",
|
||||
streamPayloadAlgo,
|
||||
date.Format("20060102T150405Z"),
|
||||
credentialScope)
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
|
||||
// signature For each chunk, you calculate the signature using the following
|
||||
// string to sign. For the first chunk, you use the seed-signature as the
|
||||
// previous signature.
|
||||
func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
|
||||
return fmt.Sprintf("%s\n%s\n%s\n%s",
|
||||
prefix,
|
||||
prevSig,
|
||||
zeroLenSig,
|
||||
hex.EncodeToString(chunkHash))
|
||||
}
|
||||
|
||||
// The provided p should have all of the previous chunk data and trailer
|
||||
// consumed already. The positioning here is expected that p[0] starts the
|
||||
// new chunk size with the ";chunk-signature=" following. The only exception
|
||||
// is if we started consuming the trailer, but hit the end of the read buffer.
|
||||
// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
|
||||
// finish consuming the final trailer bytes.
|
||||
// This parses the chunk metadata in situ without allocating an extra buffer.
|
||||
// It will just read and validate the chunk metadata and then move the
|
||||
// following chunk data to overwrite the metadata in the provided buffer.
|
||||
func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
|
||||
n := len(p)
|
||||
|
||||
if !cr.skipcheck && cr.parsedSig != "" {
|
||||
chunkhash := cr.chunkHash.Sum(nil)
|
||||
cr.chunkHash.Reset()
|
||||
|
||||
sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
|
||||
cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
|
||||
|
||||
if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
|
||||
return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
}
|
||||
|
||||
if cr.trailerExpected != 0 {
|
||||
if len(p) < len(chunkHdrDelim) {
|
||||
// This is the special case where we need to consume the
|
||||
// trailer, but instead hit the end of the buffer. The
|
||||
// subsequent call will finish consuming the trailer.
|
||||
cr.chunkDataLeft = 0
|
||||
cr.trailerExpected -= len(p)
|
||||
cr.skipcheck = true
|
||||
return 0, nil
|
||||
}
|
||||
// move data up to remove trailer
|
||||
copy(p, p[cr.trailerExpected:])
|
||||
n -= cr.trailerExpected
|
||||
}
|
||||
|
||||
cr.skipcheck = false
|
||||
|
||||
chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
|
||||
cr.currentChunkSize = chunkSize
|
||||
cr.parsedSig = sig
|
||||
if err == errskipHeader {
|
||||
cr.chunkDataLeft = 0
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if chunkSize == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
cr.trailerExpected = len(chunkHdrDelim)
|
||||
|
||||
// move data up to remove chunk header
|
||||
copy(p, p[bufOffset:n])
|
||||
n -= bufOffset
|
||||
|
||||
// if remaining buffer larger than chunk data,
|
||||
// parse next header in buffer
|
||||
if int64(n) > chunkSize {
|
||||
cr.chunkDataLeft = 0
|
||||
cr.chunkHash.Write(p[:chunkSize])
|
||||
n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
|
||||
return n + int(chunkSize), err
|
||||
} else {
|
||||
cr.chunkDataLeft = chunkSize - int64(n)
|
||||
cr.chunkHash.Write(p[:n])
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
||||
// Task 3: Calculate Signature
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
|
||||
func getSigningKey(secret, region string, date time.Time) []byte {
|
||||
dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
|
||||
dateRegionKey := hmac256(dateKey, []byte(region))
|
||||
dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
|
||||
signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
|
||||
return signingKey
|
||||
}
|
||||
|
||||
func hmac256(key []byte, data []byte) []byte {
|
||||
hash := hmac.New(sha256.New, key)
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
var (
|
||||
specialValues = map[payloadType]bool{
|
||||
payloadTypeUnsigned: true,
|
||||
payloadTypeStreamingUnsignedTrailer: true,
|
||||
payloadTypeStreamingSigned: true,
|
||||
payloadTypeStreamingSignedTrailer: true,
|
||||
payloadTypeStreamingEcdsa: true,
|
||||
payloadTypeStreamingEcdsaTrailer: true,
|
||||
}
|
||||
errInvalidChunkFormat = errors.New("invalid chunk header format")
|
||||
errskipHeader = errors.New("skip to next header")
|
||||
)
|
||||
|
||||
func (pt payloadType) isValid() bool {
|
||||
return pt == payloadTypeUnsigned ||
|
||||
pt == payloadTypeStreamingUnsignedTrailer ||
|
||||
pt == payloadTypeStreamingSigned ||
|
||||
pt == payloadTypeStreamingSignedTrailer ||
|
||||
pt == payloadTypeStreamingEcdsa ||
|
||||
pt == payloadTypeStreamingEcdsaTrailer
|
||||
}
|
||||
|
||||
type checksumType string
|
||||
|
||||
const (
|
||||
checksumTypeCrc32 checksumType = "x-amz-checksum-crc32"
|
||||
checksumTypeCrc32c checksumType = "x-amz-checksum-crc32c"
|
||||
checksumTypeSha1 checksumType = "x-amz-checksum-sha1"
|
||||
checksumTypeSha256 checksumType = "x-amz-checksum-sha256"
|
||||
checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
|
||||
maxHeaderSize = 1024
|
||||
)
|
||||
|
||||
func (c checksumType) isValid() bool {
|
||||
return c == checksumTypeCrc32 ||
|
||||
c == checksumTypeCrc32c ||
|
||||
c == checksumTypeSha1 ||
|
||||
c == checksumTypeSha256 ||
|
||||
c == checksumTypeCrc64nvme
|
||||
}
|
||||
|
||||
// IsSpecialPayload checks for streaming/unsigned authorization types
|
||||
func IsSpecialPayload(str string) bool {
|
||||
return specialValues[payloadType(str)]
|
||||
}
|
||||
|
||||
func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
|
||||
contentSha256 := payloadType(ctx.Get("X-Amz-Content-Sha256"))
|
||||
if !contentSha256.isValid() {
|
||||
//TODO: Add proper APIError
|
||||
return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
|
||||
// Theis returns the chunk payload size, signature, data start offset, and
|
||||
// error if any. See the AWS documentation for the chunk header format. The
|
||||
// header[0] byte is expected to be the first byte of the chunk size here.
|
||||
func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
|
||||
if cr.stash != nil {
|
||||
tmp := make([]byte, maxHeaderSize)
|
||||
copy(tmp, cr.stash)
|
||||
copy(tmp[len(cr.stash):], header)
|
||||
header = tmp
|
||||
cr.stash = nil
|
||||
}
|
||||
|
||||
checksumType := checksumType(ctx.Get("X-Amz-Trailer"))
|
||||
if checksumType != "" && !checksumType.isValid() {
|
||||
//TODO: Add proper APIError
|
||||
return nil, fmt.Errorf("invalid X-Amz-Trailer: %v", checksumType)
|
||||
semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
|
||||
if semicolonIndex == -1 {
|
||||
cr.stash = make([]byte, len(header))
|
||||
copy(cr.stash, header)
|
||||
cr.trailerExpected = 0
|
||||
return 0, "", 0, errskipHeader
|
||||
}
|
||||
|
||||
switch contentSha256 {
|
||||
case payloadTypeStreamingUnsignedTrailer:
|
||||
return NewUnsignedChunkReader(r, checksumType)
|
||||
//TODO: Add other chunk readers
|
||||
sigIndex := semicolonIndex + len(chunkHdrStr)
|
||||
sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
|
||||
if sigEndIndex == -1 {
|
||||
cr.stash = make([]byte, len(header))
|
||||
copy(cr.stash, header)
|
||||
cr.trailerExpected = 0
|
||||
return 0, "", 0, errskipHeader
|
||||
}
|
||||
|
||||
return NewSignedChunkReader(r, authdata, region, secret, date)
|
||||
chunkSizeBytes := header[:semicolonIndex]
|
||||
chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
|
||||
if err != nil {
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
|
||||
signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
|
||||
dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
|
||||
|
||||
return chunkSize, signature, dataStartOffset, nil
|
||||
}
|
||||
|
||||
@@ -1,289 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
// chunked uploads described in:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
|
||||
|
||||
const (
|
||||
chunkHdrStr = ";chunk-signature="
|
||||
chunkHdrDelim = "\r\n"
|
||||
zeroLenSig = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
awsV4 = "AWS4"
|
||||
awsS3Service = "s3"
|
||||
awsV4Request = "aws4_request"
|
||||
streamPayloadAlgo = "AWS4-HMAC-SHA256-PAYLOAD"
|
||||
)
|
||||
|
||||
// ChunkReader reads from chunked upload request body, and returns
|
||||
// object data stream
|
||||
type ChunkReader struct {
|
||||
r io.Reader
|
||||
signingKey []byte
|
||||
prevSig string
|
||||
parsedSig string
|
||||
currentChunkSize int64
|
||||
chunkDataLeft int64
|
||||
trailerExpected int
|
||||
stash []byte
|
||||
chunkHash hash.Hash
|
||||
strToSignPrefix string
|
||||
skipcheck bool
|
||||
}
|
||||
|
||||
// NewChunkReader reads from request body io.Reader and parses out the
|
||||
// chunk metadata in stream. The headers are validated for proper signatures.
|
||||
// Reading from the chunk reader will read only the object data stream
|
||||
// without the chunk headers/trailers.
|
||||
func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
|
||||
return &ChunkReader{
|
||||
r: r,
|
||||
signingKey: getSigningKey(secret, region, date),
|
||||
// the authdata.Signature is validated in the auth-reader,
|
||||
// so we can use that here without any other checks
|
||||
prevSig: authdata.Signature,
|
||||
chunkHash: sha256.New(),
|
||||
strToSignPrefix: getStringToSignPrefix(date, region),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Read satisfies the io.Reader for this type
|
||||
func (cr *ChunkReader) Read(p []byte) (int, error) {
|
||||
n, err := cr.r.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
return cr.processBuffer(p[:n])
|
||||
}
|
||||
|
||||
func (cr *ChunkReader) processBuffer(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if cr.chunkDataLeft > 0 {
|
||||
dataRemaining := cr.chunkDataLeft
|
||||
if int64(len(p)) < cr.chunkDataLeft {
|
||||
dataRemaining = int64(len(p))
|
||||
}
|
||||
cr.chunkDataLeft -= dataRemaining
|
||||
cr.chunkHash.Write(p[:dataRemaining])
|
||||
n, err := cr.processBuffer(p[dataRemaining:])
|
||||
return int(dataRemaining) + n, err
|
||||
}
|
||||
|
||||
shrunk, err := cr.parseAndRemoveChunkInfo(p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
p = p[:len(p)-shrunk]
|
||||
dataRemaining := cr.chunkDataLeft
|
||||
if int64(len(p)) < cr.chunkDataLeft {
|
||||
dataRemaining = int64(len(p))
|
||||
}
|
||||
cr.chunkDataLeft -= dataRemaining
|
||||
cr.chunkHash.Write(p[:dataRemaining])
|
||||
|
||||
n, err := cr.processBuffer(p[dataRemaining:])
|
||||
return int(dataRemaining) + n, err
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
|
||||
// This part is the same for all chunks,
|
||||
// only the previous signature and hash of current chunk changes
|
||||
func getStringToSignPrefix(date time.Time, region string) string {
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/%s",
|
||||
date.Format("20060102"),
|
||||
region,
|
||||
awsS3Service,
|
||||
awsV4Request)
|
||||
|
||||
return fmt.Sprintf("%s\n%s\n%s",
|
||||
streamPayloadAlgo,
|
||||
date.Format("20060102T150405Z"),
|
||||
credentialScope)
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
|
||||
// signature For each chunk, you calculate the signature using the following
|
||||
// string to sign. For the first chunk, you use the seed-signature as the
|
||||
// previous signature.
|
||||
func getChunkStringToSign(prefix, prevSig string, chunkHash []byte) string {
|
||||
return fmt.Sprintf("%s\n%s\n%s\n%s",
|
||||
prefix,
|
||||
prevSig,
|
||||
zeroLenSig,
|
||||
hex.EncodeToString(chunkHash))
|
||||
}
|
||||
|
||||
// The provided p should have all of the previous chunk data and trailer
|
||||
// consumed already. The positioning here is expected that p[0] starts the
|
||||
// new chunk size with the ";chunk-signature=" following. The only exception
|
||||
// is if we started consuming the trailer, but hit the end of the read buffer.
|
||||
// In this case, parseAndRemoveChunkInfo is called with skipcheck=true to
|
||||
// finish consuming the final trailer bytes.
|
||||
// This parses the chunk metadata in situ without allocating an extra buffer.
|
||||
// It will just read and validate the chunk metadata and then move the
|
||||
// following chunk data to overwrite the metadata in the provided buffer.
|
||||
func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
|
||||
n := len(p)
|
||||
|
||||
if !cr.skipcheck && cr.parsedSig != "" {
|
||||
chunkhash := cr.chunkHash.Sum(nil)
|
||||
cr.chunkHash.Reset()
|
||||
|
||||
sigstr := getChunkStringToSign(cr.strToSignPrefix, cr.prevSig, chunkhash)
|
||||
cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
|
||||
|
||||
if cr.currentChunkSize != 0 && cr.prevSig != cr.parsedSig {
|
||||
return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
}
|
||||
|
||||
dataMoved := 0
|
||||
|
||||
if cr.trailerExpected != 0 {
|
||||
if len(p) < len(chunkHdrDelim) {
|
||||
// This is the special case where we need to consume the
|
||||
// trailer, but instead hit the end of the buffer. The
|
||||
// subsequent call will finish consuming the trailer.
|
||||
cr.chunkDataLeft = 0
|
||||
cr.trailerExpected -= len(p)
|
||||
cr.skipcheck = true
|
||||
return 0, nil
|
||||
}
|
||||
// move data up to remove trailer
|
||||
copy(p, p[cr.trailerExpected:])
|
||||
n -= cr.trailerExpected
|
||||
dataMoved += cr.trailerExpected
|
||||
}
|
||||
|
||||
cr.skipcheck = false
|
||||
|
||||
chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
|
||||
cr.chunkDataLeft = chunkSize
|
||||
cr.currentChunkSize = chunkSize
|
||||
cr.parsedSig = sig
|
||||
if err == errskipHeader {
|
||||
cr.chunkDataLeft = 0
|
||||
return dataMoved, nil
|
||||
}
|
||||
if err != nil {
|
||||
return dataMoved, err
|
||||
}
|
||||
if chunkSize == 0 {
|
||||
return dataMoved, io.EOF
|
||||
}
|
||||
|
||||
cr.trailerExpected = len(chunkHdrDelim)
|
||||
|
||||
// move data up to remove chunk header
|
||||
copy(p, p[bufOffset:n])
|
||||
n -= bufOffset
|
||||
dataMoved += bufOffset
|
||||
|
||||
return dataMoved, nil
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
||||
// Task 3: Calculate Signature
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
|
||||
func getSigningKey(secret, region string, date time.Time) []byte {
|
||||
dateKey := hmac256([]byte(awsV4+secret), []byte(date.Format(yyyymmdd)))
|
||||
dateRegionKey := hmac256(dateKey, []byte(region))
|
||||
dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
|
||||
signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
|
||||
return signingKey
|
||||
}
|
||||
|
||||
func hmac256(key []byte, data []byte) []byte {
|
||||
hash := hmac.New(sha256.New, key)
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidChunkFormat = errors.New("invalid chunk header format")
|
||||
errskipHeader = errors.New("skip to next header")
|
||||
)
|
||||
|
||||
const (
|
||||
maxHeaderSize = 1024
|
||||
)
|
||||
|
||||
// This returns the chunk payload size, signature, data start offset, and
|
||||
// error if any. See the AWS documentation for the chunk header format. The
|
||||
// header[0] byte is expected to be the first byte of the chunk size here.
|
||||
func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
|
||||
stashLen := len(cr.stash)
|
||||
if stashLen > maxHeaderSize {
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
if cr.stash != nil {
|
||||
tmp := make([]byte, maxHeaderSize)
|
||||
copy(tmp, cr.stash)
|
||||
copy(tmp[len(cr.stash):], header)
|
||||
header = tmp
|
||||
cr.stash = nil
|
||||
}
|
||||
|
||||
semicolonIndex := bytes.Index(header, []byte(chunkHdrStr))
|
||||
if semicolonIndex == -1 {
|
||||
cr.stash = make([]byte, len(header))
|
||||
copy(cr.stash, header)
|
||||
cr.trailerExpected = 0
|
||||
return 0, "", 0, errskipHeader
|
||||
}
|
||||
|
||||
sigIndex := semicolonIndex + len(chunkHdrStr)
|
||||
sigEndIndex := bytes.Index(header[sigIndex:], []byte(chunkHdrDelim))
|
||||
if sigEndIndex == -1 {
|
||||
cr.stash = make([]byte, len(header))
|
||||
copy(cr.stash, header)
|
||||
cr.trailerExpected = 0
|
||||
return 0, "", 0, errskipHeader
|
||||
}
|
||||
|
||||
chunkSizeBytes := header[:semicolonIndex]
|
||||
chunkSize, err := strconv.ParseInt(string(chunkSizeBytes), 16, 64)
|
||||
if err != nil {
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
|
||||
signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
|
||||
dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
|
||||
|
||||
return chunkSize, signature, dataStartOffset - stashLen, nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user